feat: support search table, column

This commit is contained in:
vran
2022-05-29 17:30:01 +08:00
parent e9a07b1098
commit 497ad6b58a
32 changed files with 2424 additions and 67 deletions

View File

@@ -28,4 +28,16 @@ public class AsyncConfig implements AsyncConfigurer {
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
return executor;
}
@Bean
public Executor fullTextRefreshThreadPoolTaskExecutor() {
final int maxCorePoolSize = 8;
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
int availableProcessorCount = Runtime.getRuntime().availableProcessors();
int corePoolSize = Math.min(maxCorePoolSize, availableProcessorCount);
executor.setCorePoolSize(corePoolSize);
executor.setAllowCoreThreadTimeOut(true);
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
return executor;
}
}

View File

@@ -0,0 +1,36 @@
package com.databasir.core.domain.document.converter;
import com.databasir.dao.tables.pojos.*;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
@Mapper(componentModel = "spring")
public interface DocumentFullTextPojoConverter {
/**
* groupName、groupDescription, projectName, projectDescription 等信息需要动态获取,所以不保存
*/
@Mapping(target = "groupId", source = "group.id")
@Mapping(target = "projectId", source = "db.projectId")
@Mapping(target = "databaseDocumentId", source = "db.id")
@Mapping(target = "databaseDocumentVersion", source = "db.version")
@Mapping(target = "databaseProductName", source = "db.productName")
@Mapping(target = "tableDocumentId", source = "table.id")
@Mapping(target = "tableColumnDocumentId", source = "column.id")
@Mapping(target = "tableName", source = "table.name")
@Mapping(target = "tableComment", source = "table.comment")
@Mapping(target = "colName", source = "column.name")
@Mapping(target = "colComment", source = "column.comment")
@Mapping(target = "id", ignore = true)
@Mapping(target = "createAt", ignore = true)
@Mapping(target = "updateAt", ignore = true)
@Mapping(target = "groupName", ignore = true)
@Mapping(target = "groupDescription", ignore = true)
@Mapping(target = "projectName", ignore = true)
@Mapping(target = "projectDescription", ignore = true)
DocumentFullTextPojo toPojo(GroupPojo group,
ProjectPojo project,
DatabaseDocumentPojo db,
TableDocumentPojo table,
TableColumnDocumentPojo column);
}

View File

@@ -49,6 +49,8 @@ import static java.util.Collections.emptyList;
@Slf4j
public class DocumentService {
private final GroupDao groupDao;
private final ProjectDao projectDao;
private final ProjectSyncRuleDao projectSyncRuleDao;
@@ -73,10 +75,14 @@ public class DocumentService {
private final DocumentDiscussionDao documentDiscussionDao;
private final DocumentFullTextDao documentFullTextDao;
private final DocumentDescriptionDao documentDescriptionDao;
private final DocumentPojoConverter documentPojoConverter;
private final DocumentFullTextPojoConverter documentFullTextPojoConverter;
private final DocumentResponseConverter documentResponseConverter;
private final DocumentSimpleResponseConverter documentSimpleResponseConverter;
@@ -166,10 +172,11 @@ public class DocumentService {
Long version,
Integer projectId) {
var pojo = documentPojoConverter.toDatabasePojo(projectId, meta, version);
var dbDocPojo = documentPojoConverter.toDatabasePojo(projectId, meta, version);
final Integer docId;
try {
docId = databaseDocumentDao.insertAndReturnId(pojo);
docId = databaseDocumentDao.insertAndReturnId(dbDocPojo);
dbDocPojo.setId(docId);
} catch (DuplicateKeyException e) {
log.warn("ignore insert database document projectId={} version={}", projectId, version);
throw new DatabasirException(DomainErrors.DATABASE_DOCUMENT_DUPLICATE_KEY);
@@ -178,6 +185,7 @@ public class DocumentService {
TableDocumentPojo tableMeta =
documentPojoConverter.toTablePojo(docId, table);
Integer tableMetaId = tableDocumentDao.insertAndReturnId(tableMeta);
tableMeta.setId(tableMetaId);
// column
var columns = documentPojoConverter.toColumnPojo(docId, tableMetaId, table.getColumns());
tableColumnDocumentDao.batchInsert(columns);
@@ -201,11 +209,28 @@ public class DocumentService {
// trigger
var triggers = documentPojoConverter.toTriggerPojo(docId, tableMetaId, table.getTriggers());
tableTriggerDocumentDao.batchInsert(triggers);
// save full text
saveDocumentFullText(projectId, dbDocPojo, tableMeta);
});
log.info("save new version document success: projectId = {}, name = {}, version = {}",
projectId, meta.getDatabaseName(), version);
}
private void saveDocumentFullText(Integer projectId,
DatabaseDocumentPojo database,
TableDocumentPojo table) {
ProjectPojo project = projectDao.selectById(projectId);
GroupPojo group = groupDao.selectById(project.getGroupId());
List<TableColumnDocumentPojo> columns = tableColumnDocumentDao.selectByTableDocumentId(table.getId());
// clear outdated data before save
documentFullTextDao.deleteByTableId(table.getId());
List<DocumentFullTextPojo> fullTextPojoList = columns.stream()
.map(column -> documentFullTextPojoConverter.toPojo(group, project, database, table, column))
.collect(Collectors.toList());
documentFullTextDao.batchInsert(fullTextPojoList);
}
public Optional<DatabaseDocumentSimpleResponse> getSimpleOneByProjectId(Integer projectId,
Long version,
Long originalVersion) {

View File

@@ -0,0 +1,17 @@
package com.databasir.core.domain.group.event;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class GroupCreated {
private Integer groupId;
private String groupName;
private String groupDescription;
}

View File

@@ -0,0 +1,12 @@
package com.databasir.core.domain.group.event;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class GroupDeleted {
private Integer groupId;
}

View File

@@ -0,0 +1,17 @@
package com.databasir.core.domain.group.event;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class GroupUpdated {
private Integer groupId;
private String groupName;
private String groupDescription;
}

View File

@@ -4,6 +4,10 @@ import com.databasir.core.domain.DomainErrors;
import com.databasir.core.domain.group.converter.GroupPojoConverter;
import com.databasir.core.domain.group.converter.GroupResponseConverter;
import com.databasir.core.domain.group.data.*;
import com.databasir.core.domain.group.event.GroupCreated;
import com.databasir.core.domain.group.event.GroupDeleted;
import com.databasir.core.domain.group.event.GroupUpdated;
import com.databasir.core.infrastructure.event.EventPublisher;
import com.databasir.dao.impl.*;
import com.databasir.dao.tables.pojos.GroupPojo;
import com.databasir.dao.tables.pojos.UserPojo;
@@ -34,6 +38,8 @@ public class GroupService {
private final ProjectDao projectDao;
private final EventPublisher eventPublisher;
private final ProjectSyncRuleDao projectSyncRuleDao;
private final GroupPojoConverter groupPojoConverter;
@@ -55,6 +61,7 @@ public class GroupService {
})
.collect(Collectors.toList());
userRoleDao.batchInsert(roles);
eventPublisher.publish(new GroupCreated(groupId, request.getName(), request.getDescription()));
return groupId;
}
@@ -74,6 +81,7 @@ public class GroupService {
})
.collect(Collectors.toList());
userRoleDao.batchInsert(roles);
eventPublisher.publish(new GroupUpdated(request.getId(), request.getName(), request.getDescription()));
}
public void delete(Integer groupId) {
@@ -82,6 +90,7 @@ public class GroupService {
List<Integer> projectIds = projectDao.selectProjectIdsByGroupId(groupId);
projectSyncRuleDao.disableAutoSyncByProjectIds(projectIds);
projectDao.deleteByGroupId(groupId);
eventPublisher.publish(new GroupDeleted(groupId));
}
public Page<GroupPageResponse> list(Pageable pageable, GroupPageCondition condition) {

View File

@@ -0,0 +1,12 @@
package com.databasir.core.domain.project.event;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class ProjectDeleted {
private Integer projectId;
}

View File

@@ -0,0 +1,24 @@
package com.databasir.core.domain.project.event;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class ProjectSaved {
private Integer groupId;
private Integer projectId;
private String projectName;
private String projectDescription;
private String databaseType;
private String databaseName;
private String schemaName;
}

View File

@@ -9,7 +9,10 @@ import com.databasir.core.domain.project.converter.ProjectSimpleTaskResponseConv
import com.databasir.core.domain.project.data.*;
import com.databasir.core.domain.project.data.task.ProjectSimpleTaskResponse;
import com.databasir.core.domain.project.data.task.ProjectTaskListCondition;
import com.databasir.core.domain.project.event.ProjectDeleted;
import com.databasir.core.domain.project.event.ProjectSaved;
import com.databasir.core.infrastructure.connection.DatabaseConnectionService;
import com.databasir.core.infrastructure.event.EventPublisher;
import com.databasir.dao.enums.ProjectSyncTaskStatus;
import com.databasir.dao.impl.*;
import com.databasir.dao.tables.pojos.*;
@@ -55,6 +58,8 @@ public class ProjectService {
private final ProjectSimpleTaskResponseConverter projectSimpleTaskResponseConverter;
private final EventPublisher eventPublisher;
public ProjectDetailResponse getOne(Integer id) {
return projectDao.selectOptionalById(id)
.map(schemaSource -> {
@@ -88,6 +93,17 @@ public class ProjectService {
ProjectSyncRulePojo syncRule = projectPojoConverter.of(request.getProjectSyncRule(), projectId);
projectSyncRuleDao.insertAndReturnId(syncRule);
var event = ProjectSaved.builder()
.groupId(project.getGroupId())
.projectId(projectId)
.projectName(project.getName())
.projectDescription(project.getDescription())
.databaseType(request.getDataSource().getDatabaseType())
.databaseName(dataSource.getDatabaseName())
.schemaName(dataSource.getSchemaName())
.build();
eventPublisher.publish(event);
return projectId;
}
@@ -119,6 +135,17 @@ public class ProjectService {
// update project info
ProjectPojo project = projectPojoConverter.of(request);
projectDao.updateById(project);
ProjectSaved event = ProjectSaved.builder()
.groupId(project.getGroupId())
.projectId(project.getId())
.projectName(project.getName())
.projectDescription(project.getDescription())
.databaseType(request.getDataSource().getDatabaseType())
.databaseName(dataSource.getDatabaseName())
.schemaName(dataSource.getSchemaName())
.build();
eventPublisher.publish(event);
} else {
throw DomainErrors.PROJECT_NOT_FOUND.exception();
}
@@ -137,6 +164,7 @@ public class ProjectService {
public void delete(Integer projectId) {
projectDao.updateDeletedById(true, projectId);
projectSyncRuleDao.disableAutoSyncByProjectId(projectId);
eventPublisher.publish(new ProjectDeleted(projectId));
}
public Page<ProjectSimpleResponse> list(Integer userId, Pageable page, ProjectListCondition condition) {

View File

@@ -2,32 +2,72 @@ package com.databasir.core.domain.search;
import com.databasir.core.domain.search.converter.SearchResponseConverter;
import com.databasir.core.domain.search.data.SearchResponse;
import com.databasir.dao.impl.DocumentFullTextDao;
import com.databasir.dao.impl.GroupDao;
import com.databasir.dao.impl.ProjectDao;
import com.databasir.dao.tables.pojos.DocumentFullTextPojo;
import com.databasir.dao.tables.pojos.GroupPojo;
import com.databasir.dao.tables.pojos.ProjectPojo;
import lombok.RequiredArgsConstructor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class SearchService {
private final ProjectDao projectDao;
private final SearchResponseConverter searchResponseConverter;
private final DocumentFullTextDao documentFullTextDao;
private final GroupDao groupDao;
private final SearchResponseConverter searchResponseConverter;
private final ProjectDao projectDao;
public SearchResponse search(String query) {
var groupPojoList = groupDao.selectByName(query);
var groupResults = searchResponseConverter.toGroupResults(groupPojoList);
var projectList = projectDao.selectByProjectNameOrDatabaseOrSchemaOrGroup(query);
var projectResults = searchResponseConverter.toProjectResults(projectList);
public SearchResponse search(Pageable pageable, String query) {
Page<DocumentFullTextPojo> columnPageData = documentFullTextDao.selectColumnPage(pageable, query);
Page<DocumentFullTextPojo> tablePageData = documentFullTextDao.selectTablePage(pageable, query);
// table 和 column 的项目名、组名等信息需要从关联表取
Set<Integer> projectIds = new HashSet<>();
projectIds.addAll(columnPageData.getContent()
.stream().map(o -> o.getProjectId()).collect(Collectors.toList()));
projectIds.addAll(tablePageData.getContent()
.stream().map(o -> o.getProjectId()).collect(Collectors.toList()));
Map<Integer, ProjectPojo> projectMapById = projectDao.selectInIds(projectIds)
.stream()
.collect(Collectors.toMap(o -> o.getId(), o -> o));
Page<DocumentFullTextPojo> projectPageData = documentFullTextDao.selectProjectPage(pageable, query);
Set<Integer> groupIds = new HashSet<>();
groupIds.addAll(columnPageData.getContent()
.stream().map(o -> o.getGroupId()).collect(Collectors.toList()));
groupIds.addAll(tablePageData.getContent()
.stream().map(o -> o.getGroupId()).collect(Collectors.toList()));
groupIds.addAll(projectPageData.getContent()
.stream().map(o -> o.getGroupId()).collect(Collectors.toList()));
Map<Integer, GroupPojo> groupMapById = groupDao.selectInIds(groupIds)
.stream()
.collect(Collectors.toMap(o -> o.getId(), o -> o));
// convert
var columns = columnPageData.map(item -> searchResponseConverter.toItem(item, projectMapById, groupMapById));
var tables = tablePageData.map(item -> searchResponseConverter.toItem(item, projectMapById, groupMapById));
var projects = projectPageData.map(item -> searchResponseConverter.toItem(item, groupMapById));
var groups = documentFullTextDao.selectGroupPage(pageable, query)
.map(searchResponseConverter::toItem);
// build response
SearchResponse response = new SearchResponse();
response.setGroups(groupResults);
response.setProjects(projectResults);
// TODO support Table search
response.setColumnPageData(columns);
response.setProjectPageData(projects);
response.setTablePageData(tables);
response.setGroupPageData(groups);
return response;
}
}

View File

@@ -1,16 +1,44 @@
package com.databasir.core.domain.search.converter;
import com.databasir.core.domain.search.data.SearchResponse;
import com.databasir.dao.tables.pojos.DocumentFullTextPojo;
import com.databasir.dao.tables.pojos.GroupPojo;
import com.databasir.dao.value.ProjectQueryPojo;
import com.databasir.dao.tables.pojos.ProjectPojo;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import java.util.List;
import java.util.Map;
@Mapper(componentModel = "spring")
public interface SearchResponseConverter {
List<SearchResponse.GroupSearchResult> toGroupResults(List<GroupPojo> groups);
default SearchResponse.Item toItem(DocumentFullTextPojo pojo,
Map<Integer, ProjectPojo> projectMapById,
Map<Integer, GroupPojo> groupMapById) {
ProjectPojo project = projectMapById.get(pojo.getProjectId());
GroupPojo group = groupMapById.get(pojo.getGroupId());
return toItem(pojo, group.getName(), group.getDescription(), project.getName(), project.getDescription());
}
List<SearchResponse.ProjectSearchResult> toProjectResults(List<ProjectQueryPojo> projects);
default SearchResponse.Item toItem(DocumentFullTextPojo pojo,
Map<Integer, GroupPojo> groupMapById) {
var group = groupMapById.get(pojo.getGroupId());
return toItem(pojo,
group.getName(),
group.getDescription(),
pojo.getProjectName(),
pojo.getProjectDescription());
}
@Mapping(target = "groupName", source = "groupName")
@Mapping(target = "groupDescription", source = "groupDescription")
@Mapping(target = "projectName", source = "projectName")
@Mapping(target = "projectDescription", source = "projectDescription")
SearchResponse.Item toItem(DocumentFullTextPojo item,
String groupName,
String groupDescription,
String projectName,
String projectDescription);
SearchResponse.Item toItem(DocumentFullTextPojo pojo);
}

View File

@@ -4,9 +4,7 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Collections;
import java.util.List;
import org.springframework.data.domain.Page;
@Data
@NoArgsConstructor
@@ -14,31 +12,33 @@ import java.util.List;
@Builder
public class SearchResponse {
@Builder.Default
private List<GroupSearchResult> groups = Collections.emptyList();
private Page<Item> groupPageData;
@Builder.Default
private List<ProjectSearchResult> projects = Collections.emptyList();
private Page<Item> projectPageData;
private Page<Item> tablePageData;
private Page<Item> columnPageData;
@Data
public static class GroupSearchResult {
private Integer id;
private String name;
private String description;
}
@Data
public static class ProjectSearchResult {
private Integer projectId;
public static class Item {
private Integer groupId;
private Integer projectId;
private Integer databaseDocumentId;
private Integer databaseDocumentVersion;
private Integer tableDocumentId;
private Integer tableColumnDocumentId;
private String groupName;
private String groupDescription;
private String projectName;
private String projectDescription;
@@ -47,5 +47,18 @@ public class SearchResponse {
private String schemaName;
private String databaseProductName;
private String databaseType;
private String tableName;
private String tableComment;
private String colName;
private String colComment;
}
}

View File

@@ -0,0 +1,61 @@
package com.databasir.core.infrastructure.event.subscriber;
import com.databasir.core.domain.group.event.GroupCreated;
import com.databasir.core.domain.group.event.GroupDeleted;
import com.databasir.core.domain.group.event.GroupUpdated;
import com.databasir.dao.Tables;
import com.databasir.dao.impl.DocumentFullTextDao;
import com.databasir.dao.tables.pojos.DocumentFullTextPojo;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
@Component
@RequiredArgsConstructor
@Slf4j
@Async("fullTextRefreshThreadPoolTaskExecutor")
public class GroupEventSubscriber {
private final DocumentFullTextDao documentFullTextDao;
@EventListener(classes = GroupUpdated.class)
public void refreshFullTextWhenUpdated(GroupUpdated event) {
if (!documentFullTextDao.exists(Tables.DOCUMENT_FULL_TEXT.GROUP_ID.eq(event.getGroupId())
.and(Tables.DOCUMENT_FULL_TEXT.PROJECT_ID.isNull()))) {
DocumentFullTextPojo pojo = new DocumentFullTextPojo();
pojo.setGroupId(event.getGroupId());
pojo.setGroupName(event.getGroupName());
pojo.setGroupDescription(event.getGroupDescription());
documentFullTextDao.insertAndReturnId(pojo);
log.info("group not exists, save new full text by event({}) success", event);
} else {
int result = documentFullTextDao.updateGroupInfoByGroupId(event.getGroupName(),
event.getGroupDescription(),
event.getGroupId());
log.info("update full text group({}) info success, effect rows {}", event.getGroupId(), result);
}
}
@EventListener(classes = GroupDeleted.class)
public void deleteFullTextWhenDeleted(GroupDeleted event) {
int result = documentFullTextDao.deleteByGroupId(event.getGroupId());
log.info("delete full text by group({}) success, effect rows {}", event.getGroupId(), result);
}
@EventListener(classes = GroupCreated.class)
public void addFullTextWhenCreated(GroupCreated event) {
if (!documentFullTextDao.exists(Tables.DOCUMENT_FULL_TEXT.GROUP_ID.eq(event.getGroupId())
.and(Tables.DOCUMENT_FULL_TEXT.PROJECT_ID.isNull()))) {
DocumentFullTextPojo pojo = new DocumentFullTextPojo();
pojo.setGroupId(event.getGroupId());
pojo.setGroupName(event.getGroupName());
pojo.setGroupDescription(event.getGroupDescription());
documentFullTextDao.insertAndReturnId(pojo);
log.info("save full text by event({}) success", event);
} else {
log.warn("ignore event {} because document already exists", event);
}
}
}

View File

@@ -0,0 +1,59 @@
package com.databasir.core.infrastructure.event.subscriber;
import com.databasir.core.domain.project.event.ProjectDeleted;
import com.databasir.core.domain.project.event.ProjectSaved;
import com.databasir.dao.Tables;
import com.databasir.dao.impl.DocumentFullTextDao;
import com.databasir.dao.tables.pojos.DocumentFullTextPojo;
import com.databasir.dao.value.FullTextProjectInfoUpdatePojo;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
@Component
@RequiredArgsConstructor
@Slf4j
@Async("fullTextRefreshThreadPoolTaskExecutor")
public class ProjectEventSubscriber {
private final DocumentFullTextDao documentFullTextDao;
@EventListener(classes = ProjectSaved.class)
@Transactional
public void refreshFullTextWhenUpdated(ProjectSaved event) {
if (!documentFullTextDao.exists(Tables.DOCUMENT_FULL_TEXT.PROJECT_ID.eq(event.getProjectId())
.and(Tables.DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID.isNull()))) {
DocumentFullTextPojo pojo = new DocumentFullTextPojo();
pojo.setGroupId(event.getGroupId());
pojo.setProjectId(event.getProjectId());
pojo.setProjectName(event.getProjectName());
pojo.setProjectDescription(event.getProjectDescription());
pojo.setDatabaseName(event.getDatabaseName());
pojo.setSchemaName(event.getSchemaName());
pojo.setDatabaseType(event.getDatabaseType());
documentFullTextDao.insertAndReturnId(pojo);
log.info("save full text by event ({}) success", event);
} else {
FullTextProjectInfoUpdatePojo updatePojo = FullTextProjectInfoUpdatePojo.builder()
.projectId(event.getProjectId())
.projectName(event.getProjectName())
.projectDescription(event.getProjectDescription())
.databaseType(event.getDatabaseType())
.databaseName(event.getDatabaseName())
.schemaName(event.getSchemaName())
.build();
int result = documentFullTextDao.updateProjectInfoByProjectId(updatePojo);
log.info("update full text project({}) info success, effect rows {}", event, result);
}
}
@EventListener(classes = ProjectDeleted.class)
public void deleteFullTextWhenDeleted(ProjectDeleted event) {
int result = documentFullTextDao.deleteByProjectId(event.getProjectId());
log.info("delete full text by project({}) success, effect rows {}", event.getProjectId(), result);
}
}