7x31/databasir
1
0
Fork 0
mirror of https://github.com/vran-dev/databasir.git synced 2025-09-13 22:57:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support search table, column

Browse Source
This commit is contained in:
vran
2022-05-29 17:30:01 +08:00
parent e9a07b1098
commit 497ad6b58a
32 changed files with 2424 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dao/src/main/java/com/databasir/dao/impl/BaseDao.java
View File

@@ -100,7 +100,7 @@ public abstract class BaseDao<R> {
+ condition));
}
public List<R> selectInIds(List<? extends Serializable> ids) {
public List<R> selectInIds(Collection<? extends Serializable> ids) {
if (ids == null || ids.isEmpty()) {
return Collections.emptyList();
}

201
dao/src/main/java/com/databasir/dao/impl/DocumentFullTextDao.java Normal file
View File

@@ -0,0 +1,201 @@
package com.databasir.dao.impl;
import com.databasir.dao.tables.pojos.DocumentFullTextPojo;
import com.databasir.dao.value.FullTextProjectInfoUpdatePojo;
import lombok.Getter;
import org.jooq.DSLContext;
import org.jooq.TableField;
import org.jooq.impl.DSL;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Objects;
import static com.databasir.dao.Tables.DOCUMENT_FULL_TEXT;
@Repository
public class DocumentFullTextDao extends BaseDao<DocumentFullTextPojo> {
public static final String[] EMPTY = new String[0];
@Autowired
@Getter
private DSLContext dslContext;
public DocumentFullTextDao() {
super(DOCUMENT_FULL_TEXT, DocumentFullTextPojo.class);
}
public Page<DocumentFullTextPojo> selectColumnPage(Pageable pageable, String keyword) {
String[] fullTextMatchCols = new String[]{
DOCUMENT_FULL_TEXT.COL_NAME.getName(),
DOCUMENT_FULL_TEXT.COL_COMMENT.getName(),
DOCUMENT_FULL_TEXT.DATABASE_PRODUCT_NAME.getName()
};
String colSegment = String.join(",", fullTextMatchCols);
// sample: match(col_name, col_comment) against('+databasir' in boolean mode)
String fullTextMatchSqlSegment = new StringBuilder(64)
.append("MATCH(").append(colSegment).append(") ")
.append("AGAINST('+").append(keyword).append("' IN BOOLEAN MODE)")
.toString();
// count
Integer total = dslContext.selectDistinct(DSL.count(DOCUMENT_FULL_TEXT.TABLE_COLUMN_DOCUMENT_ID))
.from(DOCUMENT_FULL_TEXT)
.where(DOCUMENT_FULL_TEXT.TABLE_COLUMN_DOCUMENT_ID.isNotNull().and(fullTextMatchSqlSegment))
.fetchOne(0, int.class);
// content
List<DocumentFullTextPojo> content = dslContext.select(DOCUMENT_FULL_TEXT.fields())
.from(DOCUMENT_FULL_TEXT)
.where(DOCUMENT_FULL_TEXT.TABLE_COLUMN_DOCUMENT_ID.isNotNull().and(fullTextMatchSqlSegment))
.limit(pageable.getOffset(), pageable.getPageSize())
.fetchInto(DocumentFullTextPojo.class);
return new PageImpl<>(content, pageable, total.longValue());
}
public Page<DocumentFullTextPojo> selectTablePage(Pageable pageable, String keyword) {
String[] matchCols = new String[]{
DOCUMENT_FULL_TEXT.TABLE_NAME.getName(),
DOCUMENT_FULL_TEXT.TABLE_COMMENT.getName(),
DOCUMENT_FULL_TEXT.DATABASE_PRODUCT_NAME.getName()
};
TableField[] groupColumns = new TableField[]{
DOCUMENT_FULL_TEXT.GROUP_ID,
DOCUMENT_FULL_TEXT.GROUP_NAME,
DOCUMENT_FULL_TEXT.GROUP_DESCRIPTION,
DOCUMENT_FULL_TEXT.PROJECT_ID,
DOCUMENT_FULL_TEXT.PROJECT_NAME,
DOCUMENT_FULL_TEXT.PROJECT_DESCRIPTION,
DOCUMENT_FULL_TEXT.DATABASE_NAME,
DOCUMENT_FULL_TEXT.SCHEMA_NAME,
DOCUMENT_FULL_TEXT.DATABASE_PRODUCT_NAME,
DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID,
DOCUMENT_FULL_TEXT.TABLE_NAME,
DOCUMENT_FULL_TEXT.TABLE_COMMENT,
};
String colSegment = String.join(",", matchCols);
// sample: match(col_name, col_comment) against('+databasir' in boolean mode)
String fullTextMatchSqlSegment = new StringBuilder(64)
.append("MATCH(").append(colSegment).append(") ")
.append("AGAINST('+").append(keyword).append("' IN BOOLEAN MODE)")
.toString();
// count
Integer total = dslContext.selectDistinct(DSL.count(DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID))
.from(DOCUMENT_FULL_TEXT)
.where(DOCUMENT_FULL_TEXT.PROJECT_ID.isNotNull()
.and(DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID.isNull())
.and(fullTextMatchSqlSegment))
.fetchOne(0, int.class);
// content
List<DocumentFullTextPojo> content = dslContext.select(groupColumns)
.from(DOCUMENT_FULL_TEXT)
.where(DOCUMENT_FULL_TEXT.PROJECT_ID.isNotNull()
.and(DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID.isNotNull())
.and(fullTextMatchSqlSegment))
.groupBy(groupColumns)
.limit(pageable.getOffset(), pageable.getPageSize())
.fetchInto(DocumentFullTextPojo.class);
return new PageImpl<>(content, pageable, total.longValue());
}
public Page<DocumentFullTextPojo> selectProjectPage(Pageable pageable, String keyword) {
String[] matchCols = new String[]{
DOCUMENT_FULL_TEXT.PROJECT_NAME.getName(),
DOCUMENT_FULL_TEXT.PROJECT_DESCRIPTION.getName(),
DOCUMENT_FULL_TEXT.SCHEMA_NAME.getName(),
DOCUMENT_FULL_TEXT.DATABASE_NAME.getName(),
DOCUMENT_FULL_TEXT.DATABASE_TYPE.getName(),
};
String colSegment = String.join(",", matchCols);
// sample: match(col_name, col_comment) against('+databasir' in boolean mode)
String fullTextMatchSqlSegment = new StringBuilder(64)
.append("MATCH(").append(colSegment).append(") ")
.append("AGAINST('+").append(keyword).append("' IN BOOLEAN MODE)")
.toString();
// count
Integer total = dslContext.selectDistinct(DSL.count(DOCUMENT_FULL_TEXT.PROJECT_ID))
.from(DOCUMENT_FULL_TEXT)
.where(DOCUMENT_FULL_TEXT.PROJECT_ID.isNotNull()
.and(DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID.isNull())
.and(fullTextMatchSqlSegment))
.fetchOne(0, int.class);
// content
List<DocumentFullTextPojo> content = dslContext.select(DOCUMENT_FULL_TEXT.fields())
.from(DOCUMENT_FULL_TEXT)
.where(DOCUMENT_FULL_TEXT.PROJECT_ID.isNotNull()
.and(DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID.isNull())
.and(fullTextMatchSqlSegment))
.limit(pageable.getOffset(), pageable.getPageSize())
.fetchInto(DocumentFullTextPojo.class);
return new PageImpl<>(content, pageable, total.longValue());
}
public Page<DocumentFullTextPojo> selectGroupPage(Pageable pageable, String keyword) {
String[] matchCols = new String[]{
DOCUMENT_FULL_TEXT.GROUP_NAME.getName(),
DOCUMENT_FULL_TEXT.GROUP_DESCRIPTION.getName()
};
String colSegment = String.join(",", matchCols);
String fullTextMatchSqlSegment = new StringBuilder(64)
.append("MATCH(").append(colSegment).append(") ")
.append("AGAINST('+").append(keyword).append("' IN BOOLEAN MODE)")
.toString();
// count
Integer total = dslContext.selectDistinct(DSL.count(DOCUMENT_FULL_TEXT.GROUP_ID))
.from(DOCUMENT_FULL_TEXT)
.where(DOCUMENT_FULL_TEXT.GROUP_ID.isNotNull()
.and(DOCUMENT_FULL_TEXT.PROJECT_ID.isNull())
.and(fullTextMatchSqlSegment))
.fetchOne(0, int.class);
// content
List<DocumentFullTextPojo> content = dslContext.select(DOCUMENT_FULL_TEXT.fields())
.from(DOCUMENT_FULL_TEXT)
.where(DOCUMENT_FULL_TEXT.GROUP_ID.isNotNull()
.and(DOCUMENT_FULL_TEXT.PROJECT_ID.isNull())
.and(fullTextMatchSqlSegment))
.limit(pageable.getOffset(), pageable.getPageSize())
.fetchInto(DocumentFullTextPojo.class);
return new PageImpl<>(content, pageable, total.longValue());
}
public int deleteByTableId(Integer tableDocumentId) {
return this.delete(DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID.eq(tableDocumentId));
}
public int deleteByGroupId(Integer groupId) {
return this.delete(DOCUMENT_FULL_TEXT.GROUP_ID.eq(groupId));
}
public int deleteByProjectId(Integer projectId) {
return this.delete(DOCUMENT_FULL_TEXT.PROJECT_ID.eq(projectId));
}
public int updateGroupInfoByGroupId(String groupName, String groupDescription, Integer groupId) {
String description = Objects.requireNonNullElse(groupDescription, "");
return this.getDslContext()
.update(DOCUMENT_FULL_TEXT)
.set(DOCUMENT_FULL_TEXT.GROUP_NAME, groupName)
.set(DOCUMENT_FULL_TEXT.GROUP_DESCRIPTION, description)
.where(DOCUMENT_FULL_TEXT.GROUP_ID.eq(groupId).and(DOCUMENT_FULL_TEXT.PROJECT_ID.isNull()))
.execute();
}
public int updateProjectInfoByProjectId(FullTextProjectInfoUpdatePojo updatePojo) {
return this.getDslContext()
.update(DOCUMENT_FULL_TEXT)
.set(DOCUMENT_FULL_TEXT.PROJECT_NAME, updatePojo.getProjectName())
.set(DOCUMENT_FULL_TEXT.PROJECT_DESCRIPTION, updatePojo.getProjectDescription())
.set(DOCUMENT_FULL_TEXT.DATABASE_NAME, updatePojo.getDatabaseName())
.set(DOCUMENT_FULL_TEXT.SCHEMA_NAME, updatePojo.getSchemaName())
.set(DOCUMENT_FULL_TEXT.DATABASE_TYPE, updatePojo.getDatabaseType())
.where(DOCUMENT_FULL_TEXT.PROJECT_ID.eq(updatePojo.getProjectId())
.and(DOCUMENT_FULL_TEXT.TABLE_DOCUMENT_ID.isNull()))
.execute();
}
}

3
dao/src/main/java/com/databasir/dao/impl/GroupDao.java
View File

@@ -10,6 +10,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Repository;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -47,7 +48,7 @@ public class GroupDao extends BaseDao<GroupPojo> {
}
@Override
public List<GroupPojo> selectInIds(List<? extends Serializable> ids) {
public List<GroupPojo> selectInIds(Collection<? extends Serializable> ids) {
if (ids == null || ids.isEmpty()) {
return Collections.emptyList();
}

27
dao/src/main/java/com/databasir/dao/impl/ProjectDao.java
View File

@@ -2,7 +2,6 @@ package com.databasir.dao.impl;
import com.databasir.dao.tables.pojos.ProjectPojo;
import com.databasir.dao.value.GroupProjectCountPojo;
import com.databasir.dao.value.ProjectQueryPojo;
import lombok.Getter;
import org.jooq.Condition;
import org.jooq.DSLContext;
@@ -20,7 +19,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import static com.databasir.dao.Tables.*;
import static com.databasir.dao.Tables.DATA_SOURCE;
import static com.databasir.dao.Tables.PROJECT;
@Repository
public class ProjectDao extends BaseDao<ProjectPojo> {
@@ -127,27 +127,4 @@ public class ProjectDao extends BaseDao<ProjectPojo> {
.fetchMap(PROJECT.ID, PROJECT.GROUP_ID);
}
public List<ProjectQueryPojo> selectByProjectNameOrDatabaseOrSchemaOrGroup(String query) {
return getDslContext()
.select(
PROJECT.ID.as("project_id"),
PROJECT.NAME.as("project_name"),
PROJECT.DESCRIPTION.as("project_description"),
DATA_SOURCE.DATABASE_NAME,
DATA_SOURCE.SCHEMA_NAME,
GROUP.NAME.as("group_name"),
GROUP.ID.as("group_id")
)
.from(PROJECT)
.leftJoin(DATA_SOURCE).on(DATA_SOURCE.PROJECT_ID.eq(PROJECT.ID))
.leftJoin(GROUP).on(GROUP.ID.eq(PROJECT.GROUP_ID))
.where(PROJECT.DELETED.eq(false)
.and(GROUP.DELETED.eq(false))
.and(PROJECT.NAME.contains(query)
.or(DATA_SOURCE.DATABASE_NAME.contains(query))
.or(DATA_SOURCE.SCHEMA_NAME.contains(query))
.or(GROUP.NAME.contains(query)))
)
.fetchInto(ProjectQueryPojo.class);
}
}

21
dao/src/main/java/com/databasir/dao/value/FullTextProjectInfoUpdatePojo.java Normal file
View File

@@ -0,0 +1,21 @@
package com.databasir.dao.value;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class FullTextProjectInfoUpdatePojo {
private Integer projectId;
private String projectName;
private String projectDescription;
private String databaseType;
private String databaseName;
private String schemaName;
}

36
dao/src/main/resources/db/init/init.sql
View File

@@ -378,4 +378,38 @@ CREATE TABLE IF NOT EXISTS project_sync_task
INDEX idx_project_id (project_id),
INDEX idx_user_id (user_id)
) CHARSET utf8mb4
COLLATE utf8mb4_unicode_ci;
COLLATE utf8mb4_unicode_ci;
CREATE TABLE document_full_text
(
id INT PRIMARY KEY AUTO_INCREMENT,
`group_id` INT NOT NULL COMMENT 'ref to group.id',
`project_id` INT COMMENT 'ref to project.id, may null',
`database_document_id` INT COMMENT 'ref to database_document.id, may null',
`database_document_version` INT COMMENT 'rf to database_document.version, may null',
`table_document_id` INT COMMENT 'ref to table_document.id, may null',
`table_column_document_id` INT COMMENT 'ref to table_column_document.id, may null',
`group_name` VARCHAR(255),
`group_description` VARCHAR(512),
`project_name` VARCHAR(255),
`project_description` TEXT,
`database_name` TEXT,
`schema_name` TEXT,
`database_product_name` TEXT,
`database_type` TEXT,
`table_name` TEXT,
`table_comment` TEXT,
`col_name` TEXT,
`col_comment` TEXT,
update_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
create_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX idx_group_id (group_id),
INDEX idx_project_id (project_id),
INDEX idx_table_document_id (table_document_id),
FULLTEXT fidx_column (col_name, col_comment, database_product_name) WITH PARSER ngram,
FULLTEXT fidx_table (`table_name`, table_comment, database_product_name) WITH PARSER ngram,
FULLTEXT fidx_project (project_name, project_description, schema_name, database_name,
database_type) WITH PARSER ngram,
FULLTEXT fidx_group (group_name, group_description) WITH PARSER ngram
) CHARSET utf8mb4
COLLATE utf8mb4_unicode_ci;

55
dao/src/main/resources/db/init/v1.0.5~v1.0.6 data migration.sql Normal file
View File

@@ -0,0 +1,55 @@
-- migration group info
INSERT INTO document_full_text(group_id, group_name, group_description)
SELECT `group`.id, `group`.name, `group`.description
FROM `group`
LEFT JOIN document_full_text dft ON `group`.id = dft.group_id
WHERE dft.group_id IS NULL
AND `group`.deleted = FALSE;
-- migration project info
INSERT INTO document_full_text(group_id, project_id, project_name, project_description, database_name, schema_name,
database_type)
SELECT project.group_id,
project.id,
project.name,
project.description,
ds.database_name,
ds.schema_name,
ds.database_type
FROM project
LEFT JOIN document_full_text ON project.id = document_full_text.project_id
LEFT JOIN `group` g ON project.group_id = g.id
LEFT JOIN data_source ds ON project.id = ds.project_id
WHERE project.deleted = FALSE
AND g.deleted = FALSE
AND document_full_text.table_document_id IS NULL
AND document_full_text.project_id IS NULL;
-- migration column
INSERT INTO document_full_text(group_id, project_id, database_document_id, database_document_version,
table_document_id, table_column_document_id, database_name, schema_name,
database_product_name, table_name, table_comment, col_name, col_comment)
SELECT pj.group_id,
pj.id,
dd.id,
dd.version,
td.id,
tcd.id,
ds.database_type,
ds.schema_name,
dd.product_name,
td.name,
td.comment,
tcd.name,
tcd.comment
FROM table_column_document tcd
LEFT JOIN document_full_text dft ON dft.table_column_document_id = tcd.id
INNER JOIN table_document td ON tcd.table_document_id = td.id
INNER JOIN database_document dd ON tcd.database_document_id = dd.id
INNER JOIN project pj ON dd.project_id = pj.id
INNER JOIN data_source ds ON pj.id = ds.project_id
WHERE pj.deleted = FALSE
AND dd.is_archive = FALSE
AND dft.table_column_document_id IS NULL
AND dft.project_id IS NULL

33
dao/src/main/resources/db/migration/V1.4.4__document_full_text.sql Normal file
View File

@@ -0,0 +1,33 @@
CREATE TABLE document_full_text
(
id INT PRIMARY KEY AUTO_INCREMENT,
`group_id` INT NOT NULL COMMENT 'ref to group.id',
`project_id` INT COMMENT 'ref to project.id, may null',
`database_document_id` INT COMMENT 'ref to database_document.id, may null',
`database_document_version` INT COMMENT 'rf to database_document.version, may null',
`table_document_id` INT COMMENT 'ref to table_document.id, may null',
`table_column_document_id` INT COMMENT 'ref to table_column_document.id, may null',
`group_name` VARCHAR(255),
`group_description` VARCHAR(512),
`project_name` VARCHAR(255),
`project_description` TEXT,
`database_name` TEXT,
`schema_name` TEXT,
`database_product_name` TEXT,
`database_type` TEXT,
`table_name` TEXT,
`table_comment` TEXT,
`col_name` TEXT,
`col_comment` TEXT,
update_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
create_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX IDX_GROUP_ID (group_id),
INDEX IDX_PROJECT_ID (project_id),
INDEX IDX_TABLE_DOCUMENT_ID (table_document_id),
FULLTEXT FIDX_COLUMN (col_name, col_comment, database_product_name) WITH PARSER ngram,
FULLTEXT FIDX_TABLE (`table_name`, table_comment, database_product_name) WITH PARSER ngram,
FULLTEXT FIDX_PROJECT (project_name, project_description, SCHEMA_NAME, database_name,
database_type) WITH PARSER ngram,
FULLTEXT FIDX_GROUP (group_name, group_description) WITH PARSER ngram
) CHARSET utf8mb4
COLLATE utf8mb4_unicode_ci;