From be9e4c8b61ebff1e4841a04b73fb118813e72acd Mon Sep 17 00:00:00 2001 From: mystarry-sky Date: Mon, 25 Dec 2023 18:38:31 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9F=BA=E7=A1=80=E5=85=83=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BF=A1=E6=81=AF-=E5=88=97=E4=BF=A1=E6=81=AF=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=96=B9=E5=BC=8F=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/log4j2.xml | 2 +- .../common/entry/extract/ColumnsMetaData.java | 39 ++++++++++++-- .../entry/extract/PrimaryColumnBean.java | 37 +++++++++++++ .../extract/data/BaseDataService.java | 44 +++++++++++++-- .../data/access/CsvDataAccessService.java | 41 ++++++++++---- .../data/access/DataAccessService.java | 16 ++++++ .../data/access/MysqlDataAccessService.java | 11 ++++ .../data/access/OpgsDataAccessService.java | 14 +++-- .../data/access/OracleDataAccessService.java | 11 ++++ .../extract/data/mapper/MetaDataMapper.java | 17 ++++++ .../data/mapper/OpgsMetaDataMapper.java | 13 ----- .../extract/service/MetaDataService.java | 12 +++-- .../extract/util/MetaDataUtil.java | 2 +- .../resources/mapper/MysqlMetaDataMapper.xml | 16 ++++++ .../resources/mapper/OpgsMetaDataMapper.xml | 54 +++++++------------ .../resources/mapper/OracleMetaDataMapper.xml | 26 +++++---- 16 files changed, 271 insertions(+), 84 deletions(-) create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/PrimaryColumnBean.java diff --git a/config/log4j2.xml b/config/log4j2.xml index 115b2bc..fc49d78 100644 --- a/config/log4j2.xml +++ b/config/log4j2.xml @@ -81,7 +81,7 @@ - + diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ColumnsMetaData.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ColumnsMetaData.java index ae3dbe5..6145094 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ColumnsMetaData.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ColumnsMetaData.java @@ -68,22 +68,53 @@ public class ColumnsMetaData implements Comparable { private String extra; + /** + * get column name and column type + * + * @return message + */ public String getColumnMsg() { return " [" + columnName + " : " + columnType + "]"; } + /** + * get column name and format name lowercase + * + * @return lowercase column name + */ + public String getLowerCaseColumnName() { + return columnName.toLowerCase(Locale.ENGLISH); + } + + /** + * check current column is auto increment column + * + * @return true | false + */ public boolean isAutoIncrementColumn() { if (columnKey == null || columnKey != ColumnKey.PRI) { return false; } - return !StringUtils.isEmpty(getExtra()) && getExtra().toLowerCase(Locale.ENGLISH).contains(AUTO_INCREMENT); + return !StringUtils.isEmpty(getExtra()) && getExtra().toLowerCase(Locale.ENGLISH) + .contains(AUTO_INCREMENT); } + /** + * pase resultSet metadata to ColumnsMetaData + * + * @param rs rs + * @return ColumnsMetaData + * @throws SQLException + */ public static ColumnsMetaData parse(ResultSet rs) throws SQLException { ColumnsMetaData columnsMetaData = new ColumnsMetaData(); - columnsMetaData.setTableName(rs.getString(1)).setColumnName(rs.getString(2)).setOrdinalPosition(rs.getInt(3)) - .setDataType(rs.getString(4)).setColumnType(rs.getString(5)) - .setColumnKey(EnumUtil.valueOf(ColumnKey.class, rs.getString(6))).setExtra(rs.getString(7)); + columnsMetaData.setTableName(rs.getString(1)) + .setColumnName(rs.getString(2)) + .setOrdinalPosition(rs.getInt(3)) + .setDataType(rs.getString(4)) + .setColumnType(rs.getString(5)) + .setColumnKey(EnumUtil.valueOf(ColumnKey.class, rs.getString(6))) + .setExtra(rs.getString(7)); return columnsMetaData; } diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/PrimaryColumnBean.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/PrimaryColumnBean.java new file mode 100644 index 0000000..651e26e --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/PrimaryColumnBean.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022-2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +package org.opengauss.datachecker.common.entry.extract; + +import lombok.Data; + +/** + * PrimaryColumnBean + * + * @author ：wangchao + * @date ：Created in 2023/12/23 + * @since ：11 + */ +@Data +public class PrimaryColumnBean { + /** + * Table + */ + private String tableName; + /** + * Primary key column name + */ + private String columnName; +} \ No newline at end of file diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java index 1eb4276..0becc5a 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java @@ -15,12 +15,12 @@ package org.opengauss.datachecker.extract.data; -import io.confluent.kafka.serializers.KafkaAvroDeserializerConfig; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.logging.log4j.Logger; import org.opengauss.datachecker.common.entry.enums.ColumnKey; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.common.util.LogUtils; import org.opengauss.datachecker.common.util.LongHashFunctionWrapper; @@ -32,6 +32,7 @@ import org.springframework.stereotype.Service; import javax.annotation.Resource; import javax.sql.DataSource; import java.util.Comparator; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -104,6 +105,20 @@ public class BaseDataService { .collect(Collectors.toList()); } + /** + * query current db of all table primary column + * + * @return + */ + public Map> queryTablePrimaryColumns() { + List columnBeanList = dataAccessService.queryTablePrimaryColumns(); + if (CollectionUtils.isEmpty(columnBeanList)) { + return new HashMap<>(); + } + return columnBeanList.stream() + .collect(Collectors.groupingBy(PrimaryColumnBean::getTableName)); + } + private List filterByTableRules(List tableNameList) { return ruleAdapterService.executeTableRule(tableNameList); } @@ -156,7 +171,7 @@ public class BaseDataService { if (Objects.isNull(tableMetadata)) { return tableMetadata; } - updateTableColumnMetaData(tableMetadata); + updateTableColumnMetaData(tableMetadata, null); log.debug("query table metadata {} -- {} ", tableName, tableMetadata); MetaDataCache.put(tableName, tableMetadata); return tableMetadata; @@ -165,19 +180,40 @@ public class BaseDataService { /** * update table metadata, and filter column rules * - * @param tableMetadata table metadata + * @param tableMetadata table metadata + * @param primaryColumnBeans primary column */ - public void updateTableColumnMetaData(TableMetadata tableMetadata) { + public void updateTableColumnMetaData(TableMetadata tableMetadata, List primaryColumnBeans) { String tableName = tableMetadata.getTableName(); final List columns = dataAccessService.queryTableColumnsMetaData(tableName); if (Objects.isNull(columns)) { log.error("table columns metadata is null ,{}", tableName); + return; } + if (Objects.isNull(primaryColumnBeans)) { + primaryColumnBeans = dataAccessService.queryTablePrimaryColumns(tableName); + } + if (Objects.nonNull(primaryColumnBeans)) { + List primaryColumnNameList = getPrimaryColumnNames(primaryColumnBeans); + for (ColumnsMetaData column : columns) { + if (primaryColumnNameList.contains(column.getLowerCaseColumnName())) { + column.setColumnKey(ColumnKey.PRI); + } + } + } + tableMetadata.setColumnsMetas(ruleAdapterService.executeColumnRule(columns)); tableMetadata.setPrimaryMetas(getTablePrimaryColumn(columns)); tableMetadata.setTableHash(calcTableHash(columns)); } + private List getPrimaryColumnNames(List primaryColumnBeans) { + return primaryColumnBeans.stream() + .map(PrimaryColumnBean::getColumnName) + .map(String::toLowerCase) + .collect(Collectors.toList()); + } + private List getTablePrimaryColumn(List columnsMetaData) { return columnsMetaData.stream() .filter(meta -> ColumnKey.PRI.equals(meta.getColumnKey())) diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/CsvDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/CsvDataAccessService.java index 6235dbe..ea51819 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/CsvDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/CsvDataAccessService.java @@ -26,6 +26,7 @@ import org.opengauss.datachecker.common.entry.csv.CsvTableColumnMeta; import org.opengauss.datachecker.common.entry.csv.CsvTableMeta; import org.opengauss.datachecker.common.entry.enums.ColumnKey; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.common.exception.CsvDataAccessException; import org.opengauss.datachecker.common.exception.ExtractDataAccessException; @@ -72,8 +73,10 @@ public class CsvDataAccessService implements DataAccessService { throw new CsvDataAccessException("csv metadata load failed"); } Stream lineOfTables = Files.lines(pathOfTables); - return lineOfTables.parallel().map(tableJson -> JSONObject.parseObject(tableJson, CsvTableMeta.class)) - .filter(CsvTableMeta::isContain_primary_key).map(CsvTableMeta::getTable) + return lineOfTables.parallel() + .map(tableJson -> JSONObject.parseObject(tableJson, CsvTableMeta.class)) + .filter(CsvTableMeta::isContain_primary_key) + .map(CsvTableMeta::getTable) .collect(Collectors.toList()); } catch (IOException e) { @@ -104,13 +107,17 @@ public class CsvDataAccessService implements DataAccessService { CsvTableColumnMeta csvColumnMeta = JSONObject.parseObject(columnJson, CsvTableColumnMeta.class); columns.add(csvColumnMeta.toColumnsMetaData()); }); - columns.stream().sorted().collect(Collectors.groupingBy(ColumnsMetaData::getTableName)) + columns.stream() + .sorted() + .collect(Collectors.groupingBy(ColumnsMetaData::getTableName)) .forEach((table, tableColumns) -> { TableMetadata tableMetadata = tableMetadataMap.get(table); tableMetadata.setColumnsMetas(tableColumns); - tableMetadata.setPrimaryMetas( - tableColumns.stream().filter(col -> Objects.equals(col.getColumnKey(), ColumnKey.PRI)) - .sorted().collect(Collectors.toList())); + tableMetadata.setPrimaryMetas(tableColumns.stream() + .filter(col -> Objects.equals(col.getColumnKey(), + ColumnKey.PRI)) + .sorted() + .collect(Collectors.toList())); }); } catch (IOException e) { log.error("load table name of csv exception : ", e); @@ -119,6 +126,16 @@ public class CsvDataAccessService implements DataAccessService { return new ArrayList<>(tableMetadataMap.values()); } + @Override + public List queryTablePrimaryColumns() { + return null; + } + + @Override + public List queryTablePrimaryColumns(String tableName) { + return null; + } + @Override public List queryTableColumnsMetaData(String tableName) { TableMetadata tableMetadata = queryTableMetadata(tableName); @@ -164,10 +181,13 @@ public class CsvDataAccessService implements DataAccessService { } List> diffRowList = new LinkedList<>(); String csvDataRootPath = ConfigCache.getCsvData(); - String sliceFilePath = Path.of(csvDataRootPath, fileName).toString(); + String sliceFilePath = Path.of(csvDataRootPath, fileName) + .toString(); TableMetadata metadata = tableMetadataMap.get(table); - List keyIdxList = - differenceList.stream().map(Difference::getIdx).sorted().collect(Collectors.toList()); + List keyIdxList = differenceList.stream() + .map(Difference::getIdx) + .sorted() + .collect(Collectors.toList()); int fileReadIdx = 0; try (CSVReader reader = new CSVReader(new FileReader(sliceFilePath))) { String[] nextLine; @@ -187,7 +207,8 @@ public class CsvDataAccessService implements DataAccessService { private Map parse(String[] nextLine, List columns) { Map result = new TreeMap<>(); for (int idx = 0; idx < nextLine.length && idx < columns.size(); idx++) { - result.put(columns.get(idx).getColumnName(), nextLine[idx]); + result.put(columns.get(idx) + .getColumnName(), nextLine[idx]); } return result; } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/DataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/DataAccessService.java index a46c90c..4ddeaf4 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/DataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/DataAccessService.java @@ -18,6 +18,7 @@ package org.opengauss.datachecker.extract.data.access; import org.opengauss.datachecker.common.entry.check.Difference; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.springframework.jdbc.core.RowMapper; @@ -57,6 +58,21 @@ public interface DataAccessService { */ List queryTableMetadataList(); + /** + * queryTablePrimaryColumns + * + * @return primary column + */ + List queryTablePrimaryColumns(); + + /** + * queryTablePrimaryColumns + * + * @param tableName tableName + * @return primary column + */ + List queryTablePrimaryColumns(String tableName); + /** * query table ColumnsMetaData * diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java index 8659950..3ac18d6 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java @@ -17,6 +17,7 @@ package org.opengauss.datachecker.extract.data.access; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.extract.data.mapper.MysqlMetaDataMapper; @@ -61,6 +62,16 @@ public class MysqlDataAccessService extends AbstractDataAccessService { return wrapperTableMetadata(mysqlMetaDataMapper.queryTableMetadata(properties.getSchema(), tableName)); } + @Override + public List queryTablePrimaryColumns() { + return mysqlMetaDataMapper.queryTablePrimaryColumns(properties.getSchema()); + } + + @Override + public List queryTablePrimaryColumns(String tableName) { + return mysqlMetaDataMapper.queryTablePrimaryColumnsByTableName(properties.getSchema(),tableName); + } + @Override public List queryTableMetadataList() { return wrapperTableMetadata(mysqlMetaDataMapper.queryTableMetadataList(properties.getSchema())); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java index 22874b4..8c068f9 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java @@ -18,6 +18,7 @@ package org.opengauss.datachecker.extract.data.access; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.enums.OgCompatibility; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.extract.data.mapper.OpgsMetaDataMapper; @@ -56,11 +57,18 @@ public class OpgsDataAccessService extends AbstractDataAccessService { return opgsMetaDataMapper.queryTableNameList(properties.getSchema()); } + @Override + public List queryTablePrimaryColumns() { + return opgsMetaDataMapper.queryTablePrimaryColumns(properties.getSchema()); + } + + @Override + public List queryTablePrimaryColumns(String tableName) { + return opgsMetaDataMapper.queryTablePrimaryColumnsByTableName(properties.getSchema(),tableName); + } + @Override public List queryTableColumnsMetaData(String tableName) { - if (isOgCompatibilityB) { - return opgsMetaDataMapper.queryTableColumnsMetaDataB(properties.getSchema(), tableName); - } return opgsMetaDataMapper.queryTableColumnsMetaData(properties.getSchema(), tableName); } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OracleDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OracleDataAccessService.java index d61ac90..a89c87b 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OracleDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OracleDataAccessService.java @@ -17,6 +17,7 @@ package org.opengauss.datachecker.extract.data.access; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.extract.data.mapper.OracleMetaDataMapper; @@ -61,6 +62,16 @@ public class OracleDataAccessService extends AbstractDataAccessService { return wrapperTableMetadata(oracleMetaDataMapper.queryTableMetadata(properties.getSchema(), tableName)); } + @Override + public List queryTablePrimaryColumns() { + return oracleMetaDataMapper.queryTablePrimaryColumns(properties.getSchema()); + } + + @Override + public List queryTablePrimaryColumns(String tableName) { + return oracleMetaDataMapper.queryTablePrimaryColumnsByTableName(properties.getSchema(),tableName); + } + @Override public List queryTableMetadataList() { return wrapperTableMetadata(oracleMetaDataMapper.queryTableMetadataList(properties.getSchema())); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/MetaDataMapper.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/MetaDataMapper.java index 8608bcb..e0cb4bf 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/MetaDataMapper.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/MetaDataMapper.java @@ -18,6 +18,7 @@ package org.opengauss.datachecker.extract.data.mapper; import org.apache.ibatis.annotations.Param; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import java.util.List; @@ -62,6 +63,22 @@ public interface MetaDataMapper { */ List queryTableColumnsMetaData(@Param("schema") String schema, @Param("name") String tableName); + /** + * queryTablePrimaryColumns + * + * @param schema schema + * @return primary column + */ + List queryTablePrimaryColumns(@Param("schema") String schema); + + /** + * queryTablePrimaryColumns + * + * @param schema schema + * @return primary column + */ + List queryTablePrimaryColumnsByTableName(@Param("schema") String schema, @Param("name") String tableName); + /** * query table TableMetadata * diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/OpgsMetaDataMapper.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/OpgsMetaDataMapper.java index 0de2754..3081955 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/OpgsMetaDataMapper.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/OpgsMetaDataMapper.java @@ -16,12 +16,8 @@ package org.opengauss.datachecker.extract.data.mapper; import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; import org.opengauss.datachecker.common.entry.enums.OgCompatibility; -import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; - -import java.util.List; /** * OpgsMetaDataMapper @@ -40,13 +36,4 @@ public interface OpgsMetaDataMapper extends MetaDataMapper { */ @Select("show sql_compatibility;") OgCompatibility sqlCompatibility(); - - /** - * query table ColumnsMetaData - * - * @param schema schema - * @param tableName tableName - * @return ColumnsMetaData - */ - List queryTableColumnsMetaDataB(@Param("schema") String schema, @Param("name") String tableName); } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java index f238acd..ee94227 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java @@ -17,11 +17,13 @@ package org.opengauss.datachecker.extract.service; import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.Logger; +import org.opengauss.datachecker.common.config.ConfigCache; +import org.opengauss.datachecker.common.constant.ConfigConstants; import org.opengauss.datachecker.common.entry.enums.ColumnKey; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; import org.opengauss.datachecker.common.entry.extract.MetadataLoadProcess; +import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; -import org.opengauss.datachecker.common.thread.ThreadPoolFactory; import org.opengauss.datachecker.common.util.LogUtils; import org.opengauss.datachecker.common.util.ThreadUtil; import org.opengauss.datachecker.extract.cache.MetaDataCache; @@ -102,9 +104,10 @@ public class MetaDataService { if (CollectionUtils.isEmpty(tableMetadataList)) { return tableMetadataMap; } + Map> tablePrimaryColumns = baseDataService.queryTablePrimaryColumns(); List> futures = new LinkedList<>(); - int maxConnection = resourceManager.maxConnectionCount(); - ExecutorService executor = Executors.newFixedThreadPool(Math.max(1, maxConnection / 2)); + int initConnection = ConfigCache.getIntValue(ConfigConstants.DRUID_INITIAL_SIZE); + ExecutorService executor = Executors.newFixedThreadPool(Math.max(1, initConnection / 2)); metadataLoadProcess.setTotal(tableMetadataList.size()); tableMetadataList.forEach(tableMetadata -> { Future future = executor.submit(() -> { @@ -112,7 +115,8 @@ public class MetaDataService { if (resourceManager.isShutdown()) { log.warn("extract service is shutdown ,task set table metadata of table is canceled!"); } else { - baseDataService.updateTableColumnMetaData(tableMetadata); + List primaryColumnList = tablePrimaryColumns.get(tableMetadata.getTableName()); + baseDataService.updateTableColumnMetaData(tableMetadata, primaryColumnList); tableMetadataMap.put(tableMetadata.getTableName(), tableMetadata); } log.debug("load table and its columns {} hasPrimary={}", tableMetadata.getTableName(), diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/MetaDataUtil.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/MetaDataUtil.java index bf8a3e6..0dd28d8 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/MetaDataUtil.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/MetaDataUtil.java @@ -39,7 +39,7 @@ public class MetaDataUtil { List.of("integer", "int", "uint1", "uint2", "uint4", "uint8", "long", "decimal", "numeric", "smallint", "NUMBER", "tinyint", "mediumint", "bigint"); private static final List dataTypes = - List.of("integer", "int", "uint1", "uint2", "uint4", "uint8", "long", "decimal", "numeric", "NUMBER", + List.of("integer", "int", "uint1", "uint2", "uint4", "uint8", "long", "decimal", "numeric", "NUMBER","VARCHAR2", "smallint", "tinyint", "mediumint", "bigint", "character", "char", "varchar", "character varying", "CHAR"); private static final List digitalDataTypes = diff --git a/datachecker-extract/src/main/resources/mapper/MysqlMetaDataMapper.xml b/datachecker-extract/src/main/resources/mapper/MysqlMetaDataMapper.xml index 9d87c84..5d64e32 100644 --- a/datachecker-extract/src/main/resources/mapper/MysqlMetaDataMapper.xml +++ b/datachecker-extract/src/main/resources/mapper/MysqlMetaDataMapper.xml @@ -20,6 +20,11 @@ + + + + + @@ -67,6 +72,17 @@ where table_schema=#{schema} and table_name =#{name} order by ordinal_position asc + + + diff --git a/datachecker-extract/src/main/resources/mapper/OpgsMetaDataMapper.xml b/datachecker-extract/src/main/resources/mapper/OpgsMetaDataMapper.xml index 71b1135..c112b8c 100644 --- a/datachecker-extract/src/main/resources/mapper/OpgsMetaDataMapper.xml +++ b/datachecker-extract/src/main/resources/mapper/OpgsMetaDataMapper.xml @@ -16,10 +16,14 @@ - + + + + + + @@ -46,40 +50,22 @@ + + - + select tc.table_name tableName,c.column_name columnName from information_schema.table_constraints tc + left join information_schema.key_column_usage c on tc.constraint_name=c.constraint_name + where tc.constraint_type='PRIMARY KEY' tc.table_schema=#{schema} and tc.table_name =#{name} ; @@ -41,17 +43,21 @@ + + + + -- Gitee