diff --git a/config/log4j2.xml b/config/log4j2.xml index 4fa56efe111f33cf1eed921c652ed11806c04296..115b2bcd4cc3b706fc6abe2acc3f68038a5fd35a 100644 --- a/config/log4j2.xml +++ b/config/log4j2.xml @@ -14,7 +14,7 @@ ~ See the Mulan PSL v2 for more details. --> - + logs @@ -76,11 +76,15 @@ - - + + + + + + \ No newline at end of file diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/load/CheckStartLoader.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/load/CheckStartLoader.java index 0d71673eb4afbd162c830c614b07fdaf7aaae5f4..65c1c85d38405e37c7d869a6a383c0318f3c1076 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/load/CheckStartLoader.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/load/CheckStartLoader.java @@ -15,7 +15,6 @@ package org.opengauss.datachecker.check.load; -import org.opengauss.datachecker.check.modules.check.ExportCheckResult; import org.opengauss.datachecker.check.modules.report.SliceProgressService; import org.opengauss.datachecker.check.service.CheckService; import org.opengauss.datachecker.check.event.KafkaTopicDeleteProvider; diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckTableStructureService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckTableStructureService.java index 38102fe7920ac547edae6ccc914d08ca2e831616..420d7a54f5e0f7121c16224f2073b4c2da7a7781 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckTableStructureService.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckTableStructureService.java @@ -64,9 +64,11 @@ public class CheckTableStructureService { if (source.size() == sink.size()) { final List sourceUpperList = source.stream() .map(ColumnsMetaData::getColumnName) + .map(String::toLowerCase) .collect(Collectors.toList()); final List diffKeyList = sink.stream() .map(ColumnsMetaData::getColumnName) + .map(String::toLowerCase) .filter(key -> !sourceUpperList.contains(key)) .collect(Collectors.toList()); return diffKeyList.isEmpty(); @@ -143,7 +145,7 @@ public class CheckTableStructureService { .schema(database.getSchema()) .build(); taskRegisterCenter.refreshCheckedTableCompleted(tableName); - sliceCheckResultManager.addTableStructureDiffResult(tableName,result); + sliceCheckResultManager.addTableStructureDiffResult(tableName, result); log.debug("compared table[{}] field names not match source={},sink={}", tableName, getFieldNames(sourceMeta), getFieldNames(sinkMeta)); } @@ -169,7 +171,7 @@ public class CheckTableStructureService { .isExistTableMiss(true, onlyExistEndpoint) .build(); taskManagerService.refreshTableExtractStatus(tableName, Endpoint.CHECK, -1); - sliceCheckResultManager.addTableStructureDiffResult(tableName,result); + sliceCheckResultManager.addTableStructureDiffResult(tableName, result); log.error("compared the field names in table[{}](case ignored) and the result is not match", tableName); return onlyExistEndpoint; } @@ -191,7 +193,12 @@ public class CheckTableStructureService { @FunctionalInterface interface CompareTableStructure { /** - * Compare whether the source and destination table structures are the same + * Compare whether the source and destination table structures are the same. + *

+ * When comparing table structures, ignore the capitalization of field names. + * But in metadata, the capitalization of column names cannot be modified, + * otherwise there is a possibility that query columns do not exist. + * This is because the database itself determines the column case recognition pattern. * * @param source source * @param sink sink diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/ConfigConstants.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/ConfigConstants.java index d9a480c2b1557b246e255eb50d0d169b15119396..34ad178887f73b3ed71a17144c63b214b18fb3f2 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/ConfigConstants.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/ConfigConstants.java @@ -41,6 +41,7 @@ public interface ConfigConstants { String ENDPOINT = "spring.extract.endpoint"; String QUERY_DOP = "spring.extract.query-dop"; String DATA_BASE_TYPE = "spring.extract.databaseType"; + String OBJECT_SIZE_EXPANSION_FACTOR = "spring.extract.object-size-expansion-factor"; String MEMORY_MONITOR = "spring.memory-monitor-enable"; diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SliceVo.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SliceVo.java index 91b50d0e91ee3f36e881550b7ef1c655e26662c4..7eab24bcf7227a9be69bdfb956bc276a0a8e4881 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SliceVo.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SliceVo.java @@ -74,6 +74,9 @@ public class SliceVo extends BaseSlice { } public String toSimpleString() { + if (super.getTotal() == 1) { + return super.getName() + " total=" + super.getTotal() + " no=" + super.getNo() + ", [ fetch full ]"; + } return super.getName() + " total=" + super.getTotal() + " no=" + super.getNo() + ", [" + super.getBeginIdx() + " , " + super.getEndIdx() + " ]" + " fetchSize=" + super.getFetchSize(); } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java index b1691d67357bbdb85f7b37568960d42338b5c0ca..1eb4276c0fde332d84f69887ec4245d1588d5164 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java @@ -34,6 +34,7 @@ import javax.sql.DataSource; import java.util.Comparator; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -195,11 +196,23 @@ public class BaseDataService { return ruleAdapterService.executeColumnRule(columns); } + /** + * Compare whether the source and destination table structures are the same. + *

+ * When comparing table structures, ignore the capitalization of field names. + * But in metadata, the capitalization of column names cannot be modified, + * otherwise there is a possibility that query columns do not exist. + * This is because the database itself determines the column case recognition pattern. + * + * @param columnsMetas columnsMetas + * @return cloumn hash + */ private long calcTableHash(List columnsMetas) { StringBuffer buffer = new StringBuffer(); columnsMetas.sort(Comparator.comparing(ColumnsMetaData::getOrdinalPosition)); columnsMetas.forEach(column -> { - buffer.append(column.getColumnName()) + buffer.append(column.getColumnName() + .toLowerCase(Locale.ENGLISH)) .append(column.getOrdinalPosition()); }); return HASH_UTIL.hashBytes(buffer.toString()); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/ResourceManager.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/ResourceManager.java index 5e91740b69af9649a55839a0d26b8bcd7d25ffc7..28df8e0e767ca5553380c015310c83f57547a9ab 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/ResourceManager.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/ResourceManager.java @@ -85,8 +85,7 @@ public class ResourceManager { lock.lock(); try { tryAvailableTimes.incrementAndGet(); - final JvmInfo memory = MemoryManager.getJvmInfo(); - if (connectionCount.get() > 2 && memory.isAvailable(freeSize)) { + if (connectionCount.get() > 2 && checkFreeMemory(freeSize)) { connectionCount.decrementAndGet(); tryAvailableTimes.set(0); return true; @@ -101,6 +100,16 @@ public class ResourceManager { } } + public boolean checkFreeMemory(long allocSize) { + lock.lock(); + try { + long freeMemory = Runtime.getRuntime().freeMemory(); + return freeMemory > allocSize; + } finally { + lock.unlock(); + } + } + /** * check has free connection * diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/ConfigManagement.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/ConfigManagement.java index 8c5fb485b8a154d7eebc8f03742ae6dcc8961127..b89073abf0e69f01e99c98ef1abdd9b138df5b03 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/ConfigManagement.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/ConfigManagement.java @@ -58,7 +58,6 @@ public class ConfigManagement { @Value("${spring.check.extend-maximum-pool-size}") private int extendMaxPoolSize = 10; - @Value("${spring.datasource.druid.initialSize}") private int initialSize; @Value("${spring.datasource.druid.minIdle}") @@ -73,6 +72,8 @@ public class ConfigManagement { private int minEvictableIdleTimeMillis; @Value("${spring.lifecycle.timeout-per-shutdown-phase}") private int timeoutPerShutdownPhase; + @Value("${spring.extract.object-size-expansion-factor}") + private int objectSizeExpansionFactor; /** * init csv config @@ -137,9 +138,9 @@ public class ConfigManagement { ConfigCache.put(ConfigConstants.DRUID_MIN_EVICTABLE_IDLE_TIME_MILLIS, minEvictableIdleTimeMillis); } - private static void setExtractConfig(ExtractProperties properties) { + private void setExtractConfig(ExtractProperties properties) { ConfigCache.put(ConfigConstants.ENDPOINT, properties.getEndpoint()); ConfigCache.put(ConfigConstants.DATA_BASE_TYPE, properties.getDatabaseType()); + ConfigCache.put(ConfigConstants.OBJECT_SIZE_EXPANSION_FACTOR, objectSizeExpansionFactor); } - } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java index c32a9b7638189b318de36bdeb61916c15603f07a..f238acdb793e8e144e278fef6079d41a486412d4 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java @@ -115,6 +115,8 @@ public class MetaDataService { baseDataService.updateTableColumnMetaData(tableMetadata); tableMetadataMap.put(tableMetadata.getTableName(), tableMetadata); } + log.debug("load table and its columns {} hasPrimary={}", tableMetadata.getTableName(), + tableMetadata.hasPrimary()); resourceManager.release(); }); futures.add(future); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/AbstractProcessor.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/AbstractProcessor.java index 77820dfe83d4f5840c4c7c6f96ce2fae66433472..0af962804778beeddd3a364570fbef0b8c0f6860 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/AbstractProcessor.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/AbstractProcessor.java @@ -15,6 +15,8 @@ package org.opengauss.datachecker.extract.slice.process; +import org.opengauss.datachecker.common.config.ConfigCache; +import org.opengauss.datachecker.common.constant.ConfigConstants; import org.opengauss.datachecker.common.entry.extract.SliceExtend; import org.opengauss.datachecker.extract.slice.SliceProcessorContext; @@ -29,9 +31,11 @@ import java.math.BigDecimal; */ public abstract class AbstractProcessor implements SliceProcessor { protected SliceProcessorContext context; + protected int objectSizeExpansionFactor; public AbstractProcessor(SliceProcessorContext context) { this.context = context; + this.objectSizeExpansionFactor = ConfigCache.getIntValue(ConfigConstants.OBJECT_SIZE_EXPANSION_FACTOR); } /** @@ -44,7 +48,9 @@ public abstract class AbstractProcessor implements SliceProcessor { protected long estimatedMemorySize(long rowLength, long sliceSize) { BigDecimal rowLengthNum = BigDecimal.valueOf(rowLength); BigDecimal sliceSizeNum = BigDecimal.valueOf(sliceSize); - return rowLengthNum.multiply(sliceSizeNum).longValue(); + return rowLengthNum.multiply(sliceSizeNum) + .multiply(BigDecimal.valueOf(objectSizeExpansionFactor)) + .longValue(); } /** diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcSliceProcessor.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcSliceProcessor.java index 16ce1e64607a5f06fca25d0f7ccfd9f2a84f7581..dc22cf0a953666fbd281fa6f0f88f877fd8efc58 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcSliceProcessor.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcSliceProcessor.java @@ -65,6 +65,7 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { SliceExtend sliceExtend = createSliceExtend(tableMetadata.getTableHash()); if (!slice.isEmptyTable()) { QuerySqlEntry queryStatement = createQueryStatement(tableMetadata); + log.debug("table [{}] query statement : {}", queryStatement.getTable(), queryStatement.getSql()); executeQueryStatement(queryStatement, tableMetadata, sliceExtend); } else { log.info("table slice [{}] is empty ", slice.toSimpleString()); @@ -101,11 +102,13 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { long estimatedRowCount = slice.isSlice() ? slice.getFetchSize() : tableMetadata.getTableRows(); long estimatedMemorySize = estimatedMemorySize(tableMetadata.getAvgRowLength(), estimatedRowCount); connection = jdbcOperation.tryConnectionAndClosedAutoCommit(estimatedMemorySize); + log.debug("slice [{}] fetch jdbc connection.", slice.getName()); SliceKafkaAgents kafkaAgents = context.createSliceKafkaAgents(slice); SliceResultSetSender sliceSender = new SliceResultSetSender(tableMetadata, kafkaAgents); try (PreparedStatement ps = connection.prepareStatement(sqlEntry.getSql()); ResultSet resultSet = ps.executeQuery()) { + log.debug("slice [{}] jdbc execute query complete.", slice.getName()); LocalDateTime jdbcQuery = LocalDateTime.now(); jdbcQueryCost = durationBetweenToMillis(start, jdbcQuery); resultSet.setFetchSize(FETCH_SIZE); @@ -114,10 +117,10 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { Map result = new TreeMap<>(); List offsetList = new LinkedList<>(); List>> batchFutures = new LinkedList<>(); - final String tableName= slice.getTable(); + final String tableName = slice.getTable(); while (resultSet.next()) { rowCount++; - batchFutures.add(sliceSender.resultSetTranslateAndSendSync(tableName,rsmd, resultSet, result, slice.getNo())); + batchFutures.add(sliceSender.resultSetTranslateAndSendSync(tableName, rsmd, resultSet, result, slice.getNo())); if (batchFutures.size() == FETCH_SIZE) { offsetList.add(getBatchFutureRecordOffsetScope(batchFutures)); batchFutures.clear(); @@ -137,6 +140,7 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { throw new ExtractDataAccessException(ex); } finally { jdbcOperation.releaseConnection(connection); + log.debug("slice [{}] release jdbc connection.", slice.getName()); sliceAllCost = durationBetweenToMillis(start, LocalDateTime.now()); log.info("query slice [{}] cost [{} /{} /{}] milliseconds", sliceExtend.toSimpleString(), jdbcQueryCost, sendDataCost, sliceAllCost); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/MysqlResultSetHandler.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/MysqlResultSetHandler.java index 59b2248b773a4c8196f72ca44b2e8bc8e51fe77c..e2c8a8f1b18f477f2416909e3d644c51d0a36c36 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/MysqlResultSetHandler.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/MysqlResultSetHandler.java @@ -39,6 +39,7 @@ public class MysqlResultSetHandler extends ResultSetHandler { TypeHandler binaryToString = (rs, columnLabel) -> bytesToString(rs.getBytes(columnLabel)); TypeHandler blobToString = (rs, columnLabel) -> HexUtil.byteToHexTrim(rs.getBytes(columnLabel)); TypeHandler numericToString = (rs, columnLabel) -> floatingPointNumberToString(rs, columnLabel); + TypeHandler charToString = (rs, columnLabel) -> fixedLenCharToString(rs, columnLabel); typeHandlers.put(MysqlType.FLOAT_UNSIGNED, numericToString); typeHandlers.put(MysqlType.FLOAT, numericToString); @@ -50,6 +51,8 @@ public class MysqlResultSetHandler extends ResultSetHandler { typeHandlers.put(MysqlType.BINARY, binaryToString); typeHandlers.put(MysqlType.VARBINARY, binaryToString); + typeHandlers.put(MysqlType.CHAR, charToString); + typeHandlers.put(MysqlType.BLOB, blobToString); typeHandlers.put(MysqlType.LONGBLOB, blobToString); typeHandlers.put(MysqlType.MEDIUMBLOB, blobToString); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/OpenGaussResultSetHandler.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/OpenGaussResultSetHandler.java index 20b29e918c1ee101857b870562d59e36af55be73..a01a111ea133e5040479d003c78718e85d7aa6d0 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/OpenGaussResultSetHandler.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/OpenGaussResultSetHandler.java @@ -46,8 +46,9 @@ public class OpenGaussResultSetHandler extends ResultSetHandler { TypeHandler booleanToString = (rs, columnLabel) -> booleanToString(rs, columnLabel); TypeHandler numericToString = (rs, columnLabel) -> floatingPointNumberToString(rs, columnLabel); TypeHandler numeric0ToString = (rs, columnLabel) -> numeric0ToString(rs, columnLabel); - TypeHandler float4ToString = (rs, columnLabel) -> float4ToString(rs, columnLabel); + TypeHandler float4ToString = (rs, columnLabel) -> floatingPointNumberToString(rs, columnLabel); TypeHandler intToString = (rs, columnLabel) -> intToString(rs, columnLabel); + TypeHandler bpCharToString = (rs, columnLabel) -> fixedLenCharToString(rs, columnLabel); // float4 - float real typeHandlers.put(OpenGaussType.INT4, numeric0ToString); @@ -55,6 +56,7 @@ public class OpenGaussResultSetHandler extends ResultSetHandler { typeHandlers.put(OpenGaussType.FLOAT4, float4ToString); typeHandlers.put(OpenGaussType.NUMERIC, numericToString); typeHandlers.put(OpenGaussType.NUMERIC0, numeric0ToString); + typeHandlers.put(OpenGaussType.BPCHAR, bpCharToString); // byte binary blob typeHandlers.put(OpenGaussType.BYTEA, byteaToString); @@ -66,9 +68,6 @@ public class OpenGaussResultSetHandler extends ResultSetHandler { typeHandlers.put(OpenGaussType.binary, binaryToString); typeHandlers.put(OpenGaussType.varbinary, binaryToString); - // The openGauss jdbc driver obtains the character,character varying type as varchar - typeHandlers.put(OpenGaussType.BPCHAR, this::trim); - // date time timestamp typeHandlers.put(OpenGaussType.DATE, this::getDateFormat); typeHandlers.put(OpenGaussType.TIME, this::getTimeFormat); @@ -98,10 +97,6 @@ public class OpenGaussResultSetHandler extends ResultSetHandler { return rs.getString(columnLabel); } - private String float4ToString(ResultSet rs, String columnLabel) throws SQLException { - return Float.toString(rs.getFloat(columnLabel)); - } - @Override public String convert(ResultSet resultSet, int columnIdx, ResultSetMetaData rsmd) throws SQLException { String columnLabel = rsmd.getColumnLabel(columnIdx); @@ -109,7 +104,7 @@ public class OpenGaussResultSetHandler extends ResultSetHandler { if (OpenGaussType.isNumeric(columnTypeName)) { return convertNumericToString(rsmd, resultSet, columnIdx); } else if (OpenGaussType.isFloat(columnTypeName)) { - return float4ToString(resultSet, columnLabel); + return floatingPointNumberToString(resultSet, columnLabel); } else if (OpenGaussType.isBigInteger(columnTypeName)) { return numeric0ToString(resultSet, columnLabel); } else if (OpenGaussType.isInteger(columnTypeName)) { diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/OracleResultSetHandler.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/OracleResultSetHandler.java index 26982c278bf7c98968cafb9a4f29962ad612913f..010d2dd15c22d8f9d3475a0c915b8aecde357628 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/OracleResultSetHandler.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/OracleResultSetHandler.java @@ -20,7 +20,6 @@ import org.opengauss.datachecker.common.util.HexUtil; import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; -import java.sql.Clob; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -40,8 +39,9 @@ public class OracleResultSetHandler extends ResultSetHandler { private final Map typeHandlers = new ConcurrentHashMap<>(); { - TypeHandler blobToString = (rs, columnLabel) -> HexUtil.byteToHexTrim(rs.getBytes(columnLabel)); - TypeHandler clobToString = (rs, columnLabel) -> clobToString(rs.getClob(columnLabel)); + TypeHandler blobToString = (rs, columnLabel) -> blobToString(rs, columnLabel); + TypeHandler rawToString = (rs, columnLabel) -> rawToString(rs, columnLabel); + TypeHandler clobToString = (rs, columnLabel) -> clobToString(rs, columnLabel); TypeHandler xmlToString = (rs, columnLabel) -> rs.getString(columnLabel); TypeHandler numberToString = (rs, columnLabel) -> floatingPointNumberToString(rs, columnLabel); TypeHandler number0ToString = (rs, columnLabel) -> numeric0ToString(rs, columnLabel); @@ -50,6 +50,7 @@ public class OracleResultSetHandler extends ResultSetHandler { typeHandlers.put(OracleType.NUMBER, numberToString); typeHandlers.put(OracleType.NUMBER0, number0ToString); // byte binary blob + typeHandlers.put(OracleType.RAW, rawToString); typeHandlers.put(OracleType.NCLOB, clobToString); typeHandlers.put(OracleType.BLOB, blobToString); typeHandlers.put(OracleType.CLOB, clobToString); @@ -61,6 +62,43 @@ public class OracleResultSetHandler extends ResultSetHandler { typeHandlers.put(OracleType.TIMESTAMPTZ, this::getTimestampFormat); } + private String rawToString(ResultSet resultSet, String columnLabel) throws SQLException { + return resultSet.getString(columnLabel); + } + + private String blobToString(ResultSet resultSet, String columnLabel) throws SQLException { + if (resultSet.wasNull()) { + return NULL; + } + return HexUtil.byteToHexTrim(resultSet.getBytes(columnLabel)); + } + + protected synchronized String clobToString(ResultSet resultSet, String columnLabel) throws SQLException { + if (resultSet.wasNull()) { + return NULL; + } + StringBuffer sb = new StringBuffer(); + BufferedReader bf = null; + Reader reader = null; + try { + reader = resultSet.getCharacterStream(columnLabel); + if (reader == null) { + return NULL; + } + bf = new BufferedReader(reader); + String line; + while ((line = bf.readLine()) != null) { + sb.append(line); + } + } catch (IOException io) { + log.error("read clobToString error"); + } finally { + closeBufferedReader(bf); + closeReader(reader); + } + return sb.toString(); + } + @Override protected String convert(ResultSet resultSet, int columnIdx, ResultSetMetaData rsmd) throws SQLException { String columnLabel = rsmd.getColumnLabel(columnIdx); @@ -78,24 +116,9 @@ public class OracleResultSetHandler extends ResultSetHandler { } } - protected String clobToString(Clob clob) throws SQLException { - if (Objects.isNull(clob)) { - return NULL; - } - StringBuffer sb = new StringBuffer(); - try (Reader reader = clob.getCharacterStream(); BufferedReader bf = new BufferedReader(reader)) { - String line; - while ((line = bf.readLine()) != null) { - sb.append(line); - } - } catch (IOException io) { - log.error("read blob error"); - } - return sb.toString(); - } - @SuppressWarnings("all") interface OracleType { + String RAW = "RAW"; String BLOB = "BLOB"; String NCLOB = "NCLOB"; String CLOB = "CLOB"; @@ -130,4 +153,23 @@ public class OracleResultSetHandler extends ResultSetHandler { return NUMBER.equalsIgnoreCase(typeName) && scale >= NUMERIC_SCALE_F84 && scale <= NUMERIC_SCALE_0; } } + private void closeBufferedReader(BufferedReader bf) { + try { + if (Objects.nonNull(bf)) { + bf.close(); + } + } catch (IOException e) { + log.error("close BufferedReader error"); + } + } + + private void closeReader(Reader reader) { + try { + if (Objects.nonNull(reader)) { + reader.close(); + } + } catch (IOException e) { + log.error("close Reader error"); + } + } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ResultSetHandler.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ResultSetHandler.java index af422873836707e563f94c2631b3ed14e3efcb1d..a8831cef7ae3cb79e1ad33f678ab9f2c6498a873 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ResultSetHandler.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ResultSetHandler.java @@ -49,7 +49,7 @@ import java.util.stream.IntStream; public abstract class ResultSetHandler { private static Map decimalFormatCache = new HashMap<>(); private static final String decimal_format_pattern_start = "0."; - private static final String decimal_format_pattern_append_zero = "0"; + private static final String decimal_append_zero = "0"; protected static final Logger log = LogUtils.getLogger(); protected static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("yyyy-MM-dd"); @@ -104,6 +104,10 @@ public abstract class ResultSetHandler { return values; } + protected String fixedLenCharToString(ResultSet rs, String columnLabel) throws SQLException { + return rs.getString(columnLabel); + } + /** * Convert the current query result set into map according to the metadata information of the result set * @@ -131,10 +135,7 @@ public abstract class ResultSetHandler { if (decimalFormatCache.containsKey(scale)) { scaleFormatter = decimalFormatCache.get(scale); } else { - String pattern = decimal_format_pattern_start; - for (int i = 0; i < scale; i++) { - pattern = pattern + decimal_format_pattern_append_zero; - } + String pattern = decimal_format_pattern_start + decimal_append_zero.repeat(Math.max(0, scale)); scaleFormatter = new DecimalFormat(pattern); decimalFormatCache.put(scale, scaleFormatter); } @@ -143,14 +144,12 @@ public abstract class ResultSetHandler { protected String floatingPointNumberToString(@NonNull ResultSet resultSet, String columnLabel) throws SQLException { BigDecimal bigDecimal = resultSet.getBigDecimal(columnLabel); - return resultSet.wasNull() ? FLOATING_POINT_NUMBER_ZERO : - Objects.isNull(bigDecimal) ? FLOATING_POINT_NUMBER_ZERO : Double.toString(bigDecimal.doubleValue()); + return Objects.isNull(bigDecimal) ? NULL : Double.toString(bigDecimal.doubleValue()); } protected String numeric0ToString(ResultSet rs, String columnLabel) throws SQLException { BigDecimal bigDecimal = rs.getBigDecimal(columnLabel); - return rs.wasNull() ? NULL : Objects.isNull(bigDecimal) ? NULL : bigDecimal.toBigInteger() - .toString(); + return Objects.isNull(bigDecimal) ? NULL : bigDecimal.toBigInteger().toString(); } protected String getDateFormat(@NonNull ResultSet resultSet, String columnLabel) throws SQLException { diff --git a/datachecker-extract/src/main/resources/application-sink.yml b/datachecker-extract/src/main/resources/application-sink.yml index 7bc200dd07151c1246be63cc35a409c26fe7857b..ee323704535117a931110611a373e84869119f52 100644 --- a/datachecker-extract/src/main/resources/application-sink.yml +++ b/datachecker-extract/src/main/resources/application-sink.yml @@ -10,6 +10,7 @@ spring: extract: schema: jack databaseType: OG #OG opengauss + object-size-expansion-factor: 4 endpoint: SINK dataLoadMode: jdbc # jdbc or csv query-dop: 1 # jdbc Parallel Query config diff --git a/datachecker-extract/src/main/resources/application-source.yml b/datachecker-extract/src/main/resources/application-source.yml index 1c5e3d43e7e85be8c9a0908c919f8ef98509be4d..a60dc286b9fcd51768fa068eaf11e4ca924cc627 100644 --- a/datachecker-extract/src/main/resources/application-source.yml +++ b/datachecker-extract/src/main/resources/application-source.yml @@ -10,6 +10,7 @@ spring: extract: schema: test databaseType: MS # MS mysql + object-size-expansion-factor: 4 endpoint: SOURCE dataLoadMode: jdbc # jdbc or csv query-dop: 1 # jdbc Parallel Query config diff --git a/datachecker-extract/src/main/resources/mapper/OracleMetaDataMapper.xml b/datachecker-extract/src/main/resources/mapper/OracleMetaDataMapper.xml index 6f23d5824bbe87f8d29592b42bdaa18ffbb7e579..f1e2c6876046382c0b28a706f7ef07d2a6c73316 100644 --- a/datachecker-extract/src/main/resources/mapper/OracleMetaDataMapper.xml +++ b/datachecker-extract/src/main/resources/mapper/OracleMetaDataMapper.xml @@ -41,7 +41,7 @@