From 000fec8597a05cf511f42d43228b4ae171e0b283 Mon Sep 17 00:00:00 2001 From: 18867471376 <13272213728ly> Date: Mon, 19 May 2025 18:37:45 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20solon-ai-load-jdbc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- __release/solon-ai-bundle1/pom.xml | 1 + .../solon-ai-load-jdbc/NOTICE | 65 +++++++++ .../solon-ai-load-jdbc/pom.xml | 48 +++++++ .../solon/ai/rag/loader/JdbcLoadConfig.java | 100 ++++++++++++++ .../noear/solon/ai/rag/loader/JdbcLoader.java | 125 ++++++++++++++++++ .../features/ai/load/jdbc/JdbcLoaderTest.java | 61 +++++++++ 6 files changed, 400 insertions(+) create mode 100644 solon-ai-rag-loaders/solon-ai-load-jdbc/NOTICE create mode 100644 solon-ai-rag-loaders/solon-ai-load-jdbc/pom.xml create mode 100644 solon-ai-rag-loaders/solon-ai-load-jdbc/src/main/java/org/noear/solon/ai/rag/loader/JdbcLoadConfig.java create mode 100644 solon-ai-rag-loaders/solon-ai-load-jdbc/src/main/java/org/noear/solon/ai/rag/loader/JdbcLoader.java create mode 100644 solon-ai-rag-loaders/solon-ai-load-jdbc/src/test/java/features/ai/load/jdbc/JdbcLoaderTest.java diff --git a/__release/solon-ai-bundle1/pom.xml b/__release/solon-ai-bundle1/pom.xml index da6287d1..e4fe4289 100644 --- a/__release/solon-ai-bundle1/pom.xml +++ b/__release/solon-ai-bundle1/pom.xml @@ -31,6 +31,7 @@ ../../solon-ai-rag-loaders/solon-ai-load-html ../../solon-ai-rag-loaders/solon-ai-load-markdown ../../solon-ai-rag-loaders/solon-ai-load-mysql + ../../solon-ai-rag-loaders/solon-ai-load-jdbc ../../solon-ai-rag-loaders/solon-ai-load-pdf ../../solon-ai-rag-loaders/solon-ai-load-ppt ../../solon-ai-rag-loaders/solon-ai-load-word diff --git a/solon-ai-rag-loaders/solon-ai-load-jdbc/NOTICE b/solon-ai-rag-loaders/solon-ai-load-jdbc/NOTICE new file mode 100644 index 00000000..784c4678 --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-jdbc/NOTICE @@ -0,0 +1,65 @@ +Copyright 2017-2025 noear.org and authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This project includes: + Apache Commons Codec under Apache-2.0 + Apache Commons Collections under Apache License, Version 2.0 + Apache Commons Compress under Apache-2.0 + Apache Commons IO under Apache-2.0 + Apache Commons Lang under Apache-2.0 + Apache Commons Math under Apache License, Version 2.0 + Apache Log4j API under Apache-2.0 + Apache POI under Apache License, Version 2.0 + Apache POI - API based on OPC and OOXML schemas under Apache License, Version 2.0 + Apache POI - Common under Apache License, Version 2.0 + asm under BSD-3-Clause + Byte Buddy (without dependencies) under Apache License, Version 2.0 + Byte Buddy agent under Apache License, Version 2.0 + curvesapi under BSD License + JavaPoet under Apache 2.0 + JTokkit under MIT License + JUnit Jupiter (Aggregator) under Eclipse Public License v2.0 + JUnit Jupiter API under Eclipse Public License v2.0 + JUnit Jupiter Engine under Eclipse Public License v2.0 + JUnit Jupiter Params under Eclipse Public License v2.0 + JUnit Platform Commons under Eclipse Public License v2.0 + JUnit Platform Engine API under Eclipse Public License v2.0 + mockito-core under The MIT License + mockito-junit-jupiter under The MIT License + Objenesis under Apache License, Version 2.0 + org.apiguardian:apiguardian-api under The Apache License, Version 2.0 + org.opentest4j:opentest4j under The Apache License, Version 2.0 + reactive-streams under MIT-0 + SLF4J API Module under MIT License + snack3 under The Apache Software License, Version 2.0 + SnakeYAML under Apache License, Version 2.0 + solon under The Apache Software License, Version 2.0 + solon-ai under The Apache Software License, Version 2.0 + solon-ai-load-excel under The Apache Software License, Version 2.0 + solon-aot under The Apache Software License, Version 2.0 + solon-config-plus under The Apache Software License, Version 2.0 + solon-config-yaml under The Apache Software License, Version 2.0 + solon-data under The Apache Software License, Version 2.0 + solon-logging under The Apache Software License, Version 2.0 + solon-logging-simple under The Apache Software License, Version 2.0 + solon-mvc under The Apache Software License, Version 2.0 + solon-net-httputils under The Apache Software License, Version 2.0 + solon-proxy under The Apache Software License, Version 2.0 + solon-rx under The Apache Software License, Version 2.0 + solon-security-vault under The Apache Software License, Version 2.0 + solon-serialization under The Apache Software License, Version 2.0 + solon-test under The Apache Software License, Version 2.0 + SparseBitSet under The Apache Software License, Version 2.0 + XmlBeans under The Apache Software License, Version 2.0 + diff --git a/solon-ai-rag-loaders/solon-ai-load-jdbc/pom.xml b/solon-ai-rag-loaders/solon-ai-load-jdbc/pom.xml new file mode 100644 index 00000000..027cdd3f --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-jdbc/pom.xml @@ -0,0 +1,48 @@ + + + 4.0.0 + + + org.noear + solon-ai-parent + 3.3.1-SNAPSHOT + ../../solon-ai-parent/pom.xml + + + solon-ai-load-jdbc + jar + + + + org.noear + solon-ai + + + + com.zaxxer + HikariCP + provided + + + + mysql + mysql-connector-java + 8.0.33 + provided + + + + org.noear + solon-logging-simple + test + + + + org.noear + solon-test + test + + + \ No newline at end of file diff --git a/solon-ai-rag-loaders/solon-ai-load-jdbc/src/main/java/org/noear/solon/ai/rag/loader/JdbcLoadConfig.java b/solon-ai-rag-loaders/solon-ai-load-jdbc/src/main/java/org/noear/solon/ai/rag/loader/JdbcLoadConfig.java new file mode 100644 index 00000000..191ffb22 --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-jdbc/src/main/java/org/noear/solon/ai/rag/loader/JdbcLoadConfig.java @@ -0,0 +1,100 @@ +package org.noear.solon.ai.rag.loader; + +public class JdbcLoadConfig { + private String shcemaTableQuerySql; + private String shcemaTableQueryAndSchemaSqlTemplate; + private String shcemaTableQueryAndTableSqlTemplate; + private String shcemaTableQueryResultSchemeNameColumn; + private String shcemaTableQueryResultTableNameColumn; + + private String ddlQuerySqlTemplate; + private String ddlQueryResultDdlNameColumn; + + + public JdbcLoadConfig(String shcemaTableQuerySql, String shcemaTableQueryAndSchemaSqlTemplate, String shcemaTableQueryAndTableSqlTemplate, String shcemaTableQueryResultSchemeNameColumn, String shcemaTableQueryResultTableNameColumn, String ddlQuerySqlTemplate, String ddlQueryResultDdlNameColumn) { + this.shcemaTableQuerySql = shcemaTableQuerySql; + this.shcemaTableQueryAndSchemaSqlTemplate = shcemaTableQueryAndSchemaSqlTemplate; + this.shcemaTableQueryAndTableSqlTemplate = shcemaTableQueryAndTableSqlTemplate; + this.shcemaTableQueryResultSchemeNameColumn = shcemaTableQueryResultSchemeNameColumn; + this.shcemaTableQueryResultTableNameColumn = shcemaTableQueryResultTableNameColumn; + this.ddlQuerySqlTemplate = ddlQuerySqlTemplate; + this.ddlQueryResultDdlNameColumn = ddlQueryResultDdlNameColumn; + } + + public static class Builder { + private String shcemaTableQuerySql; + private String shcemaTableQueryAndSchemaSqlTemplate; + private String shcemaTableQueryAndTableSqlTemplate; + private String shcemaTableQueryResultSchemeNameColumn; + private String shcemaTableQueryResultTableNameColumn; + private String ddlQuerySqlTemplate; + private String ddlQueryResultDdlNameColumn; + + public Builder shcemaTableQuerySql(String shcemaTableQuerySql) { + this.shcemaTableQuerySql = shcemaTableQuerySql; + return this; + } + + public Builder shcemaTableQueryAndSchemaSqlTemplate(String shcemaTableQueryAndSchemaSqlTemplate) { + this.shcemaTableQueryAndSchemaSqlTemplate = shcemaTableQueryAndSchemaSqlTemplate; + return this; + } + + public Builder shcemaTableQueryAndTableSqlTemplate(String shcemaTableQueryAndTableSqlTemplate) { + this.shcemaTableQueryAndTableSqlTemplate = shcemaTableQueryAndTableSqlTemplate; + return this; + } + + public Builder shcemaTableQueryResultSchemeNameColumn(String shcemaTableQueryResultSchemeNameColumn) { + this.shcemaTableQueryResultSchemeNameColumn = shcemaTableQueryResultSchemeNameColumn; + return this; + } + + public Builder shcemaTableQueryResultTableNameColumn(String shcemaTableQueryResultTableNameColumn) { + this.shcemaTableQueryResultTableNameColumn = shcemaTableQueryResultTableNameColumn; + return this; + } + + public Builder ddlQuerySqlTemplate(String ddlQuerySqlTemplate) { + this.ddlQuerySqlTemplate = ddlQuerySqlTemplate; + return this; + } + + public Builder ddlQueryResultDdlNameColumn(String ddlQueryResultDdlNameColumn) { + this.ddlQueryResultDdlNameColumn = ddlQueryResultDdlNameColumn; + return this; + } + + public JdbcLoadConfig build() { + return new JdbcLoadConfig(this.shcemaTableQuerySql, this.shcemaTableQueryAndSchemaSqlTemplate, this.shcemaTableQueryAndTableSqlTemplate, this.shcemaTableQueryResultSchemeNameColumn, this.shcemaTableQueryResultTableNameColumn, this.ddlQuerySqlTemplate, this.ddlQueryResultDdlNameColumn); + } + } + + public String getShcemaTableQuerySql() { + return shcemaTableQuerySql; + } + + public String getShcemaTableQueryAndSchemaSqlTemplate() { + return shcemaTableQueryAndSchemaSqlTemplate; + } + + public String getShcemaTableQueryAndTableSqlTemplate() { + return shcemaTableQueryAndTableSqlTemplate; + } + + public String getShcemaTableQueryResultSchemeNameColumn() { + return shcemaTableQueryResultSchemeNameColumn; + } + + public String getShcemaTableQueryResultTableNameColumn() { + return shcemaTableQueryResultTableNameColumn; + } + + public String getDdlQuerySqlTemplate() { + return ddlQuerySqlTemplate; + } + + public String getDdlQueryResultDdlNameColumn() { + return ddlQueryResultDdlNameColumn; + } +} diff --git a/solon-ai-rag-loaders/solon-ai-load-jdbc/src/main/java/org/noear/solon/ai/rag/loader/JdbcLoader.java b/solon-ai-rag-loaders/solon-ai-load-jdbc/src/main/java/org/noear/solon/ai/rag/loader/JdbcLoader.java new file mode 100644 index 00000000..55ae7398 --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-jdbc/src/main/java/org/noear/solon/ai/rag/loader/JdbcLoader.java @@ -0,0 +1,125 @@ +/* + * Copyright 2017-2025 noear.org and authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.noear.solon.ai.rag.loader; + +import java.io.*; +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +import org.noear.snack.core.utils.StringUtil; +import org.noear.solon.ai.rag.Document; +import org.noear.solon.lang.Preview; + +import javax.sql.DataSource; + +/** + * JDBC表结构 加载器 + *

+ * 读取表结构DDL，并转换为文本 + *

+ * + * @author 刘颜 + * @since 3.3 + */ +@Preview("3.3") +public class JdbcLoader extends AbstractOptionsDocumentLoader { + private DataSource dataSource; + private JdbcLoadConfig jdbcLoadConfig; + + public JdbcLoader(DataSource dataSource) { + this.dataSource = dataSource; + //默认支持mysql + this.jdbcLoadConfig = new JdbcLoadConfig.Builder() + .shcemaTableQuerySql(" select TABLE_SCHEMA, TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA not in ('mysql', 'sys', 'information_schema') ") + .shcemaTableQueryAndSchemaSqlTemplate(" and TABLE_SCHEMA = '%s' ") + .shcemaTableQueryAndTableSqlTemplate(" and TABLE_SCHEMA = '%s' and TABLE_NAME = '%s' ") + .shcemaTableQueryResultSchemeNameColumn("TABLE_SCHEMA") + .shcemaTableQueryResultTableNameColumn("TABLE_NAME") + .ddlQuerySqlTemplate(" SHOW CREATE TABLE `%s`.`%s` ") + .ddlQueryResultDdlNameColumn("Create Table").build(); + options = new Options(); + } + + public JdbcLoader(DataSource dataSource, JdbcLoadConfig jdbcLoadConfig) { + this.dataSource = dataSource; + this.jdbcLoadConfig = jdbcLoadConfig; + options = new Options(); + } + + @Override + public List load() throws IOException { + List documents = new ArrayList<>(); + + String targetSchema = options.targetSchema; + String targetTable = options.targetTable; + + try (Connection connection = dataSource.getConnection()) { + String sql = this.jdbcLoadConfig.getShcemaTableQuerySql(); + if (StringUtil.isEmpty(targetSchema)) { + //全库全表 + } else if (StringUtil.isEmpty(targetTable)) { + //单库全表 + sql = sql.concat(String.format(this.jdbcLoadConfig.getShcemaTableQueryAndSchemaSqlTemplate(), targetSchema)); + } else { + //单库单表 + sql = sql.concat(String.format(this.jdbcLoadConfig.getShcemaTableQueryAndTableSqlTemplate(), targetSchema, targetTable)); + } + try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) { + ResultSet rowSet = preparedStatement.executeQuery(); + while (rowSet.next()) { + //每张表单独转化 + String tableSchema = rowSet.getString(this.jdbcLoadConfig.getShcemaTableQueryResultSchemeNameColumn()); + String tableName = rowSet.getString(this.jdbcLoadConfig.getShcemaTableQueryResultTableNameColumn()); + String showddl = String.format(this.jdbcLoadConfig.getDdlQuerySqlTemplate(), tableSchema, tableName); + try (PreparedStatement showddlPrepareStatement = connection.prepareStatement(showddl)) { + ResultSet showddlResultSet = showddlPrepareStatement.executeQuery(); + if (showddlResultSet.next()) { + String tableddl = showddlResultSet.getString(this.jdbcLoadConfig.getDdlQueryResultDdlNameColumn()); + String fullDDL = "CREATE TABLE `" + tableSchema + "`.`" + tableName + "` " + tableddl.substring(tableddl.indexOf("(")); + Document doc = new Document(fullDDL).metadata(this.additionalMetadata); + documents.add(doc); + } + } + } + } + } catch (RuntimeException | SQLException e) { + throw new RuntimeException(e); + } + + return documents; + } + + /** + * 选项 + */ + public static class Options { + private String targetSchema; + private String targetTable; + + /** + * targetSchema为null，不管targetTable是否为null，则是全库全表加载 + * targetSchema不为null，targetTable为null则是单库全表加载 + * targetSchema不为null，targetTable不为null则是单库单表加载 + * 否则默认是全库全表加载 + */ + public Options loadOptions(String targetSchema, String targetTable) { + this.targetSchema = targetSchema; + this.targetTable = targetTable; + return this; + } + } +} \ No newline at end of file diff --git a/solon-ai-rag-loaders/solon-ai-load-jdbc/src/test/java/features/ai/load/jdbc/JdbcLoaderTest.java b/solon-ai-rag-loaders/solon-ai-load-jdbc/src/test/java/features/ai/load/jdbc/JdbcLoaderTest.java new file mode 100644 index 00000000..432b2682 --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-jdbc/src/test/java/features/ai/load/jdbc/JdbcLoaderTest.java @@ -0,0 +1,61 @@ +package features.ai.load.jdbc; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.Test; +import org.noear.solon.ai.rag.Document; +import org.noear.solon.ai.rag.loader.JdbcLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.util.List; + +public class JdbcLoaderTest { + private static final Logger log = LoggerFactory.getLogger(JdbcLoaderTest.class); + + private static DataSource ds; + + public JdbcLoaderTest() { + String url = "jdbc:mysql://127.0.0.1:3306/test"; + String username = "root"; + String password = "root"; + + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(url); + config.setUsername(username); + config.setPassword(password); + + ds = new HikariDataSource(config); + } + + @Test + public void test1() throws Exception { + //全库全表 + JdbcLoader loader = new JdbcLoader(ds); + List docs = loader.load(); + System.out.println(docs.size()); + assert docs.size() > 0; + } + + @Test + public void test2() throws Exception { + //单库全表 + JdbcLoader loader = new JdbcLoader(ds); + loader.options(opt -> opt.loadOptions("zt", null)); + List docs = loader.load(); + System.out.println(docs.size()); + assert docs.size() > 0; + } + + @Test + public void test3() throws Exception { + //单库单表 + JdbcLoader loader = new JdbcLoader(ds); + loader.options(opt -> opt.loadOptions("zt", "location")); + List docs = loader.load(); + System.out.println(docs); + System.out.println(docs.size()); + assert docs.size() > 0; + } +} \ No newline at end of file -- Gitee