diff --git a/solon-ai-rag-loaders/solon-ai-load-mysql/NOTICE b/solon-ai-rag-loaders/solon-ai-load-mysql/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..784c4678c639156adeace4ad272636ee27f50392 --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-mysql/NOTICE @@ -0,0 +1,65 @@ +Copyright 2017-2025 noear.org and authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This project includes: + Apache Commons Codec under Apache-2.0 + Apache Commons Collections under Apache License, Version 2.0 + Apache Commons Compress under Apache-2.0 + Apache Commons IO under Apache-2.0 + Apache Commons Lang under Apache-2.0 + Apache Commons Math under Apache License, Version 2.0 + Apache Log4j API under Apache-2.0 + Apache POI under Apache License, Version 2.0 + Apache POI - API based on OPC and OOXML schemas under Apache License, Version 2.0 + Apache POI - Common under Apache License, Version 2.0 + asm under BSD-3-Clause + Byte Buddy (without dependencies) under Apache License, Version 2.0 + Byte Buddy agent under Apache License, Version 2.0 + curvesapi under BSD License + JavaPoet under Apache 2.0 + JTokkit under MIT License + JUnit Jupiter (Aggregator) under Eclipse Public License v2.0 + JUnit Jupiter API under Eclipse Public License v2.0 + JUnit Jupiter Engine under Eclipse Public License v2.0 + JUnit Jupiter Params under Eclipse Public License v2.0 + JUnit Platform Commons under Eclipse Public License v2.0 + JUnit Platform Engine API under Eclipse Public License v2.0 + mockito-core under The MIT License + mockito-junit-jupiter under The MIT License + Objenesis under Apache License, Version 2.0 + org.apiguardian:apiguardian-api under The Apache License, Version 2.0 + org.opentest4j:opentest4j under The Apache License, Version 2.0 + reactive-streams under MIT-0 + SLF4J API Module under MIT License + snack3 under The Apache Software License, Version 2.0 + SnakeYAML under Apache License, Version 2.0 + solon under The Apache Software License, Version 2.0 + solon-ai under The Apache Software License, Version 2.0 + solon-ai-load-excel under The Apache Software License, Version 2.0 + solon-aot under The Apache Software License, Version 2.0 + solon-config-plus under The Apache Software License, Version 2.0 + solon-config-yaml under The Apache Software License, Version 2.0 + solon-data under The Apache Software License, Version 2.0 + solon-logging under The Apache Software License, Version 2.0 + solon-logging-simple under The Apache Software License, Version 2.0 + solon-mvc under The Apache Software License, Version 2.0 + solon-net-httputils under The Apache Software License, Version 2.0 + solon-proxy under The Apache Software License, Version 2.0 + solon-rx under The Apache Software License, Version 2.0 + solon-security-vault under The Apache Software License, Version 2.0 + solon-serialization under The Apache Software License, Version 2.0 + solon-test under The Apache Software License, Version 2.0 + SparseBitSet under The Apache Software License, Version 2.0 + XmlBeans under The Apache Software License, Version 2.0 + diff --git a/solon-ai-rag-loaders/solon-ai-load-mysql/pom.xml b/solon-ai-rag-loaders/solon-ai-load-mysql/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..a1ccd6c1d58f4e9abda375602ef9633153073751 --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-mysql/pom.xml @@ -0,0 +1,46 @@ + + + 4.0.0 + + + org.noear + solon-ai-parent + 3.3.1-SNAPSHOT + ../../solon-ai-parent/pom.xml + + + solon-ai-load-mysql + jar + + + + org.noear + solon-ai + + + + com.zaxxer + HikariCP + + + + mysql + mysql-connector-java + 8.0.28 + + + + org.noear + solon-logging-simple + test + + + + org.noear + solon-test + test + + + \ No newline at end of file diff --git a/solon-ai-rag-loaders/solon-ai-load-mysql/src/main/java/org/noear/solon/ai/rag/loader/MysqlLoader.java b/solon-ai-rag-loaders/solon-ai-load-mysql/src/main/java/org/noear/solon/ai/rag/loader/MysqlLoader.java new file mode 100644 index 0000000000000000000000000000000000000000..2bb63c4815c4a42a2bf28ddd941f7d721b29c7ba --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-mysql/src/main/java/org/noear/solon/ai/rag/loader/MysqlLoader.java @@ -0,0 +1,119 @@ +/* + * Copyright 2017-2025 noear.org and authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.noear.solon.ai.rag.loader; + +import java.io.*; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.noear.snack.core.utils.StringUtil; +import org.noear.solon.ai.rag.Document; +import org.noear.solon.lang.Preview; + +/** + * Mysql 加载器 + *

+ * 读取Mysql的表结构DDL，并转换为文本 + *

+ * + * @author 刘颜 + * @since 3.3 + */ +@Preview("3.3") +public class MysqlLoader extends AbstractOptionsDocumentLoader { + private String url; + private String username; + private String password; + + public MysqlLoader(String url, String username, String password) { + this.url = url; + this.username = username; + this.password = password; + options = new Options(); + } + + @Override + public List load() throws IOException { + List documents = new ArrayList<>(); + + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(this.url); + config.setUsername(this.username); + config.setPassword(this.password); + String targetSchema = options.targetSchema; + String targetTable = options.targetTable; + try (HikariDataSource hikariDataSource = new HikariDataSource(config)) { + try (Connection connection = hikariDataSource.getConnection()) { + String sql = "select TABLE_SCHEMA, TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA not in ('mysql', 'sys', 'information_schema')"; + if (StringUtil.isEmpty(targetSchema)) { + //全库全表 + } else if (StringUtil.isEmpty(targetTable)) { + sql = String.format(sql.concat(" and TABLE_SCHEMA = '%s' "), targetSchema); + } else { + sql = String.format(sql.concat(" and TABLE_SCHEMA = '%s' and TABLE_NAME = '%s'"), targetSchema, targetTable); + } + try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) { + ResultSet rowSet = preparedStatement.executeQuery(); + while (rowSet.next()) { + //每张表单独转化 + String tableSchema = rowSet.getString("TABLE_SCHEMA"); + String tableName = rowSet.getString("TABLE_NAME"); + String showddl = String.format("SHOW CREATE TABLE `%s`.`%s`", tableSchema, tableName); + try (PreparedStatement showddlPrepareStatement = connection.prepareStatement(showddl)) { + ResultSet showddlResultSet = showddlPrepareStatement.executeQuery(); + if (showddlResultSet.next()) { + String tableddl = showddlResultSet.getString("Create Table"); + String fullDDL = "CREATE TABLE `" + tableSchema + "`.`" + tableName + "` " + tableddl.substring(tableddl.indexOf("(")); + Document doc = new Document(fullDDL).metadata(this.additionalMetadata); + documents.add(doc); + } + } + } + } + } + } catch (RuntimeException | SQLException e) { + throw new RuntimeException(e); + } + + return documents; + } + + /** + * 选项 + */ + public static class Options { + private String targetSchema; + private String targetTable; + + /** + * MYSQL，targetSchema为null，不管targetTable是否为null，则是全库全表加载 + * MYSQL，targetSchema不为null，targetTable为null则是单库全表加载 + * MYSQL，targetSchema不为null，targetTable不为null则是单库单表加载 + * 否则默认是全库全表(user表)加载 + */ + public Options loadOptions(String targetSchema, String targetTable) { + this.targetSchema = targetSchema; + this.targetTable = targetTable; + return this; + } + } +} \ No newline at end of file diff --git a/solon-ai-rag-loaders/solon-ai-load-mysql/src/test/java/features/ai/load/mysql/MysqlLoaderTest.java b/solon-ai-rag-loaders/solon-ai-load-mysql/src/test/java/features/ai/load/mysql/MysqlLoaderTest.java new file mode 100644 index 0000000000000000000000000000000000000000..7f66dba733102edc3e864f97492696d623d40f87 --- /dev/null +++ b/solon-ai-rag-loaders/solon-ai-load-mysql/src/test/java/features/ai/load/mysql/MysqlLoaderTest.java @@ -0,0 +1,46 @@ +package features.ai.load.mysql; + +import org.junit.jupiter.api.Test; +import org.noear.solon.ai.rag.Document; +import org.noear.solon.ai.rag.loader.MysqlLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class MysqlLoaderTest { + private static final Logger log = LoggerFactory.getLogger(MysqlLoaderTest.class); + + private static String url = "jdbc:mysql://127.0.0.1:3306/test"; + private static String username = "root"; + private static String password = "root"; + + @Test + public void test1() throws Exception { + //全库全表 + MysqlLoader loader = new MysqlLoader(url, username, password); + List docs = loader.load(); + System.out.println(docs); + assert docs.size() > 0; + } + + @Test + public void test2() throws Exception { + //单库全表 + MysqlLoader loader = new MysqlLoader(url, username, password); + loader.options(opt -> opt.loadOptions("zt", null)); + List docs = loader.load(); + System.out.println(docs); + assert docs.size() > 0; + } + + @Test + public void test3() throws Exception { + //单库单表 + MysqlLoader loader = new MysqlLoader(url, username, password); + loader.options(opt -> opt.loadOptions("zt", "location")); + List docs = loader.load(); + System.out.println(docs); + assert docs.size() > 0; + } +} \ No newline at end of file