diff --git a/CMakeLists.txt b/CMakeLists.txt index 18e3d76cbea5db91bc75a6dc8ffe9925a053d16b..d482428fd32e6284699a71cab65b5cd46166fbd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,32 +13,32 @@ add_executable(kylin-ai-business-framework-service src/main.cpp src/servicemanager.cpp src/servicemanager.h src/datamanagement/datamanagementprocessorglue.c src/datamanagement/datamanagementprocessorglue.h src/datamanagement/datamanagementprocessor.cpp src/datamanagement/datamanagementprocessor.h - src/util/python/autotokenizer.py - src/util/parser/parser.h src/util/parser/parser.cpp - src/util/parser/txtparser.h src/util/parser/txtparser.cpp - src/util/parser/pdfparser.h src/util/parser/pdfparser.cpp - src/util/python/pythonutil.h src/util/python/pythonutil.cpp - src/util/parser/officepyparserwrapper.h src/util/parser/officepyparserwrapper.cpp - src/util/python/officeparser.py - src/util/parser/docxparser.h src/util/parser/docxparser.cpp - src/util/python/docxparser.py - src/util/parser/pptxparser.h src/util/parser/pptxparser.cpp - src/util/python/pptxparser.py - src/util/parser/fileparserfactory.cpp src/util/parser/fileparserfactory.h - src/util/autotokenizer.cpp - src/util/autotokenizer.h - src/util/imageembedder.cpp - src/util/imageembedder.h - src/util/math.cpp - src/util/math.h - src/util/textembedder.cpp - src/util/textembedder.h + src/utils/python/autotokenizer.py + src/utils/parser/parser.h src/utils/parser/parser.cpp + src/utils/parser/txtparser.h src/utils/parser/txtparser.cpp + src/utils/parser/pdfparser.h src/utils/parser/pdfparser.cpp + src/utils/pythonutil.h src/utils/pythonutil.cpp + src/utils/parser/officepyparserwrapper.h src/utils/parser/officepyparserwrapper.cpp + src/utils/python/officeparser.py + src/utils/parser/docxparser.h src/utils/parser/docxparser.cpp + src/utils/python/docxparser.py + src/utils/parser/pptxparser.h src/utils/parser/pptxparser.cpp + src/utils/python/pptxparser.py + src/utils/parser/fileparserfactory.cpp src/utils/parser/fileparserfactory.h + src/utils/autotokenizer.cpp + src/utils/autotokenizer.h + src/utils/imageembedder.cpp + src/utils/imageembedder.h + src/utils/math.cpp + src/utils/math.h + src/utils/textembedder.cpp + src/utils/textembedder.h src/embeddingtaskmanager/baseembeddingservice.h src/embeddingtaskmanager/baseembeddingservice.cpp src/embeddingtaskmanager/embeddingtask.h src/embeddingtaskmanager/embeddingtask.cpp src/embeddingtaskmanager/embeddingtaskmanager.h src/embeddingtaskmanager/embeddingtaskmanager.cpp src/embeddingtaskmanager/imageembeddingservice.h src/embeddingtaskmanager/imageembeddingservice.cpp src/embeddingtaskmanager/textembeddingservice.h src/embeddingtaskmanager/textembeddingservice.cpp - src/util/python/pythonthreadlocker.h + src/utils/pythonthreadlocker.h src/datamanagement/datamanagementjsonhelper.cpp src/datamanagement/datamanagementjsonhelper.h src/datamanagement/datamanagementdatabase.cpp @@ -75,8 +75,11 @@ target_link_libraries(kylin-ai-business-framework-service PkgConfig::vectordb ) +set(UTILS_RESOURCE_PATH /usr/share/kylin-ai-business-framework/utils) + include(GNUInstallDirs) install(TARGETS kylin-ai-business-framework-service LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) +install(DIRECTORY src/utils/python DESTINATION ${UTILS_RESOURCE_PATH}) diff --git a/src/datamanagement/datamanagementdatabase.cpp b/src/datamanagement/datamanagementdatabase.cpp index 0e03334f01ba702c036bb61780e7114bb25d1c54..386b2c38c5ea156d0f5c5e5f15e08fa00196734d 100644 --- a/src/datamanagement/datamanagementdatabase.cpp +++ b/src/datamanagement/datamanagementdatabase.cpp @@ -5,7 +5,7 @@ #include #include -static const char* VECTOR_DATABASE_PATH = "/usr/share/kylin-ai-runtime/datamanagement/database/"; +static const char* VECTOR_DATABASE_PATH = "/usr/share/kylin-ai-business-framework/datamanagement/database/"; static const char* VECTOR_DATABASE_NAME = "search"; static const char* FILE_INFO_COLLECTION_NAME = "files-info"; static const char* VISION_FILE_CONTENT_COLLECTION_NAME = "vision-files-content-vector"; diff --git a/src/datamanagement/datamanagementservice.cpp b/src/datamanagement/datamanagementservice.cpp index 002bf637bf8b229c48e45692a6b44009427fdb1f..7a73b8dd328687b050dd1d75a28dbd3d2bfb17f5 100644 --- a/src/datamanagement/datamanagementservice.cpp +++ b/src/datamanagement/datamanagementservice.cpp @@ -1,11 +1,11 @@ #include "datamanagementservice.h" #include "thirdparty/threadpool/async.h" #include "datamanagement/datamanagementjsonhelper.h" -#include "../util/parser/parser.h" -#include "../util/parser/fileparserfactory.h" +#include "utils/parser/parser.h" +#include "utils/parser/fileparserfactory.h" #include "datamanagement/segmenttokenizer.h" -#include "../embeddingtaskmanager/embeddingtaskmanager.h" -#include "../embeddingtaskmanager/embeddingtask.h" +#include "embeddingtaskmanager/embeddingtaskmanager.h" +#include "embeddingtaskmanager/embeddingtask.h" #include diff --git a/src/embeddingtaskmanager/imageembeddingservice.h b/src/embeddingtaskmanager/imageembeddingservice.h index 061c5352adcfe784fc11be319cb6f89204ec945b..a23df7c12a2e88c83f66c70a51520d21512bb62b 100644 --- a/src/embeddingtaskmanager/imageembeddingservice.h +++ b/src/embeddingtaskmanager/imageembeddingservice.h @@ -2,7 +2,7 @@ #define IMAGEEMBEDDINGSERVICE_H #include "embeddingtaskmanager/baseembeddingservice.h" -#include "util/imageembedder.h" +#include "utils/imageembedder.h" #include diff --git a/src/embeddingtaskmanager/textembeddingservice.cpp b/src/embeddingtaskmanager/textembeddingservice.cpp index ce9f34494f18345f807b6976605b8980e781c539..e9440225687c26aa8103b4678b857803e93bb73b 100644 --- a/src/embeddingtaskmanager/textembeddingservice.cpp +++ b/src/embeddingtaskmanager/textembeddingservice.cpp @@ -1,5 +1,5 @@ #include "textembeddingservice.h" -#include "util/textembedder.h" +#include "utils/textembedder.h" #include diff --git a/src/embeddingtaskmanager/textembeddingservice.h b/src/embeddingtaskmanager/textembeddingservice.h index 66f1426f1b020ab5c257f774bbbfc732a7dbd597..6343ad71b8ce40c4c99eeb4d84300581eb27552f 100644 --- a/src/embeddingtaskmanager/textembeddingservice.h +++ b/src/embeddingtaskmanager/textembeddingservice.h @@ -2,7 +2,7 @@ #define TEXTEMBEDDINGSERVICE_H #include "embeddingtaskmanager/baseembeddingservice.h" -#include "util/textembedder.h" +#include "utils/textembedder.h" #include diff --git a/src/util/autotokenizer.cpp b/src/utils/autotokenizer.cpp similarity index 97% rename from src/util/autotokenizer.cpp rename to src/utils/autotokenizer.cpp index 575f62dbaefd2fe6c8d281aee52bc9b0b9d4eb11..2c45c75d6b11a18eb9a07009c6ebb1f0d615fdb5 100644 --- a/src/util/autotokenizer.cpp +++ b/src/utils/autotokenizer.cpp @@ -1,7 +1,7 @@ #include "autotokenizer.h" -#include "python/pythonutil.h" -#include "python/pythonthreadlocker.h" +#include "utils/pythonutil.h" +#include "utils/pythonthreadlocker.h" #include diff --git a/src/util/autotokenizer.h b/src/utils/autotokenizer.h similarity index 100% rename from src/util/autotokenizer.h rename to src/utils/autotokenizer.h diff --git a/src/util/imageembedder.cpp b/src/utils/imageembedder.cpp similarity index 93% rename from src/util/imageembedder.cpp rename to src/utils/imageembedder.cpp index 1a8795a361321545ddd0e949b91e63ea436e0af0..a5868b5dd135f7409e086e93d7065664fc0ce897 100644 --- a/src/util/imageembedder.cpp +++ b/src/utils/imageembedder.cpp @@ -5,9 +5,9 @@ #include -static const char* TXT_MODEL_FOLDER = "/usr/share/kylin-ai-runtime/datamanagement/models/cn-clip-onnx"; -static const char* VISION_MODEL_PATH = "/usr/share/kylin-ai-runtime/datamanagement/models/cn-clip-onnx/vit-b-16.img.fp32.onnx"; -static const char* TXT_MODEL_PATH = "/usr/share/kylin-ai-runtime/datamanagement/models/cn-clip-onnx/vit-b-16.txt.fp32.onnx"; +static const char* TXT_MODEL_FOLDER = "/usr/share/kylin-datamanagement-models/cn-clip-onnx"; +static const char* VISION_MODEL_PATH = "/usr/share/kylin-datamanagement-models/cn-clip-onnx/vit-b-16.img.fp32.onnx"; +static const char* TXT_MODEL_PATH = "/usr/share/kylin-datamanagement-models/cn-clip-onnx/vit-b-16.txt.fp32.onnx"; static const int INPUT_CONTEXT_LENGTH = 52; diff --git a/src/util/imageembedder.h b/src/utils/imageembedder.h similarity index 100% rename from src/util/imageembedder.h rename to src/utils/imageembedder.h diff --git a/src/util/math.cpp b/src/utils/math.cpp similarity index 100% rename from src/util/math.cpp rename to src/utils/math.cpp diff --git a/src/util/math.h b/src/utils/math.h similarity index 100% rename from src/util/math.h rename to src/utils/math.h diff --git a/src/util/parser/docxparser.cpp b/src/utils/parser/docxparser.cpp similarity index 100% rename from src/util/parser/docxparser.cpp rename to src/utils/parser/docxparser.cpp diff --git a/src/util/parser/docxparser.h b/src/utils/parser/docxparser.h similarity index 100% rename from src/util/parser/docxparser.h rename to src/utils/parser/docxparser.h diff --git a/src/util/parser/fileparserfactory.cpp b/src/utils/parser/fileparserfactory.cpp similarity index 100% rename from src/util/parser/fileparserfactory.cpp rename to src/utils/parser/fileparserfactory.cpp diff --git a/src/util/parser/fileparserfactory.h b/src/utils/parser/fileparserfactory.h similarity index 100% rename from src/util/parser/fileparserfactory.h rename to src/utils/parser/fileparserfactory.h diff --git a/src/util/parser/officepyparserwrapper.cpp b/src/utils/parser/officepyparserwrapper.cpp similarity index 96% rename from src/util/parser/officepyparserwrapper.cpp rename to src/utils/parser/officepyparserwrapper.cpp index c97dcc7408ae0d6c4d44ee3a7a1e693eaa081905..a3b21a6271f1f1f9a13c68990c3e6e62c31c1fe6 100644 --- a/src/util/parser/officepyparserwrapper.cpp +++ b/src/utils/parser/officepyparserwrapper.cpp @@ -1,7 +1,7 @@ #include "officepyparserwrapper.h" -#include "../python/pythonutil.h" -#include "../python/pythonthreadlocker.h" +#include "utils/pythonutil.h" +#include "utils/pythonthreadlocker.h" OfficePyParserWrapper::OfficePyParserWrapper() { diff --git a/src/util/parser/officepyparserwrapper.h b/src/utils/parser/officepyparserwrapper.h similarity index 100% rename from src/util/parser/officepyparserwrapper.h rename to src/utils/parser/officepyparserwrapper.h diff --git a/src/util/parser/parser.cpp b/src/utils/parser/parser.cpp similarity index 100% rename from src/util/parser/parser.cpp rename to src/utils/parser/parser.cpp diff --git a/src/util/parser/parser.h b/src/utils/parser/parser.h similarity index 100% rename from src/util/parser/parser.h rename to src/utils/parser/parser.h diff --git a/src/util/parser/pdfparser.cpp b/src/utils/parser/pdfparser.cpp similarity index 100% rename from src/util/parser/pdfparser.cpp rename to src/utils/parser/pdfparser.cpp diff --git a/src/util/parser/pdfparser.h b/src/utils/parser/pdfparser.h similarity index 100% rename from src/util/parser/pdfparser.h rename to src/utils/parser/pdfparser.h diff --git a/src/util/parser/pptxparser.cpp b/src/utils/parser/pptxparser.cpp similarity index 100% rename from src/util/parser/pptxparser.cpp rename to src/utils/parser/pptxparser.cpp diff --git a/src/util/parser/pptxparser.h b/src/utils/parser/pptxparser.h similarity index 100% rename from src/util/parser/pptxparser.h rename to src/utils/parser/pptxparser.h diff --git a/src/util/parser/txtparser.cpp b/src/utils/parser/txtparser.cpp similarity index 100% rename from src/util/parser/txtparser.cpp rename to src/utils/parser/txtparser.cpp diff --git a/src/util/parser/txtparser.h b/src/utils/parser/txtparser.h similarity index 100% rename from src/util/parser/txtparser.h rename to src/utils/parser/txtparser.h diff --git a/src/util/python/autotokenizer.py b/src/utils/python/autotokenizer.py similarity index 100% rename from src/util/python/autotokenizer.py rename to src/utils/python/autotokenizer.py diff --git a/src/util/python/docxparser.py b/src/utils/python/docxparser.py similarity index 100% rename from src/util/python/docxparser.py rename to src/utils/python/docxparser.py diff --git a/src/util/python/officeparser.py b/src/utils/python/officeparser.py similarity index 100% rename from src/util/python/officeparser.py rename to src/utils/python/officeparser.py diff --git a/src/util/python/pptxparser.py b/src/utils/python/pptxparser.py similarity index 100% rename from src/util/python/pptxparser.py rename to src/utils/python/pptxparser.py diff --git a/src/util/python/pythonthreadlocker.h b/src/utils/pythonthreadlocker.h similarity index 100% rename from src/util/python/pythonthreadlocker.h rename to src/utils/pythonthreadlocker.h diff --git a/src/util/python/pythonutil.cpp b/src/utils/pythonutil.cpp similarity index 100% rename from src/util/python/pythonutil.cpp rename to src/utils/pythonutil.cpp diff --git a/src/util/python/pythonutil.h b/src/utils/pythonutil.h similarity index 83% rename from src/util/python/pythonutil.h rename to src/utils/pythonutil.h index c38a64ea0ec1183225addd8d866ded1c0781ceea..cf1fbdf5eddc2be02eef01bbc2cb982ceb665907 100644 --- a/src/util/python/pythonutil.h +++ b/src/utils/pythonutil.h @@ -7,7 +7,7 @@ #include #include -#define DATA_MANAGEMENT_PYTHON_PATH "/usr/share/kylin-ai-runtime/datamanagement/python/" +#define DATA_MANAGEMENT_PYTHON_PATH "/usr/share/kylin-ai-business-framework/utils/python/" namespace pythonutil { diff --git a/src/util/textembedder.cpp b/src/utils/textembedder.cpp similarity index 96% rename from src/util/textembedder.cpp rename to src/utils/textembedder.cpp index 3eaaa980082ecdbe71689d1d4c932bcd91a1dd3e..2c0b64bd2387412662576a12b1864fbe68437c59 100644 --- a/src/util/textembedder.cpp +++ b/src/utils/textembedder.cpp @@ -3,8 +3,8 @@ #include -static const char* MODEL_FOLDER = "/usr/share/kylin-ai-runtime/datamanagement/models/bge-m3-onnx"; -static const char* MODEL_PATH = "/usr/share/kylin-ai-runtime/datamanagement/models/bge-m3-onnx/model.onnx"; +static const char* MODEL_FOLDER = "/usr/share/kylin-datamanagement-models/bge-m3-onnx"; +static const char* MODEL_PATH = "/usr/share/kylin-datamanagement-models/bge-m3-onnx/model.onnx"; TextEmbedder::TextEmbedder() : tokenizer_(MODEL_FOLDER) diff --git a/src/util/textembedder.h b/src/utils/textembedder.h similarity index 100% rename from src/util/textembedder.h rename to src/utils/textembedder.h