From b95f34bc2721a1c37446617ee5648901baaef979 Mon Sep 17 00:00:00 2001 From: wangyan Date: Sat, 11 May 2024 15:27:44 +0800 Subject: [PATCH 1/4] =?UTF-8?q?Fix:=E4=BC=98=E5=8C=96=E4=BF=AE=E6=94=B9emb?= =?UTF-8?q?edding=E7=9A=84=E5=87=BD=E6=95=B0=E5=90=8D=E5=AD=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datamanagement/datamanagementservice.cpp | 22 +++++++++----------- src/datamanagement/datamanagementservice.h | 8 +++---- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/datamanagement/datamanagementservice.cpp b/src/datamanagement/datamanagementservice.cpp index 7a73b8d..40a5e0d 100644 --- a/src/datamanagement/datamanagementservice.cpp +++ b/src/datamanagement/datamanagementservice.cpp @@ -204,7 +204,7 @@ std::vector DataManagementService::makeDatasByImageFilePathAndFormat continue; } - std::vector vector = imageEmbedderImage(filepath); + std::vector vector = imageSideEmbeddingImage(filepath); if (vector.empty()) { std::cerr << "File: " << filepath << " embedding error" << std::endl; continue; @@ -239,9 +239,9 @@ std::vector DataManagementService::makeDatasByTextFilePathAndFormat( } SegmentTokenizer::segmentTokenize(file.contents, 100); - std::cout << "Start embedding text file " << path << std::endl; - std::vector> vectors = textEmbedderTexts(file.contents); - std::cout << "Finish embedding text file " << path << std::endl; + std::cout << "start encode text file " << path << std::endl; + std::vector> vectors = textSideEmbeddingTexts(file.contents); + std::cout << "finish encode text file " << path << std::endl; if (vectors.empty()) { std::cerr << "File: " << path << " embedding error" << std::endl; @@ -264,10 +264,8 @@ std::string DataManagementService::doSimilaritySearch( return std::string(); } - std::cout << "Start embedding for search " << searchConditions << std::endl; - std::vector visionSideEmbedding = imageEmbedderText(condition.first); - std::vector textSideEmbedding = textEmbedderText(condition.first); - std::cout << "Finish embedding for search" << std::endl; + std::vector visionSideEmbedding = imageSideEmbeddingText(condition.first); + std::vector textSideEmbedding = textSideEmbeddingText(condition.first); if (visionSideEmbedding.empty() && textSideEmbedding.empty()) { errorCode = DATA_MANAGEMENT_INVAILD_PATH; return std::string(); @@ -416,7 +414,7 @@ void DataManagementService::doUpdateFilesContent(const std::string &fileinfosJso database_.destroyClient(); } -std::vector DataManagementService::textEmbedderText(const std::string& text) +std::vector DataManagementService::textSideEmbeddingText(const std::string& text) { std::vector textEmbedding; EmbeddingTask task(text, EmbeddingTask::EmbeddingDataType::Text); @@ -425,7 +423,7 @@ std::vector DataManagementService::textEmbedderText(const std::string& te return textEmbedding; } -std::vector> DataManagementService::textEmbedderTexts(const std::vector& texts) +std::vector> DataManagementService::textSideEmbeddingTexts(const std::vector& texts) { std::vector> textEmbeddings; std::vector textEmbedding; @@ -444,7 +442,7 @@ std::vector> DataManagementService::textEmbedderTexts(const s return textEmbeddings; } -std::vector DataManagementService::imageEmbedderText(const std::string& text) +std::vector DataManagementService::imageSideEmbeddingText(const std::string& text) { std::vector textEmbedding; EmbeddingTask task(text, EmbeddingTask::EmbeddingDataType::Text); @@ -453,7 +451,7 @@ std::vector DataManagementService::imageEmbedderText(const std::string& t return textEmbedding; } -std::vector DataManagementService::imageEmbedderImage(const std::string& filePath) +std::vector DataManagementService::imageSideEmbeddingImage(const std::string& filePath) { std::vector imageEmbedding; EmbeddingTask task(filePath, EmbeddingTask::EmbeddingDataType::FilePath); diff --git a/src/datamanagement/datamanagementservice.h b/src/datamanagement/datamanagementservice.h index 1c8226f..83e0b46 100644 --- a/src/datamanagement/datamanagementservice.h +++ b/src/datamanagement/datamanagementservice.h @@ -56,10 +56,10 @@ private: bool isSupportedImageFormat(const std::string& format) const; bool isSupportedTextFormat(const std::string& format) const; - std::vector textEmbedderText(const std::string& text); - std::vector> textEmbedderTexts(const std::vector& texts); - std::vector imageEmbedderText(const std::string& text); - std::vector imageEmbedderImage(const std::string& filePath); + std::vector textSideEmbeddingText(const std::string& text); + std::vector> textSideEmbeddingTexts(const std::vector& texts); + std::vector imageSideEmbeddingText(const std::string& text); + std::vector imageSideEmbeddingImage(const std::string& filePath); private: DataManagementDatabase database_; -- Gitee From 0c49850a06a72a7bff92e6db589f0a80b3996ca8 Mon Sep 17 00:00:00 2001 From: wangyan Date: Mon, 13 May 2024 14:41:10 +0800 Subject: [PATCH 2/4] =?UTF-8?q?Fix:=20=E5=A2=9E=E5=8A=A0=E5=88=A4=E7=A9=BA?= =?UTF-8?q?=E5=A4=84=E7=90=86=EF=BC=8C=E4=B8=8D=E5=B0=86=E7=A9=BA=E7=9A=84?= =?UTF-8?q?=E5=90=91=E9=87=8F=E5=8C=96=E7=BB=93=E6=9E=9C=E5=8A=A0=E5=85=A5?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E9=98=9F=E5=88=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datamanagement/datamanagementservice.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/datamanagement/datamanagementservice.cpp b/src/datamanagement/datamanagementservice.cpp index 40a5e0d..d0de71a 100644 --- a/src/datamanagement/datamanagementservice.cpp +++ b/src/datamanagement/datamanagementservice.cpp @@ -426,17 +426,21 @@ std::vector DataManagementService::textSideEmbeddingText(const std::strin std::vector> DataManagementService::textSideEmbeddingTexts(const std::vector& texts) { std::vector> textEmbeddings; - std::vector textEmbedding; std::vector tasks; for (int i = 0; i < texts.size(); i++) { + if(texts.empty()) { + continue; + } EmbeddingTask task(texts.at(i), EmbeddingTask::EmbeddingDataType::Text); embeddingTaskManager_->addTextSideTask(task); tasks.push_back(task); } for (int i = 0; i < tasks.size(); i++) { - textEmbedding = embeddingTaskManager_->getTextSideResult(tasks.at(i).id()); - textEmbeddings.push_back(textEmbedding); + std::vector textEmbedding = embeddingTaskManager_->getTextSideResult(tasks.at(i).id()); + if(!textEmbedding.empty()) { + textEmbeddings.push_back(textEmbedding); + } } return textEmbeddings; -- Gitee From cccc9702648e5a091788803aad214a390f83d6fc Mon Sep 17 00:00:00 2001 From: wangyan Date: Mon, 13 May 2024 15:45:46 +0800 Subject: [PATCH 3/4] =?UTF-8?q?Fix:=E4=BF=AE=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E7=9A=84=E5=88=9B=E5=BB=BA=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datamanagement/datamanagementdatabase.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/datamanagement/datamanagementdatabase.cpp b/src/datamanagement/datamanagementdatabase.cpp index 386b2c3..d8d9a65 100644 --- a/src/datamanagement/datamanagementdatabase.cpp +++ b/src/datamanagement/datamanagementdatabase.cpp @@ -5,13 +5,17 @@ #include #include -static const char* VECTOR_DATABASE_PATH = "/usr/share/kylin-ai-business-framework/datamanagement/database/"; static const char* VECTOR_DATABASE_NAME = "search"; static const char* FILE_INFO_COLLECTION_NAME = "files-info"; static const char* VISION_FILE_CONTENT_COLLECTION_NAME = "vision-files-content-vector"; static const char* TEXT_FILE_CONTENT_COLLECTION_NAME = "text-files-content-vector"; namespace { +std::string getVectorDatabasePath() { + std::string home_env = std::getenv("HOME") ? std::string(std::getenv("HOME")) : ""; + std::string vector_database_path = home_env + "/.local/kylin-ai-business-framework/datamanagement/database/"; + return vector_database_path; +} std::string generateUUID() { std::random_device rd; @@ -52,7 +56,7 @@ DataManagementDatabase::~DataManagementDatabase() bool DataManagementDatabase::createClient(uid_t uid) { - std::string databasePath = VECTOR_DATABASE_PATH + std::to_string(uid) + "/" + VECTOR_DATABASE_NAME; + std::string databasePath = getVectorDatabasePath() + "/" + VECTOR_DATABASE_NAME; if (database_.createClient(databasePath) != VectorDBErrorCode::Success) { return false; -- Gitee From 815869f17480575f98473e4acc74063c8e2e8cd1 Mon Sep 17 00:00:00 2001 From: wangyan Date: Mon, 13 May 2024 16:18:54 +0800 Subject: [PATCH 4/4] =?UTF-8?q?Feat:=E5=A2=9E=E5=8A=A0kylin-ai-business-fr?= =?UTF-8?q?amework-service=E8=87=AA=E5=90=AF=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 4 ++-- .../user/kylin-ai-business-framework-service.service | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 configs/lib/systemd/user/kylin-ai-business-framework-service.service diff --git a/CMakeLists.txt b/CMakeLists.txt index d482428..231e5fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,7 +79,7 @@ set(UTILS_RESOURCE_PATH /usr/share/kylin-ai-business-framework/utils) include(GNUInstallDirs) install(TARGETS kylin-ai-business-framework-service - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + RUNTIME DESTINATION /usr/bin ) install(DIRECTORY src/utils/python DESTINATION ${UTILS_RESOURCE_PATH}) +install(DIRECTORY configs/lib/systemd/user/ DESTINATION /usr/lib/systemd/user/) \ No newline at end of file diff --git a/configs/lib/systemd/user/kylin-ai-business-framework-service.service b/configs/lib/systemd/user/kylin-ai-business-framework-service.service new file mode 100644 index 0000000..6b33ff6 --- /dev/null +++ b/configs/lib/systemd/user/kylin-ai-business-framework-service.service @@ -0,0 +1,12 @@ +[Unit] +Description=Kylin Ai Business Framework Service +After=login@.service + +[Service] +ExecStart=/usr/bin/kylin-ai-business-framework-service +Restart=always +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=default.target \ No newline at end of file -- Gitee