From fb4813eaf98098423f959434d967b603dad95531 Mon Sep 17 00:00:00 2001 From: Taoqiuyu <1042656162@qq.com> Date: Fri, 30 May 2025 14:36:29 +0800 Subject: [PATCH] =?UTF-8?q?embedding=20ddr=E7=94=A8=E4=BE=8B=E9=9A=8F?= =?UTF-8?q?=E6=9C=BA=E5=A4=B1=E8=B4=A5=EF=BC=8C=E5=9B=9E=E9=80=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tests/emb_table/embedding_ddr_test.cpp | 316 --------------------- 1 file changed, 316 deletions(-) diff --git a/src/tests/emb_table/embedding_ddr_test.cpp b/src/tests/emb_table/embedding_ddr_test.cpp index c2aad833..1555cc50 100644 --- a/src/tests/emb_table/embedding_ddr_test.cpp +++ b/src/tests/emb_table/embedding_ddr_test.cpp @@ -74,21 +74,6 @@ protected: stringstream savePath; }; -TEST_F(EmbeddingDDRTest, TestEmptyFuncNoReturn) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - const size_t testNum = 100; - vector testKeys; - for (size_t i = 0; i < testNum; ++i) { - testKeys.push_back(i); - } - - table->Key2Offset(testKeys, TRAIN_CHANNEL_ID); - table->Key2OffsetForDp(testKeys, EVAL_CHANNEL_ID); - table->EvictKeys(testKeys); - EXPECT_EQ(testKeys.size(), testNum); -} - TEST_F(EmbeddingDDRTest, EvictDeleteEmb) { shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); @@ -100,304 +85,3 @@ TEST_F(EmbeddingDDRTest, EvictDeleteEmb) table->EvictDeleteEmb(testKeys); EXPECT_EQ(testKeys.size(), 100); } - -TEST_F(EmbeddingDDRTest, TestSaveShouldCreateFileWhenSaveDeltaIsFalse) -{ - const string tableName = "test1"; - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - int result = ock::ctr::Factory::Create(factory); - EXPECT_EQ(result, 0); - auto embCache = std::shared_ptr(); - factory->CreateEmbCacheManager(embCache); - table->SetEmbCache(embCache); - - stringstream savePathTwo; - savePathTwo << "test_dir/SaveDeltaFalse" << rankInfo_.rankId; - map keyInfo; - table->SetFileSystemPtr(savePathTwo.str()); - table->Save(savePathTwo.str(), 1, false, keyInfo); - stringstream saveKeyPathTwo; - saveKeyPathTwo << savePathTwo.str() << "/" << tableName << "/key"; - stringstream saveEmbeddingPathTwo; - saveEmbeddingPathTwo << savePathTwo.str() << "/" << tableName << "/embedding"; - - EXPECT_EQ(access(saveKeyPathTwo.str().c_str(), F_OK), 0); - EXPECT_EQ(access(saveEmbeddingPathTwo.str().c_str(), F_OK), 0); - embCache->Destroy(); -} - -TEST_F(EmbeddingDDRTest, TestSaveShouldCreateFileWhenSaveDeltaIsTrue) -{ - const string tableName = "test1"; - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - factory->CreateEmbCacheManager(table->embCache); - - stringstream savePathOne; - savePathOne << "test_dir/SaveDeltaTrue" << rankInfo_.rankId; - map keyInfo; - table->SetFileSystemPtr(savePathOne.str()); - table->Save(savePathOne.str(), 1, true, keyInfo); - stringstream saveKeyPathOne; - saveKeyPathOne << savePathOne.str() << "/" << tableName << "/key"; - stringstream saveEmbeddingPathOne; - saveEmbeddingPathOne << savePathOne.str() << "/" << tableName << "/embedding"; - - EXPECT_EQ(access(saveKeyPathOne.str().c_str(), F_OK), 0); - EXPECT_EQ(access(saveEmbeddingPathOne.str().c_str(), F_OK), 0); -} - - -TEST_F(EmbeddingDDRTest, TestLoadShouldThrowOckErrorWhenEmbCacheNoData) -{ - const string tableName = "test1"; - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - factory->CreateEmbCacheManager(table->embCache); - stringstream savePathTwo; - savePathTwo << "test_dir/SaveDeltaFalse" << rankInfo_.rankId; - - stringstream fileKeyPath; - fileKeyPath << savePathTwo.str() << "/" << tableName << "/key" << "/" << "slice_" << rankInfo_.rankId << ".data"; - stringstream newFileKeyPath; - newFileKeyPath << savePathTwo.str() << "/" << tableName << "/key" << "/" << "slice.data"; - RenameFilePath(fileKeyPath.str(), newFileKeyPath.str()); - - stringstream fileEmbeddingPath; - fileEmbeddingPath << savePathTwo.str() << "/" << tableName << "/embedding/slice_" << rankInfo_.rankId << ".data"; - stringstream newFileEmbeddingPath; - newFileEmbeddingPath << savePathTwo.str() << "/" << tableName << "/embedding" << "/" << "slice.data"; - RenameFilePath(fileEmbeddingPath.str(), newFileEmbeddingPath.str()); - - map keyInfo; - table->SetFileSystemPtr(savePathTwo.str()); - map> trainKeySetTwo; - vector warmStartTablesTwo; - EXPECT_THROW(table->Load(savePathTwo.str(), trainKeySetTwo, warmStartTablesTwo), std::invalid_argument); -} - -TEST_F(EmbeddingDDRTest, TestSaveKeyShouldCreateFileWhenNoError) -{ - const string tableName = "test1"; - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - const size_t keySize = 5; - vector testKeys = {4, 8, 12, 16, 20}; - - table->SetFileSystemPtr(savePath.str()); - table->SaveKey(savePath.str(), testKeys); - - stringstream saveKeyPath; - saveKeyPath << savePath.str() << "/" << tableName << "/key"; - EXPECT_EQ(access(saveKeyPath.str().c_str(), F_OK), 0); -} - -TEST_F(EmbeddingDDRTest, TestSaveEmbeddingShouldCreateFileWhenNoError) -{ - const string tableName = "test1"; - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - const size_t keySize = 5; - const size_t embSize = 10; - table->embSize_ = embSize; - vector> testEmbeddings(keySize, vector(table->embSize_)); - for (size_t i = 0; i < keySize; ++i) { - for (size_t j = 0; j < table->embSize_; ++j) { - testEmbeddings[i][j] = float(i) ; - } - } - - table->SetFileSystemPtr(savePath.str()); - table->SaveEmbedding(savePath.str(), testEmbeddings); - - stringstream saveEmbeddingPath; - saveEmbeddingPath << savePath.str() << "/" << tableName << "/embedding"; - EXPECT_EQ(access(saveEmbeddingPath.str().c_str(), F_OK), 0); -} - -TEST_F(EmbeddingDDRTest, TestSaveOptimizerSlotShouldCreateFileWhenNoError) -{ - const string tableName = "test1"; - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - OptimizerInfo info; - info.optimName = "Adam"; - info.optimParams = {"momentum", "velocity"}; - table->SetOptimizerInfo(info); - - const size_t keySize = 5; - const size_t embSize = 10; - table->embSize_ = embSize; - const size_t doubleEmbDim = 20; - vector> optimizerSlots(keySize, vector(doubleEmbDim)); - for (size_t i = 0; i < keySize; ++i) { - for (size_t j = 0; j < doubleEmbDim; ++j) { - optimizerSlots[i][j] = float(i) ; - } - } - - table->SetFileSystemPtr(savePath.str()); - table->SaveOptimizerSlot(savePath.str(), optimizerSlots, keySize); - - stringstream saveMomentumPath; - saveMomentumPath << savePath.str() << "/" << tableName << "/" << info.optimName + "_" + info.optimParams[0]; - stringstream saveVelocityPath; - saveVelocityPath << savePath.str() << "/" << tableName << "/" << info.optimName + "_" + info.optimParams[1]; - EXPECT_EQ(access(saveMomentumPath.str().c_str(), F_OK), 0); - EXPECT_EQ(access(saveVelocityPath.str().c_str(), F_OK), 0); -} - -TEST_F(EmbeddingDDRTest, TestSaveOptimizerSlotShouldNoReturnWhenSizeIsZero) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - const size_t keySize = 5; - vector> optimizerSlots; - table->SaveOptimizerSlot(savePath.str(), optimizerSlots, keySize); -} - -TEST_F(EmbeddingDDRTest, TestSaveOptimizerSlotShouldThrowErrorWhenSizeNoEqual) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - const size_t keySize = 5; - vector> optimizerSlots = {{1.0, 2.0}, {3.0, 4.0}}; - EXPECT_THROW(table->SaveOptimizerSlot(savePath.str(), optimizerSlots, keySize), std::runtime_error); -} - -TEST_F(EmbeddingDDRTest, TestSetOptimizerInfo) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - OptimizerInfo info; - info.optimName = "Adam"; - info.optimParams = {"momentum", "velocity"}; - table->SetOptimizerInfo(info); - EXPECT_EQ(table->optimName, info.optimName); - EXPECT_EQ(table->optimParams, info.optimParams); -} - -TEST_F(EmbeddingDDRTest, TestLoadKeyShouldThrowErrorWhenFileNoExist) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - vector keys; - table->SetFileSystemPtr(savePath.str()); - EXPECT_THROW(table->LoadKey(savePath.str(), keys), std::runtime_error); -} - -TEST_F(EmbeddingDDRTest, TestLoadKeyShouldLoadKeySizeEqualSaveKeySizeWhenNoError) -{ - const string tableName = "test1"; - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - stringstream filePath; - filePath << savePath.str() << "/" << tableName << "/key" << "/" << "slice_" << rankInfo_.rankId << ".data"; - stringstream newFilePath; - newFilePath << savePath.str() << "/" << tableName << "/key" << "/" << "slice.data"; - RenameFilePath(filePath.str(), newFilePath.str()); - - if (rankInfo_.rankId == 0) { - const size_t keySize = 5; - vector keys; - table->SetFileSystemPtr(savePath.str()); - table->LoadKey(savePath.str(), keys); - - EXPECT_EQ(keys.size(), keySize); - EXPECT_EQ(table->hostLoadOffset.size(), keySize); - } -} - -TEST_F(EmbeddingDDRTest, TestLoadEmbeddingShouldEmbeddingSizeEqualSaveKeySizeWhenNoError) -{ - const string tableName = "test1"; - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - stringstream filePath; - filePath << savePath.str() << "/" << tableName << "/embedding" << "/" << "slice_" << rankInfo_.rankId << ".data"; - stringstream newFilePath; - newFilePath << savePath.str() << "/" << tableName << "/embedding" << "/" << "slice.data"; - RenameFilePath(filePath.str(), newFilePath.str()); - - const size_t keySize = 5; - const size_t embSize = 10; - table->embSize_ = embSize; - table->hostLoadOffset = {0, 1, 2, 3, 4}; - vector> embeddings; - table->SetFileSystemPtr(savePath.str()); - table->LoadEmbedding(savePath.str(), embeddings); - - EXPECT_EQ(embeddings.size(), keySize); -} - -TEST_F(EmbeddingDDRTest, TestLoadOptimizerSlotShouldNoReturnWhenOptimParamsSizeIsZero) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - vector> optimizerSlots; - table->LoadOptimizerSlot(savePath.str(), optimizerSlots); -} - -TEST_F(EmbeddingDDRTest, TestLoadOptimizerSlotShouldOptimizerSlotsSizeEqualSavekeySizeWhenNoError) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - OptimizerInfo info; - info.optimName = "Adam"; - info.optimParams = {"momentum", "velocity"}; - table->SetOptimizerInfo(info); - - stringstream filePath1; - filePath1 << savePath.str() << "/test1/" << info.optimName + "_" + info.optimParams[0] - << "/" << "slice_" << rankInfo_.rankId << ".data"; - stringstream newFilePath1; - newFilePath1 << savePath.str() << "/test1/" << info.optimName + "_" + info.optimParams[0] << "/" << "slice.data"; - stringstream filePath2; - filePath2 << savePath.str() << "/test1/" << info.optimName + "_" + info.optimParams[1] - << "/" << "slice_" << rankInfo_.rankId << ".data"; - stringstream newFilePath2; - newFilePath2 << savePath.str() << "/test1/" << info.optimName + "_" + info.optimParams[1] << "/" << "slice.data"; - RenameFilePath(filePath1.str(), newFilePath1.str()); - RenameFilePath(filePath2.str(), newFilePath2.str()); - - const size_t keySize = 5; - const size_t embSize = 10; - table->embSize_ = embSize; - table->hostLoadOffset = {0, 1, 2, 3, 4}; - vector> optimizerSlots; - table->SetFileSystemPtr(savePath.str()); - table->LoadOptimizerSlot(savePath.str(), optimizerSlots); - EXPECT_EQ(optimizerSlots.size(), keySize); -} - -TEST_F(EmbeddingDDRTest, TestGetDeviceOffsetShouldThrowError) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - EXPECT_THROW(table->GetDeviceOffset(), std::runtime_error); -} - -TEST_F(EmbeddingDDRTest, TestGetTableInfo) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - TableInfo tableInfo = table->GetTableInfo(); - EXPECT_EQ(tableInfo.devVocabSize, embInfo_.devVocabSize); - EXPECT_EQ(tableInfo.hostVocabSize, embInfo_.hostVocabSize); - EXPECT_EQ(tableInfo.name, embInfo_.name); -} - -TEST_F(EmbeddingDDRTest, TestCapacity) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - EXPECT_EQ(table->capacity(), 0); -} - -TEST_F(EmbeddingDDRTest, TestSyncLatestEmbeddingShouldThrowOckErrorWhenEmbCacheNoData) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - factory->CreateEmbCacheManager(table->embCache); - shared_ptr hdTransfer = std::make_shared(); - table->SetHDTransfer(hdTransfer.get()); - EXPECT_THROW(table->SyncLatestEmbedding(1), std::invalid_argument); - hdTransfer->Destroy(); -} - -TEST_F(EmbeddingDDRTest, TestBackUpTrainStatusShouldThrowError) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - factory->CreateEmbCacheManager(table->embCache); - EXPECT_THROW(table->BackUpTrainStatus(), std::runtime_error); -} - -TEST_F(EmbeddingDDRTest, TestRecoverTrainStatusShouldThrowError) -{ - shared_ptr table = std::make_shared(embInfo_, rankInfo_, 0); - factory->CreateEmbCacheManager(table->embCache); - EXPECT_THROW(table->RecoverTrainStatus(), std::runtime_error); -} \ No newline at end of file -- Gitee