diff --git a/MPC/kcal_python/README.md b/MPC/kcal_python/README.md index 86569df3cfceb035dd9610108ab7169c148fb7bf..c2ed2c606d5e6f46575f71ece16539371bac96c4 100644 --- a/MPC/kcal_python/README.md +++ b/MPC/kcal_python/README.md @@ -92,7 +92,7 @@ pip install *.whl --force-reinstall 为方便演示, 这里仅在一台机器上进行测试, 实际情况在两台分离部署的`cvm`内进行测试, 将`test`目录直接拷贝进`cvm`内的`/home/admin/dev`下, 连接`cvm`, 并打开两个终端, 分别运行以下指令, 即可进行`PSI`的测试, 数据量按需修改`test/demo.py`文件 ```bash +# 若使用虚拟环境，需要 export PYTHONPATH=path/to/myvenv/lib64/python3.11/site-packfages:$PYTHONPATH python test/demo.py --server --host "127.0.0.1" -p 9090 python test/demo.py --client --host "127.0.0.1" -p 9090 ``` - diff --git a/MPC/kcal_python/kcal/kcal.pyi b/MPC/kcal_python/kcal/kcal.pyi index f78e2f8500235973d5b3ebc0054c1f39d25b675c..7f730a0014a50069522930a4ba857d0c63415f49 100644 --- a/MPC/kcal_python/kcal/kcal.pyi +++ b/MPC/kcal_python/kcal/kcal.pyi @@ -28,41 +28,41 @@ class Add(Arithmetic): class AlgorithmsType: """ Members: - + PSI - + PIR - + MAKE_SHARE - + REVEAL_SHARE - + ADD - + SUB - + MUL - + DIV - + LESS - + LESS_EQUAL - + GREATER - + GREATER_EQUAL - + EQUAL - + NO_EQUAL - + SUM - + AVG - + MAX - + MIN """ ADD: typing.ClassVar[AlgorithmsType] # value = @@ -213,9 +213,9 @@ class Div(Arithmetic): class DummyMode: """ Members: - + NORMAL - + DUMMY """ DUMMY: typing.ClassVar[DummyMode] # value = @@ -418,7 +418,7 @@ class Pir(OperatorBase): def ServerAnswer(self) -> int: ... - def ServerPreProcess(self, arg0: DG_PairList) -> int: + def ServerPreProcess(self, arg0: Input, arg1: Input) -> int: ... def __init__(self) -> None: @@ -444,9 +444,9 @@ class RevealShare(Arithmetic): class ShareType: """ Members: - + FIX_POINT - + NON_FIX_POINT """ FIX_POINT: typing.ClassVar[ShareType] # value = @@ -512,9 +512,9 @@ class Sum(Arithmetic): class TeeMode: """ Members: - + OUTPUT_INDEX - + OUTPUT_STRING """ OUTPUT_INDEX: typing.ClassVar[TeeMode] # value = diff --git a/MPC/kcal_python/src/kcal_wrapper.cc b/MPC/kcal_python/src/kcal_wrapper.cc index a03343c459c58e9b942b491522f38b53d9ea187d..66a9ddcb25d3e248dfaabb90804eda13d630483a 100644 --- a/MPC/kcal_python/src/kcal_wrapper.cc +++ b/MPC/kcal_python/src/kcal_wrapper.cc @@ -2,12 +2,14 @@ #include #include + #include #include "context_ext.h" #include "kcal/core/operator_base.h" #include "kcal/core/operator_manager.h" #include "kcal/operator/all_operator_register.h" +#include "kcal/operator/kcal_pir.h" #include "kcal/operator/kcal_psi.h" #include "kcal/utils/io.h" @@ -43,6 +45,58 @@ void FeedKcalInput(const py::list &pyList, io::KcalInput *kcalInput) (*internalInput)->dataType = MPC_STRING; } +void FeedKcalPairList(const py::list &key, const py::list &value, io::KcalPairList *pairList) +{ + if (key.size() != value.size()) { + // 打印日志 + throw std::runtime_error("key value size don't match"); + }; + auto size = key.size(); + pairList->Get()->dgPair = new (std::nothrow) DG_Pair[size]; + if (!pairList->Get()->dgPair) { + throw std::bad_alloc(); + } + size_t i = 0; + for (i = 0; i < key.size(); ++i) { + pairList->Get()->dgPair[i].key = new (std::nothrow) DG_String(); + pairList->Get()->dgPair[i].key = new (std::nothrow) DG_String(); + pairList->Get()->dgPair[i].value = new (std::nothrow) DG_String(); + if (!pairList->Get()->dgPair[i].key || !pairList->Get()->dgPair[i].value) { + pairList->Get()->size = i + 1; + throw std::bad_alloc(); + } + // 填充key + { + if (!PyUnicode_Check(key[i].ptr())) { + throw std::runtime_error("need str"); + } + Py_ssize_t sz; + const char *utf8 = PyUnicode_AsUTF8AndSize(key[i].ptr(), &sz); + if (!utf8) { + throw std::bad_alloc(); + } + pairList->Get()->dgPair[i].key->str = strdup(utf8); + pairList->Get()->dgPair[i].key->str = strdup(utf8); + pairList->Get()->dgPair[i].key->size = static_cast(sz) + 1; + } + // 填充 value + { + if (!PyUnicode_Check(value[i].ptr())) { + throw std::runtime_error("need str"); + } + Py_ssize_t sz; + const char *utf8 = PyUnicode_AsUTF8AndSize(value[i].ptr(), &sz); + if (!utf8) { + throw std::bad_alloc(); + } + pairList->Get()->dgPair[i].value->str = strdup(utf8); + pairList->Get()->dgPair[i].value->str = strdup(utf8); + pairList->Get()->dgPair[i].value->size = static_cast(sz) + 1; + } + } + pairList->Get()->size = size; +} + void FeedPsiOutput(io::KcalOutput &kcalOutput, py::list &pyList, DG_TeeMode mode) { auto *outPtr = kcalOutput.Get(); @@ -55,7 +109,22 @@ void FeedPsiOutput(io::KcalOutput &kcalOutput, py::list &pyList, DG_TeeMode mode } } -} // namespace +void FeedKcalOutput(io::KcalOutput &kcalOutput, py::list &pyList) +{ + auto *outPtr = kcalOutput.Get(); + auto dataType = kcalOutput.Get()->dataType; + for (size_t i = 0; i < outPtr->size; ++i) { + if (dataType == MPC_STRING) { + pyList.append(outPtr->data.strings[i].str); + } else if (dataType == MPC_INT) { + pyList.append(outPtr->data.u64Numbers[i]); + } else if (dataType == MPC_DOUBLE) { + pyList.append(outPtr->data.doubleNumbers[i]); + } + } +} + +} // namespace class PyCallbackAdapter { public: @@ -68,7 +137,6 @@ public: try { py::dict nodeInfoDict; nodeInfoDict["nodeId"] = nodeInfo.nodeId; - // zero-copy py::memoryview dataMview = py::memoryview::from_buffer( const_cast(data), {static_cast(dataLen)}, {sizeof(uint8_t)}); @@ -93,7 +161,6 @@ public: try { py::dict nodeInfoDict; nodeInfoDict["nodeId"] = nodeInfo.nodeId; - // zero-copy py::memoryview bufferMview = py::memoryview::from_buffer(buffer, {static_cast(maxLen)}, {sizeof(uint8_t)}, false); @@ -120,21 +187,20 @@ void BindIoClasses(py::module_ &m) .def(py::init()) .def_static("Create", &io::KcalMpcShare::Create, py::return_value_policy::take_ownership) .def("Set", &io::KcalMpcShare::Set) - .def("Get", [](io::KcalMpcShare &self) -> DG_MpcShare* { return self.Get(); }, + .def( + "Get", [](io::KcalMpcShare &self) -> DG_MpcShare * { return self.Get(); }, py::return_value_policy::reference) .def("Size", &io::KcalMpcShare::Size) .def("Type", &io::KcalMpcShare::Type); py::class_(m, "MpcShareSet") .def(py::init<>()) - .def_static("Create", - [](const std::vector &shares) { - return io::KcalMpcShareSet::Create(shares); - }, + .def_static( + "Create", [](const std::vector &shares) { return io::KcalMpcShareSet::Create(shares); }, py::return_value_policy::take_ownership) - .def("Get", [](io::KcalMpcShareSet &self) -> DG_MpcShareSet* { return self.Get(); }, + .def( + "Get", [](io::KcalMpcShareSet &self) -> DG_MpcShareSet * { return self.Get(); }, py::return_value_policy::reference); - py::class_(m, "Input") .def(py::init<>()) .def(py::init()) @@ -143,7 +209,6 @@ void BindIoClasses(py::module_ &m) .def("Get", &io::KcalInput::Get, py::return_value_policy::reference) .def("Fill", &io::KcalInput::Fill) .def("Size", &io::KcalInput::Size); - // Alias of Input m.attr("Output") = m.attr("Input"); } @@ -161,6 +226,29 @@ void BindOtherOperators(py::module_ &m) FeedPsiOutput(kcalOutput, output, mode); return ret; }); + py::class_>(m, "Pir") + .def(py::init<>()) + .def("ServerPreProcess", + [](Pir &self, const py::list &key, py::list &value) -> int { + std::unique_ptr kcalInput(io::KcalPairList::Create()); + // build DG_PairList + FeedKcalPairList(key, value, kcalInput.get()); + int ret = self.ServerPreProcess(kcalInput->Get()); + return ret; + }) + .def("ClientQuery", + [](Pir &self, const py::list &input, py::list &output, DG_DummyMode mode) -> int { + std::unique_ptr kcalInput(io::KcalInput::Create()); + FeedKcalInput(input, kcalInput.get()); + io::KcalOutput kcalOutput; + int ret = self.ClientQuery(kcalInput->Get(), kcalOutput.GetSecondaryPointer(), mode); + FeedKcalOutput(kcalOutput, output); + return ret; + }) + .def("ServerAnswer", [](Pir &self) -> int { + int ret = self.ServerAnswer(); + return ret; + }); } PYBIND11_MODULE(kcal, m) @@ -169,6 +257,7 @@ PYBIND11_MODULE(kcal, m) py::enum_(m, "AlgorithmsType") .value("PSI", KCAL_AlgorithmsType::PSI) + .value("PIR", KCAL_AlgorithmsType::PIR) .export_values(); py::enum_(m, "TeeMode") @@ -225,6 +314,8 @@ PYBIND11_MODULE(kcal, m) switch (type) { case KCAL_AlgorithmsType::PSI: return OperatorManager::CreateOperator(context->GetKcalContext()); + case KCAL_AlgorithmsType::PIR: + return OperatorManager::CreateOperator(context->GetKcalContext()); default: throw std::runtime_error("Unsupported operator type"); } @@ -240,6 +331,10 @@ PYBIND11_MODULE(kcal, m) if (ret != 0) { throw std::runtime_error("BuildDgString failed"); } + ret = io::DataHelper::BuildDgString(strings, &dg); + if (ret != 0) { + throw std::runtime_error("BuildDgString failed"); + } return py::cast(dg); }); @@ -248,4 +343,4 @@ PYBIND11_MODULE(kcal, m) m.def("release_mpc_share", [](DG_MpcShare *share) { io::DataHelper::ReleaseMpcShare(&share); }); } -} // namespace kcal +} // namespace kcal diff --git a/MPC/kcal_python/test/arith_demo.py b/MPC/kcal_python/test/arith_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..0110f33c34c0078f0005244daed8fd579aa4fb8f --- /dev/null +++ b/MPC/kcal_python/test/arith_demo.py @@ -0,0 +1,101 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +from __future__ import annotations + +import socket +import sys + +import kcal +import argparse + +import socket_util + +_client_socket = None +_server_socket = None + +kcal.register_all_ops() + +""" +server: + nodeId: 0 + socket: _client_socket +client: + nodeId: 1 + socket: _server_socket +""" + + +def get_fd(node_info: dict) -> socket.socket: + return _server_socket if node_info['nodeId'] == 0 else _client_socket + + +def on_send_data(node_info: dict, data_buffer: memoryview) -> int: + s = get_fd(node_info) + return socket_util.send_data(s, data_buffer) + + +def on_recv_data(node_info: dict, buffer: memoryview) -> int: + s = get_fd(node_info) + return socket_util.recv_data(s, buffer) + + +def psi_demo(is_server: bool): + config = kcal.Config() + config.nodeId = 0 if is_server else 1 + config.worldSize = 2 + config.fixBits = 3 + config.threadCount = 32 + config.useSMAlg = False + + context = kcal.Context.create(config, on_send_data, on_recv_data) + + makeshare_op = kcal.create_operator(context, kcal.AlgorithmsType.NAKESHARE) + revealshare_op = kcal.create_operator(context, kcal.AlgorithmsType.NAKESHARE) + mul_op = kcal.create_operator(context, kcal.AlgorithmsType.MUL) + input0 = ["4", "3", "2", "1"] + input1 = ["1", "3", "4", "5"] + output = [] + import time + start_time = time.time() + if is_server: + makeshare_op.run() + makeshare_op.run() + mul_op.run() + revealshare_op.run() + else: + makeshare_op.run() + makeshare_op.run() + mul_op.run() + revealshare_op.run() + print(len(output)) + end_time = time.time() + duration_ms = (end_time - start_time) * 1000 # ms + print(f"run cost: {duration_ms:.2f} ms") + + +def main(argv=None): + parser = argparse.ArgumentParser(description="KCAL python wrapper demo.") + try: + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--server", action="store_true", default=False, help="start server") + group.add_argument("--client", action="store_true", default=False, help="start client") + parser.add_argument("--host", type=str, default="127.0.0.1") + parser.add_argument("-p", "--port", type=int, required=True) + args = parser.parse_args(argv) + except argparse.ArgumentParser: + parser.print_help() + sys.exit(1) + + global _client_socket, _server_socket + if args.server: + _client_socket = socket_util.init_server(args.host, args.port) + psi_demo(True) + _client_socket.close() + elif args.client: + _server_socket = socket_util.init_client(args.host, args.port) + psi_demo(False) + _server_socket.close() + + +if __name__ == "__main__": + main() diff --git a/MPC/kcal_python/test/demo.py b/MPC/kcal_python/test/demo.py index f6471dbb581bfba7749e55cd84d160d27c286cce..a2704191ee95f94b4685cae5c798e9f76b45a0cb 100644 --- a/MPC/kcal_python/test/demo.py +++ b/MPC/kcal_python/test/demo.py @@ -16,7 +16,7 @@ _server_socket = None kcal.register_all_ops() """ -server: +server: nodeId: 0 socket: _client_socket client: diff --git a/MPC/kcal_python/test/pir_demo.py b/MPC/kcal_python/test/pir_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..147c867a0f59db5e1746180037c1841d7993f2ff --- /dev/null +++ b/MPC/kcal_python/test/pir_demo.py @@ -0,0 +1,113 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +from __future__ import annotations + +import socket +import sys + +import kcal +import argparse + +import socket_util + +_client_socket = None +_server_socket = None + +kcal.register_all_ops() + +""" +server: + nodeId: 0 + socket: _client_socket +client: + nodeId: 1 + socket: _server_socket +""" + + +def get_fd(node_info: dict) -> socket.socket: + return _server_socket if node_info['nodeId'] == 0 else _client_socket + + +def on_send_data(node_info: dict, data_buffer: memoryview) -> int: + s = get_fd(node_info) + return socket_util.send_data(s, data_buffer) + + +def on_recv_data(node_info: dict, buffer: memoryview) -> int: + s = get_fd(node_info) + return socket_util.recv_data(s, buffer) + +def shake_hand(nodeId:int): + node_info = dict(nodeId = 1 - nodeId) + s = get_fd(node_info) + data_buffer = bytearray(b"hello") + if s == _server_socket: + socket_util.send_data(s, memoryview(data_buffer)) + socket_util.recv_data(s, memoryview(data_buffer)) + else: + socket_util.recv_data(s, memoryview(data_buffer)) + socket_util.send_data(s, memoryview(data_buffer)) + +def pir_demo(is_server: bool): + config = kcal.Config() + config.nodeId = 0 if is_server else 1 + config.worldSize = 2 + config.fixBits = 3 + config.threadCount = 32 + config.useSMAlg = False + + context = kcal.Context.create(config, on_send_data, on_recv_data) + + op = kcal.create_operator(context, kcal.AlgorithmsType.PIR) + + key = ["aaaa", "bbbb", "cccc", "dddd"] + value = ["1111111", "22222222222", "33333333333", "44444444"] + query = ["aaaa", "abcd"] + output = [] + import time + if is_server: + start_time = time.time() + op.ServerPreProcess(key, value) + end_time = time.time() + duration_ms = (end_time - start_time) * 1000 # ms + print(f"ServerPreprocess run cost: {duration_ms:.2f} ms") + start_time = time.time() + op.ServerAnswer() + end_time = time.time() + duration_ms = (end_time - start_time) * 1000 # ms + print(f"ServerAnswer run cost: {duration_ms:.2f} ms") + else: + start_time = time.time() + op.ClientQuery(query, output, kcal.DummyMode.NORMAL) + end_time = time.time() + duration_ms = (end_time - start_time) * 1000 # ms + print(f"ClientQuery run cost: {duration_ms:.2f} ms") + print("query result: ", output) + +def main(argv=None): + parser = argparse.ArgumentParser(description="KCAL python wrapper demo.") + try: + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--server", action="store_true", default=False, help="start server") + group.add_argument("--client", action="store_true", default=False, help="start client") + parser.add_argument("--host", type=str, default="127.0.0.1") + parser.add_argument("-p", "--port", type=int, required=True) + args = parser.parse_args(argv) + except argparse.ArgumentParser: + parser.print_help() + sys.exit(1) + + global _client_socket, _server_socket + if args.server: + _client_socket = socket_util.init_server(args.host, args.port) + pir_demo(True) + _client_socket.close() + elif args.client: + _server_socket = socket_util.init_client(args.host, args.port) + pir_demo(False) + _server_socket.close() + + +if __name__ == "__main__": + main() diff --git a/MPC/middleware/kcal/utils/io.cc b/MPC/middleware/kcal/utils/io.cc index 42b53eac17f4c9c1c74d945314e1df3742e5d345..c8a71f5925c7be03ab78bbe92c4929a8c9b0d1e2 100644 --- a/MPC/middleware/kcal/utils/io.cc +++ b/MPC/middleware/kcal/utils/io.cc @@ -34,13 +34,31 @@ int DataHelper::BuildDgString(const std::vector &strings, DG_String return DG_SUCCESS; } +void DataHelper::ReleaseDgPairList(DG_PairList *pairList) +{ + if (pairList) { + for (size_t i = 0; i < pairList->size; i++) { + if (pairList->dgPair[i].key) { + delete[] pairList->dgPair[i].key->str; + delete pairList->dgPair[i].key; + }; + if (pairList->dgPair[i].value) { + delete[] pairList->dgPair[i].value->str; + delete pairList->dgPair[i].value; + }; + } + delete [] pairList->dgPair; + pairList = nullptr; + } +} + void DataHelper::ReleaseOutput(DG_TeeOutput **output) { if (output == nullptr || *output == nullptr) { return; } if ((*output)->dataType == MPC_STRING && (*output)->data.strings != nullptr) { - for (int i = 0; i < (*output)->size; ++i) { + for (size_t i = 0; i < (*output)->size; ++i) { delete[] (*output)->data.strings[i].str; } delete[] (*output)->data.strings; @@ -142,4 +160,10 @@ KcalInput *KcalInput::Create() return input.release(); } +KcalPairList *KcalPairList::Create() +{ + std::unique_ptr pairList = std::make_unique(); + std::unique_ptr input = std::make_unique(pairList.release()); + return input.release(); +} } // namespace kcal::io diff --git a/MPC/middleware/kcal/utils/io.h b/MPC/middleware/kcal/utils/io.h index e55c345b9d37f29f6cd1a47ce79d2e0040554c72..20b38d58134a3523f334558c21d47aec444dd954 100644 --- a/MPC/middleware/kcal/utils/io.h +++ b/MPC/middleware/kcal/utils/io.h @@ -27,6 +27,7 @@ public: static void ReleaseOutput(DG_TeeOutput **output); static void ReleaseMpcShare(DG_MpcShare **share); + static void ReleaseDgPairList(DG_PairList *pairList); }; class KcalMpcShare { @@ -85,6 +86,17 @@ private: using KcalOutput = KcalInput; +class KcalPairList { +public: + KcalPairList() = default; + explicit KcalPairList(DG_PairList *pairList) : pairList_(pairList) {}; + ~KcalPairList() {DataHelper::ReleaseDgPairList(pairList_);}; + static KcalPairList *Create(); + DG_PairList *Get() {return pairList_;}; + DG_PairList **GetSecondaryPointer() {return &pairList_;}; +private: + DG_PairList *pairList_ = nullptr; +}; } // namespace kcal::io #endif // KCAL_MIDDLEWARE_IO_H diff --git a/MPC/third_party_adaptor/secrerflow/psi/README.md b/MPC/third_party_adaptor/secrerflow/psi/README.md index a8307303a8a533d6a2dcc887302c0dde634bab6a..7c25556afc4563adc31643f5792f7e8e92571471 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/README.md +++ b/MPC/third_party_adaptor/secrerflow/psi/README.md @@ -32,6 +32,7 @@ # clone 仓库，并创建一个本地分支 git clone --branch "v0.6.0.dev250507" https://github.com/secretflow/psi.git + # 进到蚂蚁 psi 目录下 cd /home/admin/dev/psi git switch -c kcal-on-v0.6.0 @@ -61,12 +62,14 @@ # bazel build //... --copt=-Wno-error=sign-compare -c opt # 创建临时目录 bin，存放 main 可执行文件 + # 编译完成后，会在 bazel-bin/psi/apps/psi_launcher 目录下生成 main 可执行文件 mkdir bin cp ./bazel-bin/psi/apps/psi_launcher/main ./bin/main ``` + ### 部署至 virtCCA 内 可让`/home/admin/dev/psi`的目录结构与`virtCCA`内部保持一致，方便进行测试 @@ -177,6 +180,7 @@ scp /tmp/receiver_input.csv \ 配置文件已在`patch`中提供，只需修改下列说明的部分进行测试 +下面以`kcal_sender.json`为例 下面以`kcal_sender.json`为例 ```json @@ -190,11 +194,13 @@ scp /tmp/receiver_input.csv \ }, "role": "ROLE_SENDER", "broadcast_result": true + "broadcast_result": true }, "input_config": { "type": "IO_TYPE_FILE_CSV", "path": "/tmp/sender_input.csv" // 当前参与方运行 psi 算法的数据输入文件位置，按需修改 }, + "output_config": { "output_config": { "type": "IO_TYPE_FILE_CSV", // 文件类型不需修改 "path": "/tmp/kcal_sender_output.csv" // 两方运行完 psi 算法后，最终交集文件的输出位置，按需修改 @@ -242,11 +248,13 @@ scp /tmp/receiver_input.csv \ ```bash cd /home/admin/dev/psi - +# 调用已经复制到./bin/下的可执行程序 main，指定使用的kcal包的lib路径 # 参与方 receiver LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/examples/psi/config/kcal_receiver.json +LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/examples/psi/config/kcal_receiver.json # 参与方 sender LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/examples/psi/config/kcal_sender.json +LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/examples/psi/config/kcal_sender.json ``` 运行完以上命令后，每个`cvm`内会在配置文件中指明的`output_config.path`路径中生成交集文件 @@ -330,8 +338,10 @@ LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/ cd /home/admin/dev/psi # 服务端 LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/examples/pir/config/kcal_pir_sender.json +LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/examples/pir/config/kcal_pir_sender.json # 客户端 LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/examples/pir/config/kcal_pir_receiver.json +LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/path/to/kcal/lib ./bin/main --config $(pwd)/examples/pir/config/kcal_pir_receiver.json ``` 运行完以上命令后，客户端`cvm`内会在配置文件中指明的`kcal_pir_receiver_config.output_file`路径中生成查询结果文件 diff --git a/MPC/third_party_adaptor/secrerflow/scql/README.md b/MPC/third_party_adaptor/secrerflow/scql/README.md index 6dc1460671f8cb69adaa3a6e410277372c720a54..2b6cda4af2339fa157691bccf247128961c5b978 100644 --- a/MPC/third_party_adaptor/secrerflow/scql/README.md +++ b/MPC/third_party_adaptor/secrerflow/scql/README.md @@ -4,7 +4,7 @@ ## 前置条件 1. 需获取 `kcal` 包，含 `include`和`lib`目录，获取链接: [https://www.hikunpeng.com/developer/download](https://www.hikunpeng.com/developer/download) -2. 需要`bazel`编译构建工具，编译环境依赖参考: [devtools/dockerfiles/release-ci-aarch64.DockerFile at main · secretflow/devtools](https://github.com/secretflow/devtools/blob/main/dockerfiles/release-ci-aarch64.DockerFile) +2. 需要`bazel`编译构建工具，编译环境依赖参考: [devtools/dockerfiles/release-ci-aarch64.DockerFile at main · secretflow/devtools](https://github.com/secretflow/devtools/blob/main/dockerfiles/release-ci-aarch64.DockerFile)，构建中的工具链 (如gcc) 也需要切换使用bazel编译环境依赖的，通常在 /root/miniforge3/bin/conda 下 3. 运行环境为`virtCCA cvm，前提是用户已经启动两个 virtCCA 的机密虚机（cvm1、cvm2）` ## 中间件使能和部署步骤 @@ -25,11 +25,11 @@ ```bash cd /home/admin/dev - + # clone 0.9.3b1 tag 的代码 git clone --branch "0.9.3b1" https://github.com/secretflow/scql.git cd scql && git switch -c local - + # 应用 patch git apply /home/admin/dev/virtCCA_sdk/MPC/third_party_adaptor/secrerflow/scql/patches/kcal.patch ``` @@ -40,7 +40,7 @@ # 假设 kcal 库已下载解压在 /opt/kcal 目录下 cp -r /opt/kcal/include /home/admin/dev/scql/engine/third_party/kcal/ cp -r /opt/kcal/lib /home/admin/dev/scql/engine/third_party/kcal/ - + # 引入中间件 cp -r /home/admin/dev/virtCCA_sdk/MPC/middleware /home/admin/dev/scql/engine/third_party/kcal_middleware ``` @@ -49,10 +49,11 @@ ```bash cd /home/admin/dev/scql - + # -DEXECUTE_IN_KCAL, 该宏为使能 kcal 加速库 bazel build //engine/exe:scqlengine -c opt --copt=-DEXECUTE_IN_KCAL --define disable_tcmalloc=true - + # 若需要指定编译器参考 + # bazel build //engine/exe:scqlengine --compiler=/root/miniforge3/bin/gcc -c opt --copt=-DEXECUTE_IN_KCAL --define disable_tcmalloc=true # 创建临时目录 bin，存放 scqlengine 可执行文件 mkdir bin; cp ./bazel-bin/engine/exe/scqlengine ./bin/ ``` @@ -75,6 +76,7 @@ 替换实际的部署`virtCCA`机器的ip，\ + 若查询时超时报错，可以将 timeout 的 120s 调大以避免超时报错 ```yaml intra_server: protocol: http @@ -148,7 +150,7 @@ ```bash # alice 下的配置文件修改 mysql 的 host ip 为宿主机 ip --embed_router_conf={"datasources":[{"id":"ds001","name":"mysql db","kind":"MYSQL","connection_str":"db=alice;user=root;password=__MYSQL_ROOT_PASSWD__;host=;auto-reconnect=true"}],"rules":[{"db":"*","table":"*","datasource_id":"ds001"}]} - + # bob 下的配置文件修改 mysql 的 host ip 为宿主机 ip --embed_router_conf={"datasources":[{"id":"ds001","name":"mysql db","kind":"MYSQL","connection_str":"db=alice;user=root;password=__MYSQL_ROOT_PASSWD__;host=mysql;auto-reconnect=true"}],"rules":[{"db":"*","table":"*","datasource_id":"ds001"}]} ``` @@ -219,18 +221,18 @@ ```bash #!/bin/bash set -e - + cur_dir=$(cd $(dirname "$0") && pwd) && cd $cur_dir - + sender_ip="xxx.xxx.xxx.xxx" receiver_ip="xxx.xxx.xxx.xxx" work_dir="/home/admin" - + chmod +x ${cur_dir}/bin/* - + ssh -t root@${sender_ip} "mkdir -p /home/admin/bin /home/admin/engine/conf /home/admin/tls /home/admin/lib" ssh -t root@${receiver_ip} "mkdir -p /home/admin/bin /home/admin/engine/conf /home/admin/tls /home/admin/lib" - + LIB_FILES=( /opt/kcal/lib/libdata_guard_common.so /opt/kcal/lib/libdata_guard.so @@ -239,7 +241,7 @@ /opt/kcal/lib/libmpc_tee.so /opt/kcal/lib/libsecurec.so ) - + # engine alice 部署 scp ${cur_dir}/bin/scqlengine root@${sender_ip}:${work_dir}/bin/ scp ${cur_dir}/examples/p2p-tutorial/engine/alice/conf/gflags.conf root@${sender_ip}:${work_dir}/engine/conf/gflags.conf @@ -285,17 +287,34 @@ bazel build //engine/exe:scqlengine -c opt --define disable_tcmalloc=true + 这里修改教程第一步`create project`命令，以及后面授权 CCL`grant xxx` 命令，其它保持不变 +```bash +# 创建工程和表 +./brokerctl create project --project-id "demo" --project-conf '{"spu_runtime_cfg":{"protocol":"SEMI2K","field":"FM64","fxp_fraction_bits":"3","max_concurrency":"16"},"session_expire_seconds":"86400"}' --host http://localhost:8081 +./brokerctl get project --host http://localhost:8081 +./brokerctl invite bob --project-id "demo" --host http://localhost:8081 +./brokerctl get invitation --host http://localhost:8082 +# bob decide to join the project with invitation-id 1 +./brokerctl process invitation 1 --response "accept" --project-id "demo" --host http://localhost:8082 +# check the project, its members should contain alice and bob +./brokerctl get project --host http://localhost:8081 +# create table for alice +./brokerctl create table ta --project-id "demo" --columns "ID string, credit_rank int, income int, age int" --ref-table alice.user_credit --db-type mysql --host http://localhost:8081 +# check the table ta +./brokerctl get table ta --host http://localhost:8081 --project-id "demo" +# create table for bob +./brokerctl create table tb --project-id "demo" --columns "ID string, order_amount double, is_active int" --ref-table bob.user_stats --db-type mysql --host http://localhost:8082 +# check the table tb +./brokerctl get table tb --host http://localhost:8082 --project-id "demo" +``` + ```bash - # 创建工程修改 - ./brokerctl create project --project-id "demo" --project-conf '{"spu_runtime_cfg":{"protocol":"SEMI2K","field":"FM64","fxp_fraction_bits":"10","max_concurrency":"16"},"session_expire_seconds":"86400"}' --host http://localhost:8081 - # CCL 授权调整 # alice set CCL for table ta ./brokerctl grant alice PLAINTEXT --project-id "demo" --table-name ta --column-name ID --host http://localhost:8081 ./brokerctl grant alice PLAINTEXT --project-id "demo" --table-name ta --column-name credit_rank --host http://localhost:8081 ./brokerctl grant alice PLAINTEXT --project-id "demo" --table-name ta --column-name income --host http://localhost:8081 ./brokerctl grant alice PLAINTEXT --project-id "demo" --table-name ta --column-name age --host http://localhost:8081 - + ./brokerctl grant bob PLAINTEXT_AFTER_JOIN --project-id "demo" --table-name ta --column-name ID --host http://localhost:8081 ./brokerctl grant bob PLAINTEXT_AFTER_AGGREGATE --project-id "demo" --table-name ta --column-name credit_rank --host http://localhost:8081 ./brokerctl grant bob PLAINTEXT_AFTER_COMPARE --project-id "demo" --table-name ta --column-name income --host http://localhost:8081 @@ -304,16 +323,16 @@ bazel build //engine/exe:scqlengine -c opt --define disable_tcmalloc=true ./brokerctl grant bob PLAINTEXT --project-id "demo" --table-name tb --column-name ID --host http://localhost:8082 ./brokerctl grant bob PLAINTEXT --project-id "demo" --table-name tb --column-name order_amount --host http://localhost:8082 ./brokerctl grant bob PLAINTEXT --project-id "demo" --table-name tb --column-name is_active --host http://localhost:8082 - + ./brokerctl grant alice PLAINTEXT_AFTER_JOIN --project-id "demo" --table-name tb --column-name ID --host http://localhost:8082 ./brokerctl grant alice PLAINTEXT_AFTER_AGGREGATE --project-id "demo" --table-name tb --column-name is_active --host http://localhost:8082 ./brokerctl grant alice PLAINTEXT_AFTER_COMPARE --project-id "demo" --table-name tb --column-name order_amount --host http://localhost:8082 ``` - + 2. 执行查询 - -这里只是演示使能`kcal`后的效果，正确性对比可以去掉`kcal`的使能然后替换`virtCCA`内部的`scqlengine`重新执行下面的`sql`语句，输出结果保持一致 - + +这里只是演示使能`kcal`后的效果，正确性对比可以去掉`kcal`的使能然后替换`virtCCA`内部的`scqlengine`重新执行下面的`sql`语句，输出结果保持一致。若有超时错误，将 "./brokerctl ... --timeout 300" 调大， 单位是秒。 + ```bash ./brokerctl run "SELECT (ta.income + tb.order_amount) < tb.order_amount FROM ta INNER JOIN tb ON ta.ID = tb.ID;" --project-id "demo" --host http://localhost:8081 --timeout 300 ./brokerctl run "SELECT (ta.income - tb.order_amount) < tb.order_amount FROM ta INNER JOIN tb ON ta.ID = tb.ID;" --project-id "demo" --host http://localhost:8081 --timeout 300 @@ -330,22 +349,3 @@ bazel build //engine/exe:scqlengine -c opt --define disable_tcmalloc=true ./brokerctl run "SELECT MAX(ta.credit_rank * tb.is_active) AS max FROM ta INNER JOIN tb ON ta.ID = tb.ID;" --project-id "demo" --host http://localhost:8081 --timeout 300 ./brokerctl run "SELECT MIN(ta.credit_rank * tb.is_active) AS min FROM ta INNER JOIN tb ON ta.ID = tb.ID;" --project-id "demo" --host http://localhost:8081 --timeout 300 ``` - - - - - - - - - - - - - - - - - - - \ No newline at end of file