From 5b1fbef5595b3102bde7d05efeac4903adfa4017 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 14 Apr 2025 17:29:37 +0800 Subject: [PATCH] READ WRITE BIG BUFFER --- src/cmd/wrcmd.c | 9 --- src/common/wr_defs.h | 2 +- src/common/wr_filesystem.c | 14 ++++ src/common/wr_meta.c | 40 +++++++++++ src/common/wr_meta.h | 31 +++++++++ src/common_api/wr_api_impl.c | 108 +++++++++++++---------------- src/common_api/wr_api_impl.h | 4 +- src/service/wr_service.c | 17 ++++- test/demo/test1.c | 127 ++++++++++++++++++++++++++++++----- test/gtest/test_wr_api.cc | 21 ++++++ 10 files changed, 284 insertions(+), 89 deletions(-) create mode 100644 src/common/wr_meta.c create mode 100644 src/common/wr_meta.h diff --git a/src/cmd/wrcmd.c b/src/cmd/wrcmd.c index dc91459..4aff750 100644 --- a/src/cmd/wrcmd.c +++ b/src/cmd/wrcmd.c @@ -1339,15 +1339,6 @@ static status_t wr_load_buffer_cache_from_file(int32 file_fd, wr_vg_info_item_t return CM_SUCCESS; } -static status_t wr_get_group_num(int32 file_fd, int64 *offset, uint32 *group_num) -{ - int32 read_size = 0; - status_t status = cm_read_file(file_fd, group_num, sizeof(uint32), &read_size); - WR_RETURN_IFERR2(status, LOG_DEBUG_ERR("Failed to read group num.")); - *offset += (int64)sizeof(uint32); - return CM_SUCCESS; -} - bool32 wr_check_software_version(int32 file_fd, int64 *offset) { int32 read_size = 0; diff --git a/src/common/wr_defs.h b/src/common/wr_defs.h index 7a24f94..fa1443e 100644 --- a/src/common/wr_defs.h +++ b/src/common/wr_defs.h @@ -184,7 +184,7 @@ extern "C" { #define CM_MAX_UDS_FILE_PERMISSIONS (uint16)777 #define CM_DEF_UDS_FILE_PERMISSIONS (uint16)600 -#define WR_MAX_PACKET_SIZE (uint32)2136 /* sizeof(wr_packet_head_t) + CM_ALIGN4(WR_FILE_PATH_MAX_LENGTH + 1) */ +#define WR_MAX_PACKET_SIZE (uint32)(10240) /* 10KB */ #define WR_MAX_PACKET_DATA_SIZE (((WR_MAX_PACKET_SIZE) - sizeof(wr_packet_head_t)) - sizeof(uint32)) #define WR_PARAM_BUFFER_SIZE (uint32)1024 diff --git a/src/common/wr_filesystem.c b/src/common/wr_filesystem.c index 122ced2..318a019 100644 --- a/src/common/wr_filesystem.c +++ b/src/common/wr_filesystem.c @@ -118,6 +118,20 @@ status_t wr_filesystem_query_file_num(const char *vfs_name, uint32_t *file_num) return CM_SUCCESS; } +status_t wr_filesystem_get_file_end_position(const char *file_path, off_t *end_position) { + if (!file_path || !end_position) { + LOG_AND_RETURN_ERROR("[FS] Invalid parameters: file_path or end_position is NULL"); + } + + struct stat file_stat; + if (stat(WR_FS_GET_PATH(file_path), &file_stat) != 0) { + LOG_AND_RETURN_ERROR("[FS] Failed to stat file: %s", file_path); + } + + *end_position = file_stat.st_size; + return CM_SUCCESS; +} + status_t wr_filesystem_open(const char *file_path, int *fd) { *fd = open(WR_FS_GET_PATH(file_path), O_RDWR | O_SYNC, 0); if (*fd == -1) { diff --git a/src/common/wr_meta.c b/src/common/wr_meta.c new file mode 100644 index 0000000..c4b5437 --- /dev/null +++ b/src/common/wr_meta.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * WR is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * wr_volume.c + * + * + * IDENTIFICATION + * src/common/wr_filesystem.c + * + * ------------------------------------------------------------------------- + */ +#include "wr_filesystem.h" +#ifndef WIN32 +#include +#include +#include +#endif // !WIN32 +#include "wr_file.h" +#include "wr_thv.h" +#include "wr_param.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif diff --git a/src/common/wr_meta.h b/src/common/wr_meta.h new file mode 100644 index 0000000..acac019 --- /dev/null +++ b/src/common/wr_meta.h @@ -0,0 +1,31 @@ +#ifndef __WR_META_H__ +#define __WR_META_H__ + +#include "wr_defs.h" +#include "cm_date.h" +#include "wr_file_def.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define WR_META_MAGIC 0x57524D45 +#define WR_META_VERSION 1 +#define WR_META_CHECKSUM 0 +#define WR_META_RESERVED 0 + +#define WR_META_BLOCK_SIZE 512 + +typedef struct wr_meta { + uint32 magic; + uint32 version; + uint32 checksum; + uint32 reserved; +} wr_meta_t; + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/common_api/wr_api_impl.c b/src/common_api/wr_api_impl.c index b3430a4..9a2a35f 100644 --- a/src/common_api/wr_api_impl.c +++ b/src/common_api/wr_api_impl.c @@ -857,92 +857,78 @@ int64 wr_seek_file_impl(wr_conn_t *conn, int handle, int64 offset, int origin) return new_offset; } -status_t wr_read_write_file_core(wr_conn_t *conn, int64 offset, void *buf, int32 size, int64_t handle) +status_t wr_write_file_impl(wr_conn_t *conn, int handle, const void *buf, int size, long long offset) { - LOG_DEBUG_INF("wr write file entry, handle:%lld, size:%d", handle, size); + if (size < 0) { + LOG_DEBUG_ERR("File size is invalid: %d.", size); + return CM_ERROR; + } + LOG_DEBUG_INF("wr pwrite file entry, handle:%lld, size:%d, offset:%lld", handle, size, offset); + wr_write_file_info_t send_info; - send_info.offset = offset; send_info.handle = handle; - send_info.size = size; - send_info.buf = buf; - - status_t status = wr_msg_interact(conn, WR_CMD_WRITE_FILE, (void *)&send_info, NULL); - LOG_DEBUG_INF("wr write file leave"); - return status; -} -status_t wr_read_write_file(wr_conn_t *conn, int32 handle, void *buf, int32 size, int64 offset, bool32 is_read) -{ status_t status; - wr_file_context_t *context = NULL; - wr_rw_param_t param; + int total_size = 0; + int curr_size; + int remaining_size = size; - if (size < 0) { - LOG_DEBUG_ERR("File size is invalid: %d.", size); - return CM_ERROR; - } - LOG_DEBUG_INF("wr read write file entry, handle:%d, is_read:%u", handle, is_read); + while (total_size < size) { + curr_size = (remaining_size > WR_RW_STEP_SIZE) ? WR_RW_STEP_SIZE : remaining_size; + send_info.offset = offset + total_size; + send_info.size = curr_size; + send_info.buf = (const char *)buf + total_size; - /* WR_RETURN_IF_ERROR(wr_latch_context_by_handle(conn, handle, &context, LATCH_MODE_EXCLUSIVE)); - bool mode_match = is_read ? (context->mode & WR_FILE_MODE_READ) : (context->mode & WR_FILE_MODE_WRITE); - if (!mode_match) { - wr_unlatch(&context->latch); - WR_THROW_ERROR(ERR_WR_FILE_RDWR_INSUFF_PER, is_read ? "read" : "write", context->mode); - return CM_ERROR; + status = wr_msg_interact(conn, WR_CMD_WRITE_FILE, (void *)&send_info, NULL); + if (status != CM_SUCCESS) { + LOG_RUN_ERR("Failed to write file, total_size:%d, size:%d, offset:%lld, errmsg:%s.", + total_size, size - total_size, offset, strerror(errno)); + return CM_ERROR; + } + + total_size += curr_size; + remaining_size -= curr_size; } - */ - // wr_init_rw_param(¶m, conn, handle, context, context->offset, WR_FALSE); - // param.is_read = is_read; - status = wr_read_write_file_core(conn, offset, buf, size, handle); - // wr_unlatch(&context->latch); - LOG_DEBUG_INF("wr read write file leave"); - return status; + LOG_DEBUG_INF("wr pwrite file leave"); + return CM_SUCCESS; } -status_t wr_write_file_impl(wr_conn_t *conn, int handle, const void *buf, int size, long long offset) +status_t wr_pread_file_impl(wr_conn_t *conn, int handle, void *buf, int size, long long offset) { - wr_file_context_t *context = NULL; - wr_write_file_info_t send_info; - wr_rw_param_t param; - if (size < 0) { LOG_DEBUG_ERR("File size is invalid: %d.", size); return CM_ERROR; } - LOG_DEBUG_INF("wr pwrite file entry, handle:%lld, size:%d, offset:%lld", handle, size, offset); + LOG_DEBUG_INF("wr pread file entry, handle:%lld, size:%d, offset:%lld", handle, size, offset); - send_info.offset = offset; + wr_write_file_info_t send_info; send_info.handle = handle; - send_info.size = size; - send_info.buf = buf; - status_t status = wr_msg_interact(conn, WR_CMD_WRITE_FILE, (void *)&send_info, NULL); - LOG_DEBUG_INF("wr pwrite file leave"); + status_t status; + int total_size = 0; + int curr_size; + int remaining_size = size; - return status; -} + while (total_size < size) { + curr_size = (remaining_size > WR_RW_STEP_SIZE) ? WR_RW_STEP_SIZE : remaining_size; + send_info.offset = offset + total_size; + send_info.size = curr_size; + send_info.buf = (char *)buf + total_size; -status_t wr_pread_file_impl(wr_conn_t *conn, int handle, const void *buf, int size, long long offset) -{ - wr_write_file_info_t send_info; - wr_rw_param_t param; + status = wr_msg_interact(conn, WR_CMD_READ_FILE, (void *)&send_info, (void *)&send_info); + if (status != CM_SUCCESS) { + LOG_RUN_ERR("Failed to read file, total_size:%d, size:%d, offset:%lld, errmsg:%s.", + total_size, size - total_size, offset, strerror(errno)); + return CM_ERROR; + } - if (size < 0) { - LOG_DEBUG_ERR("File size is invalid: %d.", size); - return CM_ERROR; + total_size += curr_size; + remaining_size -= curr_size; } - LOG_DEBUG_INF("wr pwrite file entry, handle:%lld, size:%d, offset:%lld", handle, size, offset); - send_info.offset = offset; - send_info.handle = handle; - send_info.size = size; - send_info.buf = buf; - - status_t status = wr_msg_interact(conn, WR_CMD_READ_FILE, (void *)&send_info, (void *)&send_info); LOG_DEBUG_INF("wr pread file leave"); - - return status; + return CM_SUCCESS; } status_t wr_fallocate_impl(wr_conn_t *conn, int handle, int mode, long long int offset, long long int length) diff --git a/src/common_api/wr_api_impl.h b/src/common_api/wr_api_impl.h index c8bbe8b..b8db80f 100644 --- a/src/common_api/wr_api_impl.h +++ b/src/common_api/wr_api_impl.h @@ -235,7 +235,7 @@ status_t wr_get_fname_impl(int handle, char *fname, int fname_size); status_t wr_vfs_query_file_num_impl(wr_conn_t *conn, const char *vfs_name, uint32 *file_num); status_t wr_pwrite_file_impl(wr_conn_t *conn, int handle, const void *buf, int size, long long offset); -status_t wr_pread_file_impl(wr_conn_t *conn, int handle, const void *buf, int size, long long offset); +status_t wr_pread_file_impl(wr_conn_t *conn, int handle, void *buf, int size, long long offset); status_t wr_get_addr_impl(wr_conn_t *conn, int32 handle, long long offset, char *pool_name, char *image_name, char *obj_addr, unsigned int *obj_id, unsigned long int *obj_offset); gft_node_t *wr_get_node_by_path_impl(wr_conn_t *conn, const char *path); @@ -280,6 +280,8 @@ status_t wr_set_main_inst_on_server(wr_conn_t *conn); #define WR_UNLOCK_VG_META_S(vg_item, session) \ (void)wr_unlock_shm_meta_s_with_stack((session), (vg_item)->vg_latch, CM_FALSE) +#define WR_RW_STEP_SIZE (8192) + #ifdef __cplusplus } #endif diff --git a/src/service/wr_service.c b/src/service/wr_service.c index 48858ea..4dc03e3 100644 --- a/src/service/wr_service.c +++ b/src/service/wr_service.c @@ -464,17 +464,30 @@ static status_t wr_process_read_file(wr_session_t *session) WR_RETURN_IF_ERROR(wr_get_int64(&session->recv_pack, &offset)); WR_RETURN_IF_ERROR(wr_get_int64(&session->recv_pack, &handle)); WR_RETURN_IF_ERROR(wr_get_int64(&session->recv_pack, &size)); - char *buf = (char *)malloc(size); + + // Allocate one extra byte for the null terminator + char *buf = (char *)malloc(size + 1); if (buf == NULL) { LOG_DEBUG_ERR("Failed to malloc buffer for read file."); return CM_ERROR; } - memset(buf, 0, size); + + // Initialize the buffer and ensure it is null-terminated + memset(buf, 0, size + 1); + + // Read the file content into the buffer WR_RETURN_IF_ERROR(wr_filesystem_pread(handle, offset, size, buf)); + + // Convert the buffer to a text_t structure text_t data; cm_str2text(buf, &data); + + // Send the data WR_RETURN_IF_ERROR(wr_put_text(&session->send_pack, &data)); + + // Free the allocated buffer free(buf); + return CM_SUCCESS; } diff --git a/test/demo/test1.c b/test/demo/test1.c index 2729ae3..4647cfa 100644 --- a/test/demo/test1.c +++ b/test/demo/test1.c @@ -1,26 +1,123 @@ -#include -#include -#include +/* gcc test1.c -I WR/src/interface -lwrapi -L WR/output/lib */ #include -#include -#include #include +#include #include "wr_api.h" +#include "wr_errno.h" -/* gcc 1.c -I /home/czk/bianyi/WalRecord/src/interface -lwrapi -L /home/czk/bianyi/WalRecord/output/lib */ +#define TEST_LOG_DIR "./test_log" +#define TEST_DIR "testdir1" +#define TEST_FILE1 "testdir1/testfile1" +#define ONE_GB 1024 * 1024 * 1024 +#define SERVER_ADDR "127.0.0.1:19225" -int main(void) { - bool result = false; - int fd = -1; +int main() { int errorcode = 0; const char *errormsg = NULL; - char *fileName = "wr_file_write"; - int ret = wr_file_write(0x20000001, "hello world", 10); - if (ret != 0) { + wr_instance_handle g_inst_handle = NULL; + int handle1 = 0; + + // 初始化日志 + if (wr_init_logger(TEST_LOG_DIR, 255, 100, ONE_GB) != WR_SUCCESS) { + fprintf(stderr, "Failed to initialize logger\n"); + return EXIT_FAILURE; + } + + // 创建实例 + if (wr_create_instance(SERVER_ADDR, &g_inst_handle) != WR_SUCCESS) { + wr_get_error(&errorcode, &errormsg); + fprintf(stderr, "Error creating instance: %d : %s\n", errorcode, errormsg); + return EXIT_FAILURE; + } + + // 创建VFS + if (wr_vfs_create(TEST_DIR, g_inst_handle) != WR_SUCCESS) { + wr_get_error(&errorcode, &errormsg); + fprintf(stderr, "Error creating VFS: %d : %s\n", errorcode, errormsg); + return EXIT_FAILURE; + } + + // 创建文件 + if (wr_file_create(TEST_FILE1, 0, g_inst_handle) != WR_SUCCESS) { + wr_get_error(&errorcode, &errormsg); + fprintf(stderr, "Error creating file: %d : %s\n", errorcode, errormsg); + return EXIT_FAILURE; + } + + // 打开文件 + if (wr_file_open(TEST_FILE1, 0, &handle1, g_inst_handle) != WR_SUCCESS) { + wr_get_error(&errorcode, &errormsg); + fprintf(stderr, "Error opening file: %d : %s\n", errorcode, errormsg); + return EXIT_FAILURE; + } + + // 创建一个大于8KB的数据块 + const int large_data_size = 10 * 1024; // 10KB + char *large_data = (char *)malloc(large_data_size); + if (!large_data) { + fprintf(stderr, "Memory allocation failed\n"); + return EXIT_FAILURE; + } + memset(large_data, 'A', large_data_size); + + // 写入大数据块到文件 + if (wr_file_pwrite(handle1, large_data, large_data_size, 0, g_inst_handle) != WR_SUCCESS) { + wr_get_error(&errorcode, &errormsg); + fprintf(stderr, "Error writing to file: %d : %s\n", errorcode, errormsg); + free(large_data); + return EXIT_FAILURE; + } + + // 读取大数据块 + char *read_buffer = (char *)malloc(large_data_size); + if (!read_buffer) { + fprintf(stderr, "Memory allocation failed\n"); + free(large_data); + return EXIT_FAILURE; + } + if (wr_file_pread(handle1, read_buffer, large_data_size, 0, g_inst_handle) != WR_SUCCESS) { + wr_get_error(&errorcode, &errormsg); + fprintf(stderr, "Error reading from file: %d : %s\n", errorcode, errormsg); + free(large_data); + free(read_buffer); + return EXIT_FAILURE; + } + + // 验证读取的数据是否与写入的数据一致 + if (memcmp(large_data, read_buffer, large_data_size) != 0) { + fprintf(stderr, "Data mismatch\n"); + free(large_data); + free(read_buffer); + return EXIT_FAILURE; + } + + printf("Large data write and read test passed.\n"); + + // 清理动态分配的内存 + free(large_data); + free(read_buffer); + + // 关闭文件 + if (wr_file_close(handle1, g_inst_handle) != WR_SUCCESS) { + wr_get_error(&errorcode, &errormsg); + fprintf(stderr, "Error closing file: %d : %s\n", errorcode, errormsg); + return EXIT_FAILURE; + } + + // 删除文件 + if (wr_file_delete(TEST_FILE1, g_inst_handle) != WR_SUCCESS) { + wr_get_error(&errorcode, &errormsg); + fprintf(stderr, "Error deleting file: %d : %s\n", errorcode, errormsg); + return EXIT_FAILURE; + } + + // 删除VFS + if (wr_vfs_delete(TEST_DIR, g_inst_handle) != WR_SUCCESS) { wr_get_error(&errorcode, &errormsg); - printf("%d : %s\n", errorcode, errormsg); + fprintf(stderr, "Error deleting VFS: %d : %s\n", errorcode, errormsg); + return EXIT_FAILURE; } - printf("%lld\n", ret); - return 0; + printf("All operations completed successfully.\n"); + return EXIT_SUCCESS; } \ No newline at end of file diff --git a/test/gtest/test_wr_api.cc b/test/gtest/test_wr_api.cc index 278a420..9eddf4f 100644 --- a/test/gtest/test_wr_api.cc +++ b/test/gtest/test_wr_api.cc @@ -68,6 +68,27 @@ TEST_F(WrApiTest, TestWrfileOpen) { EXPECT_EQ(result3, WR_SUCCESS); } +TEST_F(WrApiTest, TestWrfileWriteReadLargeData) { + // 创建一个大于8KB的数据块 + const int large_data_size = 100 * 1024; // 10KB + char *large_data = new char[large_data_size]; + memset(large_data, 'A', large_data_size); // 用'A'填充数据 + + // 写入大数据块到文件 + EXPECT_EQ(wr_file_pwrite(handle1, large_data, large_data_size, 0, g_inst_handle), WR_SUCCESS); + + // 读取大数据块 + char *read_buffer = new char[large_data_size]; + EXPECT_EQ(wr_file_pread(handle1, read_buffer, large_data_size, 0, g_inst_handle), WR_SUCCESS); + + // 验证读取的数据是否与写入的数据一致 + EXPECT_EQ(memcmp(large_data, read_buffer, large_data_size), 0); + + // 清理动态分配的内存 + delete[] large_data; + delete[] read_buffer; +} + TEST_F(WrApiTest, TestWrfileWriteRead) { const char *data1 = "hello world 1"; const char *data2 = "hello world 2"; -- Gitee