From 81a9bc020e6916204ef4442952925a7763ab346b Mon Sep 17 00:00:00 2001 From: dbxue Date: Sat, 9 Jul 2022 17:13:28 +0800 Subject: [PATCH 1/2] new feature: add lz4 compress algrithom to gs_probackup --- src/bin/pg_probackup/CMakeLists.txt | 2 +- src/bin/pg_probackup/catalog.cpp | 4 +++ src/bin/pg_probackup/data.cpp | 44 +++++++++++++++++++++++++++ src/bin/pg_probackup/help.cpp | 4 +-- src/bin/pg_probackup/pg_probackup.cpp | 5 +++ src/bin/pg_probackup/pg_probackupa.h | 1 + 6 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_probackup/CMakeLists.txt b/src/bin/pg_probackup/CMakeLists.txt index a44f022d6..70e3d6985 100755 --- a/src/bin/pg_probackup/CMakeLists.txt +++ b/src/bin/pg_probackup/CMakeLists.txt @@ -13,7 +13,7 @@ execute_process( AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_probackup_SRC) -set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression) +set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${LZ4_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression) set(probackup_DEF_OPTIONS ${MACRO_OPTIONS} -DFRONTEND -DHAVE_LIBZ) set(probackup_COMPILE_OPTIONS ${PROTECT_OPTIONS} ${BIN_SECURE_OPTIONS} ${OS_OPTIONS} ${WARNING_OPTIONS} ${OPTIMIZE_OPTIONS} ${CHECK_OPTIONS}) diff --git a/src/bin/pg_probackup/catalog.cpp b/src/bin/pg_probackup/catalog.cpp index 9a89699a6..7fb42b5ef 100644 --- a/src/bin/pg_probackup/catalog.cpp +++ b/src/bin/pg_probackup/catalog.cpp @@ -2382,6 +2382,8 @@ parse_compress_alg(const char *arg) if (pg_strncasecmp("zlib", arg, len) == 0) return ZLIB_COMPRESS; + else if (pg_strncasecmp("lz4", arg, len) == 0) + return LZ4_COMPRESS; else if (pg_strncasecmp("pglz", arg, len) == 0) return PGLZ_COMPRESS; else if (pg_strncasecmp("none", arg, len) == 0) @@ -2404,6 +2406,8 @@ deparse_compress_alg(int alg) return "zlib"; case PGLZ_COMPRESS: return "pglz"; + case LZ4_COMPRESS: + return "lz4"; } return NULL; diff --git a/src/bin/pg_probackup/data.cpp b/src/bin/pg_probackup/data.cpp index c4ae4a9eb..f9a12cb5a 100644 --- a/src/bin/pg_probackup/data.cpp +++ b/src/bin/pg_probackup/data.cpp @@ -26,6 +26,7 @@ #include "thread.h" #include "common/fe_memutils.h" +#include "lz4.h" /* Union to ease operations on relation pages */ typedef struct DataPage @@ -142,6 +143,45 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) } #endif +/* Implementation of lz4 compression method */ +static int32 +lz4_compress(const char *src, size_t src_size, char *dst, size_t dst_size) +{ + int write_len; + + write_len = LZ4_compress_default(src, dst, src_size, dst_size); + if (write_len <= 0) { + elog(ERROR, "lz4 compress data failed, return: %d.", write_len); + elog(LOG, "lz4 compress error, src: [%s], src size: %u, dest size: %u, written size: %d.", + src, src_size, dst_size, write_len); + return -1; + } + + return write_len; +} + +/* Implementation of lz4 decompression method */ +static int32 +lz4_decompress(const char *src, size_t src_size, char *dst, size_t dst_size) +{ + int write_len; + + write_len = LZ4_decompress_safe(src, dst, src_size, dst_size); + if (write_len <= 0) { + elog(ERROR, "lz4 decompress data failed, return: %d.", write_len); + elog(LOG, "lz4 decompress error, src size: %u, dest size: %u, written size: %d.", + src_size, dst_size, write_len); + return -1; + } + + /* Upper will check if write_len equal to dst_size */ + if (write_len != (int)dst_size) { + elog(WARNING, "lz4 decompress data corrupted, actually written: %d, expected: %u.", write_len, dst_size); + } + + return write_len; +} + /* * Compresses source into dest using algorithm. Returns the number of bytes * written in the destination buffer, or -1 if compression fails. @@ -167,6 +207,8 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, #endif case PGLZ_COMPRESS: return pglz_compress((const char*)src, src_size, (char*)dst, PGLZ_strategy_always); + case LZ4_COMPRESS: + return lz4_compress((const char*)src, src_size, (char*)dst, dst_size); } return -1; @@ -199,6 +241,8 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, #endif case PGLZ_COMPRESS: return pglz_decompress((const char*)src, src_size, (char*)dst, dst_size, true); + case LZ4_COMPRESS: + return lz4_decompress((const char*)src, src_size, (char*)dst, dst_size); } return -1; diff --git a/src/bin/pg_probackup/help.cpp b/src/bin/pg_probackup/help.cpp index 1b4279188..b3f19ccb2 100644 --- a/src/bin/pg_probackup/help.cpp +++ b/src/bin/pg_probackup/help.cpp @@ -307,7 +307,7 @@ static void help_set_config(void) printf(_("\n Compression options:\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib','pglz','none' (default: 'none')\n")); + printf(_(" available options: 'lz4', 'zlib','pglz','none' (default: 'none')\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); @@ -482,7 +482,7 @@ static void help_backup(void) printf(_("\n Compression options:\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib', 'pglz', 'none' (default: none)\n")); + printf(_(" available options: 'lz4', 'zlib', 'pglz', 'none' (default: none)\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); diff --git a/src/bin/pg_probackup/pg_probackup.cpp b/src/bin/pg_probackup/pg_probackup.cpp index bbbe094ba..9e908cf83 100644 --- a/src/bin/pg_probackup/pg_probackup.cpp +++ b/src/bin/pg_probackup/pg_probackup.cpp @@ -892,6 +892,11 @@ compress_init(void) if (instance_config.compress_alg == ZLIB_COMPRESS && instance_config.compress_level == 0) elog(WARNING, "Compression level 0 will lead to data bloat!"); + if (instance_config.compress_alg == LZ4_COMPRESS && instance_config.compress_level > 1) + { + elog(WARNING, "Compression level will be set to 1 due to lz4 only supports level 1!"); + } + if (backup_subcmd == BACKUP_CMD) { #ifndef HAVE_LIBZ diff --git a/src/bin/pg_probackup/pg_probackupa.h b/src/bin/pg_probackup/pg_probackupa.h index d9def2cb8..32440bd5a 100644 --- a/src/bin/pg_probackup/pg_probackupa.h +++ b/src/bin/pg_probackup/pg_probackupa.h @@ -142,6 +142,7 @@ typedef enum CompressAlg #ifdef HAVE_LIBZ ZLIB_COMPRESS, #endif + LZ4_COMPRESS, } CompressAlg; typedef enum ForkName From 2f18ea2eeb4e8351a887dd2cabc0abb676b06410 Mon Sep 17 00:00:00 2001 From: dbxue Date: Thu, 4 Aug 2022 17:25:32 +0800 Subject: [PATCH 2/2] new feature: add lz4 compress algrithom to gs_probackup --- src/bin/pg_probackup/data.cpp | 4 ++-- src/bin/pg_probackup/pg_probackup.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_probackup/data.cpp b/src/bin/pg_probackup/data.cpp index f9a12cb5a..e659779d9 100644 --- a/src/bin/pg_probackup/data.cpp +++ b/src/bin/pg_probackup/data.cpp @@ -151,9 +151,9 @@ lz4_compress(const char *src, size_t src_size, char *dst, size_t dst_size) write_len = LZ4_compress_default(src, dst, src_size, dst_size); if (write_len <= 0) { - elog(ERROR, "lz4 compress data failed, return: %d.", write_len); elog(LOG, "lz4 compress error, src: [%s], src size: %u, dest size: %u, written size: %d.", src, src_size, dst_size, write_len); + elog(ERROR, "lz4 compress data failed, return: %d.", write_len); return -1; } @@ -168,9 +168,9 @@ lz4_decompress(const char *src, size_t src_size, char *dst, size_t dst_size) write_len = LZ4_decompress_safe(src, dst, src_size, dst_size); if (write_len <= 0) { - elog(ERROR, "lz4 decompress data failed, return: %d.", write_len); elog(LOG, "lz4 decompress error, src size: %u, dest size: %u, written size: %d.", src_size, dst_size, write_len); + elog(ERROR, "lz4 decompress data failed, return: %d.", write_len); return -1; } diff --git a/src/bin/pg_probackup/pg_probackup.cpp b/src/bin/pg_probackup/pg_probackup.cpp index 9e908cf83..784ed5d56 100644 --- a/src/bin/pg_probackup/pg_probackup.cpp +++ b/src/bin/pg_probackup/pg_probackup.cpp @@ -894,7 +894,7 @@ compress_init(void) if (instance_config.compress_alg == LZ4_COMPRESS && instance_config.compress_level > 1) { - elog(WARNING, "Compression level will be set to 1 due to lz4 only supports level 1!"); + elog(WARNING, "Compression level will be set to 1 due to lz4 only supports level 0 and 1!"); } if (backup_subcmd == BACKUP_CMD)