From 4be139a81c33ff40652079825f6f941741d18cd6 Mon Sep 17 00:00:00 2001 From: dbxue Date: Sat, 9 Jul 2022 17:22:20 +0800 Subject: [PATCH 1/2] new feature: gs_probackup support zstd compress algrithom --- src/bin/pg_probackup/CMakeLists.txt | 2 +- src/bin/pg_probackup/Makefile | 4 +-- src/bin/pg_probackup/catalog.cpp | 4 +++ src/bin/pg_probackup/data.cpp | 47 ++++++++++++++++++++++++++++ src/bin/pg_probackup/help.cpp | 4 +-- src/bin/pg_probackup/pg_probackupa.h | 1 + 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/bin/pg_probackup/CMakeLists.txt b/src/bin/pg_probackup/CMakeLists.txt index a44f022d6..2a1f17f44 100755 --- a/src/bin/pg_probackup/CMakeLists.txt +++ b/src/bin/pg_probackup/CMakeLists.txt @@ -13,7 +13,7 @@ execute_process( AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_probackup_SRC) -set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression) +set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${ZSTD_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression) set(probackup_DEF_OPTIONS ${MACRO_OPTIONS} -DFRONTEND -DHAVE_LIBZ) set(probackup_COMPILE_OPTIONS ${PROTECT_OPTIONS} ${BIN_SECURE_OPTIONS} ${OS_OPTIONS} ${WARNING_OPTIONS} ${OPTIMIZE_OPTIONS} ${CHECK_OPTIONS}) diff --git a/src/bin/pg_probackup/Makefile b/src/bin/pg_probackup/Makefile index 9f3e8178a..1b17edea8 100644 --- a/src/bin/pg_probackup/Makefile +++ b/src/bin/pg_probackup/Makefile @@ -32,12 +32,12 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global EXTRA_CLEAN += logging.h -LDFLAGS += -L$(LZ4_LIB_PATH) +LDFLAGS += -L$(LZ4_LIB_PATH) -L$(ZSTD_LIB_PATH) LIBS += -llz4 ifeq ($(enable_lite_mode), no) LIBS += -lgssapi_krb5_gauss -lgssrpc_gauss -lkrb5_gauss -lkrb5support_gauss -lk5crypto_gauss -lcom_err_gauss endif -PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_builddir)/$(subdir) -I$(LZ4_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_builddir)/$(subdir) -I$(LZ4_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH) # add page_compression so .h LDFLAGS += -L../../lib/page_compression PG_CPPFLAGS = -I../../lib/page_compression diff --git a/src/bin/pg_probackup/catalog.cpp b/src/bin/pg_probackup/catalog.cpp index 9a89699a6..f825a33d3 100644 --- a/src/bin/pg_probackup/catalog.cpp +++ b/src/bin/pg_probackup/catalog.cpp @@ -2382,6 +2382,8 @@ parse_compress_alg(const char *arg) if (pg_strncasecmp("zlib", arg, len) == 0) return ZLIB_COMPRESS; + else if (pg_strncasecmp("zstd", arg, len) == 0) + return ZSTD_COMPRESS; else if (pg_strncasecmp("pglz", arg, len) == 0) return PGLZ_COMPRESS; else if (pg_strncasecmp("none", arg, len) == 0) @@ -2404,6 +2406,8 @@ deparse_compress_alg(int alg) return "zlib"; case PGLZ_COMPRESS: return "pglz"; + case ZSTD_COMPRESS: + return "zstd"; } return NULL; diff --git a/src/bin/pg_probackup/data.cpp b/src/bin/pg_probackup/data.cpp index c4ae4a9eb..4230dce2f 100644 --- a/src/bin/pg_probackup/data.cpp +++ b/src/bin/pg_probackup/data.cpp @@ -26,6 +26,8 @@ #include "thread.h" #include "common/fe_memutils.h" +#include "zstd.h" + /* Union to ease operations on relation pages */ typedef struct DataPage @@ -142,6 +144,47 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) } #endif + +/* Implementation of zstd compression method */ +static int32 +zstd_compress(const char *src, size_t src_size, char *dst, size_t dst_size, int level) +{ + size_t write_len; + + write_len = ZSTD_compress(dst, dst_size, src, src_size, level); + if (ZSTD_isError(write_len)) { + elog(ERROR, "zstd compress data failed, return: %u.", write_len); + elog(LOG, "zstd compress error, src: [%s], src size: %u, dest size: %u, written size: %u.", + src, src_size, dst_size, write_len); + return -1; + } + + return (int32)write_len; +} + +/* Implementation of zstd decompression method */ +static int32 +zstd_decompress(const char *src, size_t src_size, char *dst, size_t dst_size) +{ + size_t write_len; + + write_len = ZSTD_decompress(dst, dst_size, src, src_size); + if (ZSTD_isError(write_len)) { + elog(ERROR, "zstd decompress data failed, return: %u.", write_len); + elog(LOG, "zstd decompress error, src size: %u, dest size: %u, written size: %u.", + src_size, dst_size, write_len); + return -1; + } + + /* Upper will check if write_len equal to dst_size */ + if (write_len != dst_size) { + elog(WARNING, "zstd decompress data corrupted, actually written: %u, expected: %u.", write_len, dst_size); + } + + return (int32)write_len; +} + + /* * Compresses source into dest using algorithm. Returns the number of bytes * written in the destination buffer, or -1 if compression fails. @@ -167,6 +210,8 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, #endif case PGLZ_COMPRESS: return pglz_compress((const char*)src, src_size, (char*)dst, PGLZ_strategy_always); + case ZSTD_COMPRESS: + return zstd_compress((const char*)src, src_size, (char*)dst, dst_size, level); } return -1; @@ -199,6 +244,8 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, #endif case PGLZ_COMPRESS: return pglz_decompress((const char*)src, src_size, (char*)dst, dst_size, true); + case ZSTD_COMPRESS: + return zstd_decompress((const char*)src, src_size, (char*)dst, dst_size); } return -1; diff --git a/src/bin/pg_probackup/help.cpp b/src/bin/pg_probackup/help.cpp index 1b4279188..908f00ea7 100644 --- a/src/bin/pg_probackup/help.cpp +++ b/src/bin/pg_probackup/help.cpp @@ -307,7 +307,7 @@ static void help_set_config(void) printf(_("\n Compression options:\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib','pglz','none' (default: 'none')\n")); + printf(_(" available options: 'zlib', 'zstd', 'pglz', 'none' (default: 'none')\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); @@ -482,7 +482,7 @@ static void help_backup(void) printf(_("\n Compression options:\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib', 'pglz', 'none' (default: none)\n")); + printf(_(" available options: 'zlib', 'zstd', 'pglz', 'none' (default: none)\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); diff --git a/src/bin/pg_probackup/pg_probackupa.h b/src/bin/pg_probackup/pg_probackupa.h index d9def2cb8..c4d20f0e5 100644 --- a/src/bin/pg_probackup/pg_probackupa.h +++ b/src/bin/pg_probackup/pg_probackupa.h @@ -142,6 +142,7 @@ typedef enum CompressAlg #ifdef HAVE_LIBZ ZLIB_COMPRESS, #endif + ZSTD_COMPRESS, } CompressAlg; typedef enum ForkName From 23d47df07a918266532fdfa0c2d6ee15b3b71f09 Mon Sep 17 00:00:00 2001 From: dbxue Date: Thu, 4 Aug 2022 17:34:49 +0800 Subject: [PATCH 2/2] new feature: gs_probackup support zstd compress algrithom --- src/bin/pg_probackup/data.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_probackup/data.cpp b/src/bin/pg_probackup/data.cpp index 4230dce2f..753dadfe8 100644 --- a/src/bin/pg_probackup/data.cpp +++ b/src/bin/pg_probackup/data.cpp @@ -153,9 +153,9 @@ zstd_compress(const char *src, size_t src_size, char *dst, size_t dst_size, int write_len = ZSTD_compress(dst, dst_size, src, src_size, level); if (ZSTD_isError(write_len)) { - elog(ERROR, "zstd compress data failed, return: %u.", write_len); elog(LOG, "zstd compress error, src: [%s], src size: %u, dest size: %u, written size: %u.", src, src_size, dst_size, write_len); + elog(ERROR, "zstd compress data failed, return: %u.", write_len); return -1; } @@ -170,9 +170,9 @@ zstd_decompress(const char *src, size_t src_size, char *dst, size_t dst_size) write_len = ZSTD_decompress(dst, dst_size, src, src_size); if (ZSTD_isError(write_len)) { - elog(ERROR, "zstd decompress data failed, return: %u.", write_len); elog(LOG, "zstd decompress error, src size: %u, dest size: %u, written size: %u.", src_size, dst_size, write_len); + elog(ERROR, "zstd decompress data failed, return: %u.", write_len); return -1; }