diff --git a/src/bin/pg_probackup/CMakeLists.txt b/src/bin/pg_probackup/CMakeLists.txt index 70e3d6985..3bfae5045 100755 --- a/src/bin/pg_probackup/CMakeLists.txt +++ b/src/bin/pg_probackup/CMakeLists.txt @@ -13,7 +13,7 @@ execute_process( AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_probackup_SRC) -set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${LZ4_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression) +set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${LZ4_INCLUDE_PATH} ${ZSTD_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression) set(probackup_DEF_OPTIONS ${MACRO_OPTIONS} -DFRONTEND -DHAVE_LIBZ) set(probackup_COMPILE_OPTIONS ${PROTECT_OPTIONS} ${BIN_SECURE_OPTIONS} ${OS_OPTIONS} ${WARNING_OPTIONS} ${OPTIMIZE_OPTIONS} ${CHECK_OPTIONS}) diff --git a/src/bin/pg_probackup/Makefile b/src/bin/pg_probackup/Makefile index 9f3e8178a..1b17edea8 100644 --- a/src/bin/pg_probackup/Makefile +++ b/src/bin/pg_probackup/Makefile @@ -32,12 +32,12 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global EXTRA_CLEAN += logging.h -LDFLAGS += -L$(LZ4_LIB_PATH) +LDFLAGS += -L$(LZ4_LIB_PATH) -L$(ZSTD_LIB_PATH) LIBS += -llz4 ifeq ($(enable_lite_mode), no) LIBS += -lgssapi_krb5_gauss -lgssrpc_gauss -lkrb5_gauss -lkrb5support_gauss -lk5crypto_gauss -lcom_err_gauss endif -PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_builddir)/$(subdir) -I$(LZ4_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_builddir)/$(subdir) -I$(LZ4_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH) # add page_compression so .h LDFLAGS += -L../../lib/page_compression PG_CPPFLAGS = -I../../lib/page_compression diff --git a/src/bin/pg_probackup/catalog.cpp b/src/bin/pg_probackup/catalog.cpp index 7fb42b5ef..7e2c9b6c6 100644 --- a/src/bin/pg_probackup/catalog.cpp +++ b/src/bin/pg_probackup/catalog.cpp @@ -2384,6 +2384,8 @@ parse_compress_alg(const char *arg) return ZLIB_COMPRESS; else if (pg_strncasecmp("lz4", arg, len) == 0) return LZ4_COMPRESS; + else if (pg_strncasecmp("zstd", arg, len) == 0) + return ZSTD_COMPRESS; else if (pg_strncasecmp("pglz", arg, len) == 0) return PGLZ_COMPRESS; else if (pg_strncasecmp("none", arg, len) == 0) @@ -2408,6 +2410,8 @@ deparse_compress_alg(int alg) return "pglz"; case LZ4_COMPRESS: return "lz4"; + case ZSTD_COMPRESS: + return "zstd"; } return NULL; diff --git a/src/bin/pg_probackup/data.cpp b/src/bin/pg_probackup/data.cpp index e659779d9..36f3157f8 100644 --- a/src/bin/pg_probackup/data.cpp +++ b/src/bin/pg_probackup/data.cpp @@ -27,6 +27,8 @@ #include "thread.h" #include "common/fe_memutils.h" #include "lz4.h" +#include "zstd.h" + /* Union to ease operations on relation pages */ typedef struct DataPage @@ -143,9 +145,9 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) } #endif + /* Implementation of lz4 compression method */ -static int32 -lz4_compress(const char *src, size_t src_size, char *dst, size_t dst_size) +static int32 lz4_compress(const char *src, size_t src_size, char *dst, size_t dst_size) { int write_len; @@ -161,8 +163,7 @@ lz4_compress(const char *src, size_t src_size, char *dst, size_t dst_size) } /* Implementation of lz4 decompression method */ -static int32 -lz4_decompress(const char *src, size_t src_size, char *dst, size_t dst_size) +static int32 lz4_decompress(const char *src, size_t src_size, char *dst, size_t dst_size) { int write_len; @@ -182,6 +183,43 @@ lz4_decompress(const char *src, size_t src_size, char *dst, size_t dst_size) return write_len; } +/* Implementation of zstd compression method */ +static int32 zstd_compress(const char *src, size_t src_size, char *dst, size_t dst_size, int level) +{ + size_t write_len; + + write_len = ZSTD_compress(dst, dst_size, src, src_size, level); + if (ZSTD_isError(write_len)) { + elog(LOG, "zstd compress error, src: [%s], src size: %u, dest size: %u, written size: %u.", + src, src_size, dst_size, write_len); + elog(ERROR, "zstd compress data failed, return: %u.", write_len); + return -1; + } + + return (int32)write_len; +} + +/* Implementation of zstd decompression method */ +static int32 zstd_decompress(const char *src, size_t src_size, char *dst, size_t dst_size) +{ + size_t write_len; + + write_len = ZSTD_decompress(dst, dst_size, src, src_size); + if (ZSTD_isError(write_len)) { + elog(LOG, "zstd decompress error, src size: %u, dest size: %u, written size: %u.", + src_size, dst_size, write_len); + elog(ERROR, "zstd decompress data failed, return: %u.", write_len); + return -1; + } + + if (write_len != dst_size) { + elog(WARNING, "zstd decompress data corrupted, actually written: %u, expected: %u.", write_len, dst_size); + } + + return (int32)write_len; +} + + /* * Compresses source into dest using algorithm. Returns the number of bytes * written in the destination buffer, or -1 if compression fails. @@ -209,6 +247,8 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, return pglz_compress((const char*)src, src_size, (char*)dst, PGLZ_strategy_always); case LZ4_COMPRESS: return lz4_compress((const char*)src, src_size, (char*)dst, dst_size); + case ZSTD_COMPRESS: + return zstd_compress((const char*)src, src_size, (char*)dst, dst_size, level); } return -1; @@ -243,6 +283,8 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, return pglz_decompress((const char*)src, src_size, (char*)dst, dst_size, true); case LZ4_COMPRESS: return lz4_decompress((const char*)src, src_size, (char*)dst, dst_size); + case ZSTD_COMPRESS: + return zstd_decompress((const char*)src, src_size, (char*)dst, dst_size); } return -1; diff --git a/src/bin/pg_probackup/help.cpp b/src/bin/pg_probackup/help.cpp index b3f19ccb2..ac453633c 100644 --- a/src/bin/pg_probackup/help.cpp +++ b/src/bin/pg_probackup/help.cpp @@ -307,7 +307,7 @@ static void help_set_config(void) printf(_("\n Compression options:\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'lz4', 'zlib','pglz','none' (default: 'none')\n")); + printf(_(" available options: 'zlib', 'lz4', 'zstd', 'pglz', 'none' (default: 'none')\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); @@ -482,7 +482,7 @@ static void help_backup(void) printf(_("\n Compression options:\n")); printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'lz4', 'zlib', 'pglz', 'none' (default: none)\n")); + printf(_(" available options: 'zlib', 'lz4', 'zstd', 'pglz', 'none' (default: none)\n")); printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); diff --git a/src/bin/pg_probackup/pg_probackupa.h b/src/bin/pg_probackup/pg_probackupa.h index 32440bd5a..79f8d7787 100644 --- a/src/bin/pg_probackup/pg_probackupa.h +++ b/src/bin/pg_probackup/pg_probackupa.h @@ -143,6 +143,7 @@ typedef enum CompressAlg ZLIB_COMPRESS, #endif LZ4_COMPRESS, + ZSTD_COMPRESS, } CompressAlg; typedef enum ForkName