database/openGauss-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

!1931 新特性：gs_probackup备份支持zstd压缩算法

Browse Source 
Merge pull request !1931 from Rongger/zstd
This commit is contained in:
opengauss-bot
2022-08-06 03:03:28 +00:00
committed by Gitee
parent ed0032e00d 93e27361c9
commit de7593dcc2
6 changed files with 56 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/bin/pg_probackup/CMakeLists.txt
View File

@ -13,7 +13,7 @@ execute_process(
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_probackup_SRC)
set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${LZ4_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression)
set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${LZ4_INCLUDE_PATH} ${ZSTD_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression)
set(probackup_DEF_OPTIONS ${MACRO_OPTIONS} -DFRONTEND -DHAVE_LIBZ)
set(probackup_COMPILE_OPTIONS ${PROTECT_OPTIONS} ${BIN_SECURE_OPTIONS} ${OS_OPTIONS} ${WARNING_OPTIONS} ${OPTIMIZE_OPTIONS} ${CHECK_OPTIONS})

4
src/bin/pg_probackup/Makefile
View File

@ -32,12 +32,12 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
EXTRA_CLEAN += logging.h
LDFLAGS += -L$(LZ4_LIB_PATH)
LDFLAGS += -L$(LZ4_LIB_PATH) -L$(ZSTD_LIB_PATH)
LIBS += -llz4
ifeq ($(enable_lite_mode), no)
LIBS += -lgssapi_krb5_gauss -lgssrpc_gauss -lkrb5_gauss -lkrb5support_gauss -lk5crypto_gauss -lcom_err_gauss
endif
PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_builddir)/$(subdir) -I$(LZ4_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH)
PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_builddir)/$(subdir) -I$(LZ4_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH)
# add page_compression so .h
LDFLAGS += -L../../lib/page_compression
PG_CPPFLAGS = -I../../lib/page_compression

4
src/bin/pg_probackup/catalog.cpp
View File

@ -2384,6 +2384,8 @@ parse_compress_alg(const char *arg)
return ZLIB_COMPRESS;
else if (pg_strncasecmp("lz4", arg, len) == 0)
return LZ4_COMPRESS;
else if (pg_strncasecmp("zstd", arg, len) == 0)
return ZSTD_COMPRESS;
else if (pg_strncasecmp("pglz", arg, len) == 0)
return PGLZ_COMPRESS;
else if (pg_strncasecmp("none", arg, len) == 0)
@ -2408,6 +2410,8 @@ deparse_compress_alg(int alg)
return "pglz";
case LZ4_COMPRESS:
return "lz4";
case ZSTD_COMPRESS:
return "zstd";
}
return NULL;

50
src/bin/pg_probackup/data.cpp
View File

@ -27,6 +27,8 @@
#include "thread.h"
#include "common/fe_memutils.h"
#include "lz4.h"
#include "zstd.h"
/* Union to ease operations on relation pages */
typedef struct DataPage
@ -143,9 +145,9 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size)
}
#endif
/* Implementation of lz4 compression method */
static int32
lz4_compress(const char *src, size_t src_size, char *dst, size_t dst_size)
static int32 lz4_compress(const char *src, size_t src_size, char *dst, size_t dst_size)
{
int write_len;
@ -161,8 +163,7 @@ lz4_compress(const char *src, size_t src_size, char *dst, size_t dst_size)
}
/* Implementation of lz4 decompression method */
static int32
lz4_decompress(const char *src, size_t src_size, char *dst, size_t dst_size)
static int32 lz4_decompress(const char *src, size_t src_size, char *dst, size_t dst_size)
{
int write_len;
@ -182,6 +183,43 @@ lz4_decompress(const char *src, size_t src_size, char *dst, size_t dst_size)
return write_len;
}
/* Implementation of zstd compression method */
static int32 zstd_compress(const char *src, size_t src_size, char *dst, size_t dst_size, int level)
{
size_t write_len;
write_len = ZSTD_compress(dst, dst_size, src, src_size, level);
if (ZSTD_isError(write_len)) {
elog(LOG, "zstd compress error, src: [%s], src size: %u, dest size: %u, written size: %u.",
src, src_size, dst_size, write_len);
elog(ERROR, "zstd compress data failed, return: %u.", write_len);
return -1;
}
return (int32)write_len;
}
/* Implementation of zstd decompression method */
static int32 zstd_decompress(const char *src, size_t src_size, char *dst, size_t dst_size)
{
size_t write_len;
write_len = ZSTD_decompress(dst, dst_size, src, src_size);
if (ZSTD_isError(write_len)) {
elog(LOG, "zstd decompress error, src size: %u, dest size: %u, written size: %u.",
src_size, dst_size, write_len);
elog(ERROR, "zstd decompress data failed, return: %u.", write_len);
return -1;
}
if (write_len != dst_size) {
elog(WARNING, "zstd decompress data corrupted, actually written: %u, expected: %u.", write_len, dst_size);
}
return (int32)write_len;
}
/*
* Compresses source into dest using algorithm. Returns the number of bytes
* written in the destination buffer, or -1 if compression fails.
@ -209,6 +247,8 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size,
return pglz_compress((const char*)src, src_size, (char*)dst, PGLZ_strategy_always);
case LZ4_COMPRESS:
return lz4_compress((const char*)src, src_size, (char*)dst, dst_size);
case ZSTD_COMPRESS:
return zstd_compress((const char*)src, src_size, (char*)dst, dst_size, level);
}
return -1;
@ -243,6 +283,8 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size,
return pglz_decompress((const char*)src, src_size, (char*)dst, dst_size, true);
case LZ4_COMPRESS:
return lz4_decompress((const char*)src, src_size, (char*)dst, dst_size);
case ZSTD_COMPRESS:
return zstd_decompress((const char*)src, src_size, (char*)dst, dst_size);
}
return -1;

4
src/bin/pg_probackup/help.cpp
View File

@ -307,7 +307,7 @@ static void help_set_config(void)
printf(_("\n Compression options:\n"));
printf(_(" --compress-algorithm=compress-algorithm\n"));
printf(_(" available options: 'lz4', 'zlib','pglz','none' (default: 'none')\n"));
printf(_(" available options: 'zlib', 'lz4', 'zstd', 'pglz', 'none' (default: 'none')\n"));
printf(_(" --compress-level=compress-level\n"));
printf(_(" level of compression [0-9] (default: 1)\n"));
@ -482,7 +482,7 @@ static void help_backup(void)
printf(_("\n Compression options:\n"));
printf(_(" --compress-algorithm=compress-algorithm\n"));
printf(_(" available options: 'lz4', 'zlib', 'pglz', 'none' (default: none)\n"));
printf(_(" available options: 'zlib', 'lz4', 'zstd', 'pglz', 'none' (default: none)\n"));
printf(_(" --compress-level=compress-level\n"));
printf(_(" level of compression [0-9] (default: 1)\n"));
printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n"));

1
src/bin/pg_probackup/pg_probackupa.h
View File

@ -143,6 +143,7 @@ typedef enum CompressAlg
ZLIB_COMPRESS,
#endif
LZ4_COMPRESS,
ZSTD_COMPRESS,
} CompressAlg;
typedef enum ForkName