hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

replace ts related to ddl with scn.

Browse Source
This commit is contained in:
obdev 2022-11-28 02:21:13 +00:00 committed by ob-robot
parent bbec6aff49
commit 8a4d14122f
539 changed files with 17685 additions and 173434 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/pull_request_template.md vendored
View File

@ -13,7 +13,6 @@ If possible, please consider writing useful notes for better and faster reviews
4. If there is a discussion in the mailing list, please add the link.
-->
### Why are the changes needed?
<!--
@ -22,18 +21,15 @@ Please clarify why the changes are needed. For instance,
2. If you fix a bug, you can clarify why it is a bug.
-->
### Will break the compatibility? How if so?
<!--
Please label `alert/break' if so, such as
Please make sure it do not break the system compatibility, such as
1. nGQL grammar changes;
2. RPC protocol can't be compatible with previous, refer to https://diwakergupta.github.io/thrift-missing-guide/#_versioning_compatibility;
2. RPC protocol can't be compatible with previous;
3. Storage format; etc.
-->
### Does this PR introduce any user-facing change?
<!--
@ -41,7 +37,6 @@ If yes, please clarify the previous behavior and the change this PR proposes - p
If no, write 'No'.
-->
### How was this patch tested?
<!--

13
.gitignore vendored
View File

@ -18,6 +18,7 @@ unittest/**/unittest_*
*.la
*.lo
*.log
!tools/timezone_V1.log
*.log.wf
*.o
*.orig
@ -149,6 +150,16 @@ src/sql/parser/sql_parser_oracle_utf8_mode_tab.c
src/sql/parser/sql_parser_oracle_utf8_mode_tab.h
src/sql/parser/_gen_parser.output
src/sql/parser/_gen_parser.error
src/pl/parser/pl_parser_mysql_mode_lex.c
src/pl/parser/pl_parser_mysql_mode_lex.h
src/pl/parser/pl_parser_mysql_mode_tab.c
src/pl/parser/pl_parser_mysql_mode_tab.h
src/pl/parser/pl_parser_oracle_mode_lex.c
src/pl/parser/pl_parser_oracle_mode_lex.h
src/pl/parser/pl_parser_oracle_mode_tab.c
src/pl/parser/pl_parser_oracle_mode_tab.h
src/pl/parser/_gen_parser.error
src/pl/parser/_gen_pl_parser.output
tags
test-driver
tools/ObRestore/mvn_repository
@ -261,8 +272,6 @@ unittest/obproxy/etc/
unittest/obproxy/failed.result
unittest/obproxy/failed.sql
unittest/rpc/rpc_bench
unittest/sql/engine/*/test_*
unittest/sql/engine/expr/*_test
unittest/sql/engine/expr/arithmatic.out
unittest/sql/executor/rpc_remote_scheduler
unittest/sql/executor/stress_interm_result_manager

0
.gitmodules vendored
View File

26
.secignore
View File

@ -7,7 +7,7 @@
# Under the segmentation lines there are the folders which you need to ignore #
##########################################################
**
http://license.coscl.org.cn/MulanPubL-2.0
http://license.coscl.org.cn/MulanPubL-2.0*
http://license.coscl.org.cn/MulanPubL-1.0
https://open.oceanbase.com*
https://www.oceanbase.com*
@ -39,6 +39,7 @@
http://llvm.org/bugs*
https://bugs.llvm.org/*
http://www.apache.org/licenses/*
https://issues.apache.org/*
https://github.com/vectorclass*
https://github.com/jcn/cowsay-py/blob/master/cowsay.py
http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
@ -124,14 +125,13 @@
madler@alumni.caltech.edu
http://en.wikipedia.org/wiki/X86-64#Linux
https://github.com/jsteemann/atoi
https://tools.ietf.org/html/rfc1951
hewei@mail.ied.ac.cn
http://trade.taobao.com/trade/*
http://buy.tmall.com/order/*
http://unit.buyer.trade.taobao.com/trade/*
http://abc.com/*
https://oceanbase.slack.com/
https://ask.oceanbase.com/
https://oceanbase.slack.com/*
https://ask.oceanbase.com/*
https://cdn.nlark.com/yuque/*
https://h5.dingtalk.com/circle/*
https://gw.alipayobjects.com/*
@ -142,8 +142,10 @@
rbrown64@csc.com.au
http://bazaar.launchpad.net/*
http://stackoverflow.com/*
https://stackoverflow.com/*
https://cloud.tencent.com/*
http://k3.alibaba-inc.com/*
https://k3.alibaba-inc.com/*
libecb@schmorp.de
http://software.schmorp.de/*
http://www-01.ibm.com/*
@ -153,9 +155,11 @@
http://www.ussg.indiana.edu/*
https://www.qqxiuzi.cn/daohang.htm
https://blog.csdn.net/*
http://blog.csdn.net/*
https://es5.github.io/*
http://rapidjson.org/*
https://tools.ietf.org/*
http://tools.ietf.org/*
https://github.com/madler/zlib/*
http://www.zlib.net
http://www.oberhumer.com/*
@ -166,12 +170,14 @@
http://gitlab.alibaba-inc.com/*
https://gerry.lamost.org/*
https://www.techonthenet.com/oracle/*
http://www.techonthenet.com/oracle/*
http://review.alibaba-inc.com/*
https://gw.alicdn.com/*
http://docs.oracle.com/*
https://oracle-base.com/*
https://www.kernel.org/doc/*
https://www.atatech.org/*
http://www.atatech.org/*
http://oceanbase.alibaba-inc.com/*
https://community.oracle.com/tech/*
https://docs.docker.com/*
@ -179,6 +185,12 @@
http://1.1.1.1:8080/*
https://mirrors.aliyun.com/*
https://bixense.com/clicolors/
martin.gieseking@uos.de
jloup@gzip.org
http://www.lz4.org
https://github.com/lz4/lz4
https://www.contributor-covenant.org/*
https://github.com/oceanbase/*
**/Doxyfile
www.doxygen.org)
https://www.gnu.org/software/libiconv/
@ -261,8 +273,6 @@
*@email.com
emily@163.com
12@126.com
765432455@qq.com
641745677@qq.com
555@qq.com
foo@aol.com
bar@aol.com
@ -338,8 +348,6 @@
*@email.com
emily@163.com
12@126.com
765432455@qq.com
641745677@qq.com
555@qq.com
foo@aol.com
bar@aol.com
@ -415,8 +423,6 @@
*@email.com
emily@163.com
12@126.com
765432455@qq.com
641745677@qq.com
555@qq.com
foo@aol.com
bar@aol.com

8
CMakeLists.txt
View File

@ -125,3 +125,11 @@ elseif(OB_INCLUDE_UNITTEST)
add_subdirectory(unittest EXCLUDE_FROM_ALL)
endif()
if (OB_BUILD_TOOLS)
add_subdirectory(tools)
elseif (OB_INCLUDE_TOOLS)
add_subdirectory(tools EXCLUDE_FROM_ALL)
endif()
include(cmake/RPM.cmake)

43
build.sh
View File

@ -17,6 +17,7 @@ NEED_MAKE=false
NEED_INIT=false
LLD_OPTION=ON
ASAN_OPTION=ON
STATIC_LINK_LGPL_DEPS_OPTION=ON
echo "$0 ${ALL_ARGS[@]}"
@ -30,28 +31,25 @@ function echo_err() {
function usage
{
echo -e "Usage:
\t./build.sh -h
\t./build.sh init
\t./build.sh clean
\t./build.sh --ce
\t./build.sh [BuildType] [--init] [--make [MakeOptions]]
\t./build.sh [BuildType] [--init] [--ob-make [MakeOptions]]
echo -e "Usage:"
echo -e "\t./build.sh -h"
echo -e "\t./build.sh init"
echo -e "\t./build.sh clean"
echo -e "\t./build.sh [BuildType] [--init] [--make [MakeOptions]]"
OPTIONS:
BuildType => debug(default), release, errsim, dissearray, rpm
MakeOptions => Options to make command, default: -j N
echo -e "\nOPTIONS:"
echo -e "\tBuildType => debug(default), release, errsim, dissearray, rpm"
echo -e "\tMakeOptions => Options to make command, default: -j N"
Examples:
\t# Build by debug mode and make with -j24.
\t./build.sh debug --make -j24
echo -e "\nExamples:"
echo -e "\t# Build by debug mode and make with -j24."
echo -e "\t./build.sh debug --make -j24"
\t# Init and build with release mode but not compile.
\t./build.sh release --init
echo -e "\n\t# Init and build with release mode but not compile."
echo -e "\t./build.sh release --init"
\t# Build with rpm mode and make with default arguments.
\t./build.sh rpm --make
"
echo -e "\n\t# Build with rpm mode and make with default arguments."
echo -e "\t./build.sh rpm --make"
}
# parse arguments
@ -64,10 +62,6 @@ function parse_args
elif [[ "$i" == "--make" ]]
then
NEED_MAKE=make
elif [[ "$i" == "--ob-make" ]]
then
NEED_MAKE=ob-make
MAKE_ARGS=()
elif [[ $NEED_MAKE == false ]]
then
BUILD_ARGS+=("$i")
@ -116,7 +110,7 @@ function do_init
exit $?
fi
time2_ms=$(echo $[$(date +%s%N)/1000000])
# 计算时间差值
cost_time_ms=$(($time2_ms - $time1_ms))
cost_time_s=`expr $cost_time_ms / 1000`
let min=cost_time_s/60
@ -198,7 +192,8 @@ function build
do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_DEBUG_LOG=ON -DENABLE_OBJ_LEAK_CHECK=ON -DOB_USE_LLD=$LLD_OPTION
;;
xrpm)
do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOB_USE_LLD=$LLD_OPTION -DENABLE_FATAL_ERROR_HANG=OFF
STATIC_LINK_LGPL_DEPS_OPTION=OFF
do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOB_USE_LLD=$LLD_OPTION -DENABLE_FATAL_ERROR_HANG=OFF -DOB_STATIC_LINK_LGPL_DEPS=$STATIC_LINK_LGPL_DEPS_OPTION
;;
xenable_smart_var_check)
do_build "$@" -DCMAKE_BUILD_TYPE=Debug -DOB_USE_LLD=$LLD_OPTION -DENABLE_SMART_VAR_CHECK=ON -DOB_ENABLE_AVX2=ON

44
cmake/Env.cmake
View File

@ -1,17 +1,15 @@
ob_define(DEBUG_PREFIX "-fdebug-prefix-map=${CMAKE_SOURCE_DIR}=.")
ob_define(AUTO_FDO_OPT "")
ob_define(OB_LD_BIN ld)
ob_define(CACHE_UUID "e3cf8ff1-e7e7-4aa3-ae43-3fc35e2bb836")
ob_define(ASAN_IGNORE_LIST "${CMAKE_SOURCE_DIR}/asan_ignore_list.txt")
ob_define(DEP_3RD_DIR "${CMAKE_SOURCE_DIR}/deps/3rd")
ob_define(DEVTOOLS_DIR "${CMAKE_SOURCE_DIR}/deps/3rd/usr/local/oceanbase/devtools")
ob_define(DEP_DIR "${CMAKE_SOURCE_DIR}/deps/3rd/usr/local/oceanbase/deps/devel")
# TODO: will remove all DEP_3RD_DIR rpm
ob_define(DEP_3RD_DIR "${CMAKE_SOURCE_DIR}/deps/3rd")
ob_define(OB_BUILD_CDC OFF)
ob_define(OB_USE_CLANG ON)
ob_define(OB_ERRSIM OFF)
ob_define(AONE_BUILD_NUMBER 1)
ob_define(BUILD_NUMBER 1)
ob_define(OB_GPERF_MODE OFF)
ob_define(OB_TRANS_ERRSIM OFF)
ob_define(OB_DIS_SEARRAY OFF)
@ -22,6 +20,7 @@ ob_define(ENABLE_FATAL_ERROR_HANG ON)
ob_define(ENABLE_SMART_VAR_CHECK OFF)
ob_define(ENABLE_COMPILE_DLL_MODE OFF)
ob_define(OB_CMAKE_RULES_CHECK ON)
ob_define(OB_STATIC_LINK_LGPL_DEPS ON)
# 'ENABLE_PERF_MODE' use for offline system insight performance test
# PERF_MODE macro controls many special code path in system
@ -34,7 +33,6 @@ ob_define(OB_MAX_UNITY_BATCH_SIZE 30)
ob_define(OB_ENABLE_UNITY ON)
if(WITH_COVERAGE)
# doc about coverage: https://yuque.antfin.com/docs/share/c6a2742b-bc66-4054-a781-681264a3b6fe?#
# -ftest-coverage to generate .gcno file
# -fprofile-arcs to generate .gcda file
# -DDBUILD_COVERAGE marco use to mark 'coverage build type' and to handle some speical case
@ -43,6 +41,10 @@ if(WITH_COVERAGE)
endif()
# should not use initial-exec for tls-model if building OBCDC.
if(NOT OB_BUILD_CDC)
add_definitions(-DENABLE_INITIAL_EXEC_TLS_MODEL)
endif()
if (OB_USE_CLANG)
find_program(OB_CC clang
@ -67,26 +69,13 @@ if (OB_USE_CLANG)
set(REORDER_LINK_OPT "-Wl,--no-rosegment,--build-id=sha1")
set(OB_LD_BIN "${DEVTOOLS_DIR}/bin/ld.lld")
endif()
set(CMAKE_CXX_FLAGS "--gcc-toolchain=${GCC9} ${DEBUG_PREFIX} ${AUTO_FDO_OPT} -fcolor-diagnostics ${REORDER_COMP_OPT} -fmax-type-align=8 ${CMAKE_ASAN_FLAG} ${CMAKE_COVERAGE_FLAG} -DCACHE_UUID=${CACHE_UUID} -std=gnu++11")
set(CMAKE_C_FLAGS "--gcc-toolchain=${GCC9} ${DEBUG_PREFIX} ${AUTO_FDO_OPT} -fcolor-diagnostics ${REORDER_COMP_OPT} -fmax-type-align=8 ${CMAKE_ASAN_FLAG} ${CMAKE_COVERAGE_FLAG} -DCACHE_UUID=${CACHE_UUID}")
set(CMAKE_CXX_LINK_FLAGS "${LD_OPT} --gcc-toolchain=${GCC9} ${DEBUG_PREFIX} ${AUTO_FDO_OPT}")
set(CMAKE_CXX_FLAGS "--gcc-toolchain=${GCC9} ${DEBUG_PREFIX} -fcolor-diagnostics ${REORDER_COMP_OPT} -fmax-type-align=8 ${CMAKE_ASAN_FLAG} ${CMAKE_COVERAGE_FLAG} -std=gnu++11")
set(CMAKE_C_FLAGS "--gcc-toolchain=${GCC9} ${DEBUG_PREFIX} -fcolor-diagnostics ${REORDER_COMP_OPT} -fmax-type-align=8 ${CMAKE_ASAN_FLAG} ${CMAKE_COVERAGE_FLAG}")
set(CMAKE_CXX_LINK_FLAGS "${LD_OPT} --gcc-toolchain=${GCC9} ${DEBUG_PREFIX}")
set(CMAKE_SHARED_LINKER_FLAGS "${LD_OPT} -Wl,-z,noexecstack ${REORDER_LINK_OPT}")
set(CMAKE_EXE_LINKER_FLAGS "${LD_OPT} -Wl,-z,noexecstack ${REORDER_LINK_OPT}")
else() # not clang, use gcc
find_program(OB_CC gcc
PATHS ${DEVTOOLS_DIR}/bin
NO_DEFAULT_PATH)
find_program(OB_CXX g++
PATHS ${DEVTOOLS_DIR}/bin
NO_DEFAULT_PATH)
if (OB_USE_LLD)
set(LD_OPT "-B${CMAKE_SOURCE_DIR}/rpm/.compile")
set(REORDER_COMP_OPT "-ffunction-sections")
endif()
set(CMAKE_CXX_FLAGS "${LD_OPT} -fdiagnostics-color ${REORDER_COMP_OPT}")
set(CMAKE_C_FLAGS "${LD_OPT} -fdiagnostics-color ${REORDER_COMP_OPT}")
set(CMAKE_SHARED_LINKER_FLAGS "-z noexecstack ${REORDER_LINK_OPT}")
set(CMAKE_EXE_LINKER_FLAGS "-z noexecstack ${REORDER_LINK_OPT}")
message("gcc9 not support currently, please set OB_USE_CLANG ON and we will finish it as soon as possible")
endif()
if (OB_CC AND OB_CXX)
@ -96,15 +85,6 @@ else()
message(FATAL_ERROR "can't find suitable compiler")
endif()
find_program(OB_COMPILE_EXECUTABLE ob-compile)
if (NOT OB_COMPILE_EXECUTABLE)
message(WARNING "ob-compile not found, compile locally.")
else()
set(CMAKE_C_COMPILER_LAUNCHER ${OB_COMPILE_EXECUTABLE})
set(CMAKE_CXX_COMPILER_LAUNCHER ${OB_COMPILE_EXECUTABLE})
set(CMAKE_C_LINKER_LAUNCHER ${OB_COMPILE_EXECUTABLE})
set(CMAKE_CXX_LINKER_LAUNCHER ${OB_COMPILE_EXECUTABLE})
endif()
option(OB_ENABLE_AVX2 "enable AVX2 and related instruction set support for x86_64" OFF)

1
cmake/Err.cmake
View File

@ -5,5 +5,4 @@ macro(ob_def_error ERR_ID ERR_MSG)
endmacro()
ob_def_error(E1001 "[E1001] Header files are not allowed in CMakeLists.txt")
ob_def_error(HELP_LINK "https://yuque.antfin.com/docs/share/f2745c4d-afc9-4c80-8348-88741f902bab?#uVf51")

12
cmake/RPM.cmake
View File

@ -24,6 +24,12 @@ set(CPACK_PACKAGE_VERSION_MINOR "${OceanBase_CE_VERSION_MINOR}")
set(CPACK_PACKAGE_VERSION_PATCH "${OceanBase_CE_VERSION_PATCH}")
set(CPACK_RPM_PACKAGE_URL "${OceanBase_CE_HOMEPAGE_URL}")
set(CPACK_RPM_PACKAGE_RELEASE_DIST ON)
## set relocation path install prefix for each component
set(CPACK_RPM_DEVEL_PACKAGE_PREFIX /usr)
set(CPACK_RPM_UTILS_PACKAGE_PREFIX /usr)
list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/home")
list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/home/admin")
list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/home/admin/oceanbase")
set(CPACK_RPM_PACKAGE_GROUP "Applications/Databases")
set(CPACK_RPM_PACKAGE_DESCRIPTION "OceanBase is a distributed relational database")
set(CPACK_RPM_PACKAGE_LICENSE "Mulan PubL v2.")
@ -33,8 +39,12 @@ set(CPACK_RPM_SPEC_MORE_DEFINE
"%global _missing_build_ids_terminate_build 0
%global _find_debuginfo_opts -g
%define __strip ${CMAKE_SOURCE_DIR}/deps/3rd/usr/local/oceanbase/devtools/bin/llvm-strip
%undefine __brp_mangle_shebangs
%define __debug_install_post %{_rpmconfigdir}/find-debuginfo.sh %{?_find_debuginfo_opts} %{_builddir}/%{?buildsubdir};%{nil}
%debug_package")
%if \\\"%name\\\" != \\\"oceanbase-ce-sql-parser\\\" && \\\"%name\\\" != \\\"oceanbase-sql-parser\\\"
%debug_package
%endif
")
## TIPS
#

2
deps/init/dep_create.sh vendored
View File

@ -122,8 +122,6 @@ WORKSACPE_DEPS_DIR="$(cd $(dirname $0); cd ..; pwd)"
WORKSPACE_DEPS_3RD=${WORKSACPE_DEPS_DIR}/3rd
WORKSAPCE_DEPS_3RD_DONE=${WORKSPACE_DEPS_3RD}/DONE
WORKSAPCE_DEPS_3RD_MD5=${WORKSPACE_DEPS_3RD}/${MD5}
WORKSAPCE_DEPS_3RD_BUSINESS=${WORKSPACE_DEPS_3RD}/BUSINESS
WORKSAPCE_DEPS_3RD_OPENSOURCE=${WORKSPACE_DEPS_3RD}/OPENSOURCE
# 开始判断本地目录依赖目录是否存在
if [ -f ${WORKSAPCE_DEPS_3RD_MD5} ]; then

2
deps/init/oceanbase.el8.aarch64.deps vendored
View File

@ -1,7 +1,7 @@
[target-default]
os=8
arch=aarch64
repo=http://yum-test.obvos.alibaba-inc.com/oceanbase/development-kit/el/8/aarch64/
repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/8/aarch64/
[target-community]
os=8

2
deps/init/oceanbase.el8.x86_64.deps vendored
View File

@ -1,7 +1,7 @@
[target-default]
os=8
arch=x86_64
repo=http://yum-test.obvos.alibaba-inc.com/oceanbase/development-kit/el/8/x86_64/
repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/8/x86_64/
[target-community]
os=8

6
deps/oblib/src/CMakeLists.txt vendored
View File

@ -109,6 +109,10 @@ endif()
target_compile_features(oblib_base_base INTERFACE cxx_std_11)
set(LGPL_DEPS "-L${DEP_DIR}/lib/mariadb -lmariadb")
if (OB_STATIC_LINK_LGPL_DEPS)
set(LGPL_DEPS "-L${DEP_DIR}/lib/mariadb -l:libmariadbclient.a")
endif()
target_link_libraries(oblib_base_base_base
INTERFACE
oss
@ -116,7 +120,7 @@ target_link_libraries(oblib_base_base_base
${DEP_DIR}/lib/libisal.a
${DEP_DIR}/lib/libssl.a
${DEP_DIR}/lib/libcrypto.a
${DEP_DIR}/lib/mariadb/libmariadbclient.a
${LGPL_DEPS}
${DEP_DIR}/lib/libunwind.a
${DEP_DIR}/lib/libz.a
-L${DEP_DIR}/var/usr/lib64

4
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp vendored
View File

@ -216,9 +216,7 @@ void ObTenantCtxAllocator::print_usage() const
return mgr->get_ctx_hold(ctx_id_, ctx_hold_bytes);
});
}
if (ctx_hold_bytes * 0.8 >= sum_item.hold_) {
_LOG_WARN("there has too much memory fragments");
}
if (ctx_hold_bytes > 0 || sum_item.used_ > 0) {
_LOG_INFO("\n[MEMORY] tenant_id=%5ld ctx_id=%25s hold=% '15ld used=% '15ld limit=% '15ld"
"\n[MEMORY] idle_size=% '10ld free_size=% '10ld"

8
deps/oblib/src/lib/charset/ob_charset.cpp vendored
View File

@ -2627,6 +2627,7 @@ int ObCharset::charset_convert(const ObCollationType from_type,
char *to_str,
uint32_t to_len,
uint32_t &result_len,
bool trim_incomplete_tail,
bool report_error /*true*/,
const ob_wc_t replaced_char /*'?'*/) {
int ret = OB_SUCCESS;
@ -2648,9 +2649,10 @@ int ObCharset::charset_convert(const ObCollationType from_type,
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected collation type", K(ret), K(from_type), K(to_type));
} else {
uint errors;
result_len = ob_convert(to_str, to_len, to_cs, from_str, from_len, from_cs, replaced_char, &errors);
if (errors != 0 && report_error) {
uint errors = 0;
result_len = ob_convert(to_str, to_len, to_cs, from_str, from_len, from_cs,
trim_incomplete_tail, replaced_char, &errors);
if (OB_UNLIKELY(errors != 0 && report_error)) {
ret = OB_ERR_INCORRECT_STRING_VALUE;
LOG_WARN("ob_convert failed", K(ret), K(errors),
K(from_type), K(to_type),

1
deps/oblib/src/lib/charset/ob_charset.h vendored
View File

@ -438,6 +438,7 @@ public:
char *to_str,
uint32_t to_len,
uint32_t &result_len,
bool trim_incomplete_tail = true,
bool report_error = true,
const ob_wc_t replaced_char = '?');
enum CONVERT_FLAG : int64_t {

4
deps/oblib/src/lib/charset/ob_ctype.h vendored
View File

@ -561,7 +561,9 @@ void ob_hash_sort_mb_bin(const ObCharsetInfo *cs __attribute__((unused)),
uint32 ob_convert(char *to, uint32 to_length, const ObCharsetInfo *to_cs,
const char *from, uint32 from_length,
const ObCharsetInfo *from_cs, const ob_wc_t replaced_char, uint *errors);
const ObCharsetInfo *from_cs,
bool trim_incomplete_tail,
const ob_wc_t replaced_char, uint *errors);
size_t ob_strnxfrm_unicode_full_bin(const ObCharsetInfo *cs,
uchar *dst, size_t dstlen, uint nweights,

11
deps/oblib/src/lib/charset/ob_ctype_bin_os.cc vendored
View File

@ -254,7 +254,7 @@ int ob_wildcmp_bin_impl(const ObCharsetInfo *cs,
INC_PTR(cs,wild_str,wild_end);
cmp=likeconv(cs,cmp);
do {
while (true) {
while (str != str_end && (unsigned char) likeconv(cs,*str) != cmp) {
str++;
}
@ -265,10 +265,15 @@ int ob_wildcmp_bin_impl(const ObCharsetInfo *cs,
int tmp=ob_wildcmp_bin_impl(cs,str,str_end,
wild_str,wild_end,escape_char,
w_one, w_many, recurse_level + 1);
if (tmp <= 0)
if (tmp <= 0) {
return(tmp);
} else if (str == str_end) {
return -1;
} else if (wild_str != wild_end && wild_str[0] == w_many) {
return -1;
}
}
} while (str != str_end && wild_str[0] != w_many);
}
return(-1);
}
}

9
deps/oblib/src/lib/charset/ob_ctype_mb_os.cc vendored
View File

@ -211,7 +211,7 @@ int ob_wildcmp_mb_impl(const ObCharsetInfo *cs,
mb_len= ob_ismbchar(cs, wild_str, wild_end);
INC_PTR(cs,wild_str,wild_end);
cmp=likeconv(cs,cmp);
do {
while (true) {
while (TRUE) {
if (str >= str_end) {
return -1;
@ -235,7 +235,12 @@ int ob_wildcmp_mb_impl(const ObCharsetInfo *cs,
if (tmp <= 0)
return (tmp);
}
} while (str != str_end && wild_str[0] != w_many);
if (str == str_end) {
return -1;
} else if (wild_str != wild_end && wild_str[0] == w_many) {
return -1;
}
}
return(-1);
}
}

19
deps/oblib/src/lib/charset/ob_ctype_os.cc vendored
View File

@ -23,6 +23,7 @@ ob_convert_internal(char *to, uint32 to_length,
const ObCharsetInfo *to_cs,
const char *from, uint32 from_length,
const ObCharsetInfo *from_cs,
bool trim_incomplete_tail,
const ob_wc_t replaced_char, uint *errors)
{
unsigned int error_num= 0;
@ -46,7 +47,14 @@ ob_convert_internal(char *to, uint32 to_length,
wc= replaced_char;
error_num++;
} else {
break;
// Not enough characters
if (!trim_incomplete_tail && cnvres != OB_CS_TOOSMALL) {
error_num++;
from++;
wc= replaced_char;
} else {
break;
}
}
pbool go = TRUE;
@ -71,12 +79,15 @@ ob_convert_internal(char *to, uint32 to_length,
uint32
ob_convert(char *to, uint32 to_length, const ObCharsetInfo *to_cs,
const char *from, uint32 from_length,
const ObCharsetInfo *from_cs, const ob_wc_t replaced_char , uint *errors)
const ObCharsetInfo *from_cs,
bool trim_incomplete_tail,
const ob_wc_t replaced_char , uint *errors)
{
uint32 length, length2;
if ((to_cs->state | from_cs->state) & OB_CS_NONASCII) {
return ob_convert_internal(to, to_length, to_cs, from, from_length, from_cs, replaced_char, errors);
return ob_convert_internal(to, to_length, to_cs, from, from_length, from_cs,
trim_incomplete_tail, replaced_char, errors);
} else {
length= length2= OB_MIN(to_length, from_length);
}
@ -101,7 +112,7 @@ ob_convert(char *to, uint32 to_length, const ObCharsetInfo *to_cs,
from_length-= copied_length;
return copied_length + ob_convert_internal(to, to_length, to_cs,
from, from_length, from_cs,
replaced_char, errors);
trim_incomplete_tail, replaced_char, errors);
}
*to++= *from++;
length--;

4
deps/oblib/src/lib/lock/cond.h vendored
View File

@ -18,8 +18,8 @@
namespace obutil
{
template<class T> class ObMonitor;
class Mutex;
class ObUtilMutex;
typedef ObUtilMutex Mutex;
class Cond
{
public:

66
deps/oblib/src/lib/lock/mutex.cpp vendored
View File

@ -14,7 +14,7 @@
#include "lib/oblog/ob_log.h"
namespace obutil
{
Mutex::Mutex()
ObUtilMutex::ObUtilMutex()
{
const int rt = pthread_mutex_init(&_mutex, NULL);
#ifdef _NO_EXCEPTION
@ -29,7 +29,7 @@ Mutex::Mutex()
#endif
}
Mutex::~Mutex()
ObUtilMutex::~ObUtilMutex()
{
const int rt = pthread_mutex_destroy(&_mutex);
assert(rt == 0);
@ -38,30 +38,7 @@ Mutex::~Mutex()
}
}
void Mutex::lock() const
{
const int rt = pthread_mutex_lock(&_mutex);
#ifdef _NO_EXCEPTION
assert( rt == 0 );
if ( rt != 0 ) {
if ( rt == EDEADLK ) {
_OB_LOG(ERROR,"%s","ThreadLockedException ");
} else {
_OB_LOG(ERROR,"%s","ThreadSyscallException");
}
}
#else
if( rt != 0 ) {
if(rt == EDEADLK) {
throw ThreadLockedException(__FILE__, __LINE__);
} else {
throw ThreadSyscallException(__FILE__, __LINE__, rt);
}
}
#endif
}
bool Mutex::trylock() const
bool ObUtilMutex::trylock() const
{
const int rt = pthread_mutex_trylock(&_mutex);
#ifdef _NO_EXCEPTION
@ -85,7 +62,34 @@ bool Mutex::trylock() const
return (rt == 0);
}
void Mutex::unlock() const
void ObUtilMutex::lock() const
{
const int rt = pthread_mutex_lock(&_mutex);
#ifdef _NO_EXCEPTION
assert( rt == 0 );
if ( rt != 0 ) {
if ( rt == EDEADLK ) {
_OB_LOG(ERROR,"%s","ThreadLockedException ");
} else {
_OB_LOG(ERROR,"%s","ThreadSyscallException");
}
}
#else
if( rt != 0 ) {
if(rt == EDEADLK) {
throw ThreadLockedException(__FILE__, __LINE__);
} else {
throw ThreadSyscallException(__FILE__, __LINE__, rt);
}
}
#endif
}
void ObUtilMutex::lock(LockState&) const
{
}
void ObUtilMutex::unlock() const
{
const int rt = pthread_mutex_unlock(&_mutex);
#ifdef _NO_EXCEPTION
@ -100,16 +104,12 @@ void Mutex::unlock() const
#endif
}
void Mutex::unlock(LockState& state) const
void ObUtilMutex::unlock(LockState& state) const
{
state.mutex = &_mutex;
}
void Mutex::lock(LockState&) const
{
}
bool Mutex::will_unlock() const
bool ObUtilMutex::will_unlock() const
{
return true;
}

15
deps/oblib/src/lib/lock/mutex.h vendored
View File

@ -18,15 +18,15 @@
namespace obutil
{
class Mutex
class ObUtilMutex
{
public:
typedef ObLockT<Mutex> Lock;
typedef ObTryLockT<Mutex> TryLock;
typedef ObLockT<ObUtilMutex> Lock;
typedef ObTryLockT<ObUtilMutex> TryLock;
Mutex();
~Mutex();
ObUtilMutex();
~ObUtilMutex();
void lock() const;
bool trylock() const;
@ -35,8 +35,8 @@ public:
private:
Mutex(const Mutex&);
Mutex& operator=(const Mutex&);
ObUtilMutex(const ObUtilMutex&);
ObUtilMutex& operator=(const ObUtilMutex&);
struct LockState
{
@ -49,5 +49,6 @@ private:
friend class Cond;
};
typedef ObUtilMutex Mutex;
}//end namespace
#endif

7
deps/oblib/src/lib/lock/ob_spin_rwlock.h vendored
View File

@ -19,6 +19,7 @@
#include "lib/ob_define.h"
#include "lib/stat/ob_latch_define.h"
#include "lib/lock/ob_latch.h"
#include "lib/alloc/alloc_struct.h"
namespace oceanbase
{
@ -34,6 +35,12 @@ public:
~SpinRWLock()
{
}
int init(const lib::ObMemAttr &mem_attr)
{
UNUSED(mem_attr);
return OB_SUCCESS;
}
void destroy() {}
public:
void set_latch_id(const uint32_t latch_id) { latch_id_ = latch_id; }
inline bool try_rdlock() { return OB_SUCCESS == latch_.try_rdlock(latch_id_); }

3
deps/oblib/src/lib/ob_name_id_def.h vendored
View File

@ -197,8 +197,7 @@ DEF_NAME(id, "id")
DEF_NAME(part_end_trans, "partition end transaction")
DEF_NAME(left_time, "left time")
DEF_NAME(is_rollback, "is rollback")
DEF_NAME(iter_cb, "iter cb")
DEF_NAME(on_succ_cb, "on_sync_log_success")
DEF_NAME(log_sync_succ_cb, "on_log_sync_success")
DEF_NAME(leader_takeover, "leader takeover")
DEF_NAME(leader_active, "leader active")
DEF_NAME(leader_revoke, "leader revoke")

3
deps/oblib/src/lib/oblog/ob_log.h vendored
View File

@ -910,8 +910,6 @@ void ObLogger::log_it(const char *mod_name,
&& OB_LIKELY(is_enable_logging())
&& OB_NOT_NULL(mod_name) && OB_NOT_NULL(file) && OB_NOT_NULL(function)
&& OB_NOT_NULL(function)) {
bool old_val = set_disable_logging(true);
DEFER(set_disable_logging(old_val));
if (is_trace_mode()) {
TraceBuffer *tb = nullptr;
if (OB_NOT_NULL(tb = get_trace_buffer())) {
@ -1113,6 +1111,7 @@ inline void ObLogger::do_log_message(const bool is_async,
Function &log_data_func)
{
int ret = OB_SUCCESS;
if(!is_enable_logging()) return;
bool old_val = set_disable_logging(true);
DEFER(set_disable_logging(old_val));
bool allow = true;

7
deps/oblib/src/lib/restore/CMakeLists.txt vendored
View File

@ -8,10 +8,6 @@ oblib_add_library(restore OBJECT
ob_storage_oss_base.h
ob_storage_path.cpp
ob_storage_path.h
cos/ob_storage_cos_obj.cpp
cos/ob_storage_cos_obj.h
ob_storage_cos.cpp
ob_storage_cos.h
ob_object_device.cpp
ob_object_device.h)
@ -31,6 +27,5 @@ target_link_libraries(oss
${DEP_DIR}/lib/libapr-1.a
${DEP_DIR}/lib/libmxml.a)
add_subdirectory(cos)
target_link_libraries(restore PUBLIC oss cos_sdk oblib_base)
target_link_libraries(restore PUBLIC oss oblib_base)

63
deps/oblib/src/lib/restore/cos/CMakeLists.txt vendored
View File

@ -1,63 +0,0 @@
# Define object library cos
add_library(cos_sdk_objs OBJECT
cos_c_sdk/cos_api.h
cos_c_sdk/cos_auth.c
cos_c_sdk/cos_auth.h
cos_c_sdk/cos_bucket.c
cos_c_sdk/cos_buf.c
cos_c_sdk/cos_buf.h
cos_c_sdk/cos_crc64.c
cos_c_sdk/cos_crc64.h
cos_c_sdk/cos_define.c
cos_c_sdk/cos_define.h
cos_c_sdk/cos_fstack.c
cos_c_sdk/cos_fstack.h
cos_c_sdk/cos_http_io.c
cos_c_sdk/cos_http_io.h
cos_c_sdk/cos_list.h
cos_c_sdk/cos_log.c
cos_c_sdk/cos_log.h
cos_c_sdk/cos_multipart.c
cos_c_sdk/cos_object.c
cos_c_sdk/cos_resumable.c
cos_c_sdk/cos_resumable.h
cos_c_sdk/cos_status.c
cos_c_sdk/cos_status.h
cos_c_sdk/cos_string.h
cos_c_sdk/cos_string.c
cos_c_sdk/cos_sys_define.h
cos_c_sdk/cos_sys_util.c
cos_c_sdk/cos_sys_util.h
cos_c_sdk/cos_transport.c
cos_c_sdk/cos_transport.h
cos_c_sdk/cos_utility.c
cos_c_sdk/cos_utility.h
cos_c_sdk/cos_xml.c
cos_c_sdk/cos_xml.h
ob_singleton.h
ob_cos_wrapper.h
ob_cos_wrapper.cpp)
include_directories(./cos_c_sdk)
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
# using GCC
target_compile_options(cos_sdk_objs PRIVATE -fvisibility=hidden -Wno-unused-parameter -Wno-type-limits)
else ()
# using Clang
target_compile_options(cos_sdk_objs PRIVATE -fvisibility=hidden -Wno-error=unused-parameter -Wno-error=type-limits)
endif()
target_link_libraries(cos_sdk_objs PUBLIC oblib_base_base)
add_custom_command(
OUTPUT cos_sdk_objs.o
COMMAND ${OB_LD_BIN} -r $<TARGET_OBJECTS:cos_sdk_objs> -o cos_sdk_objs.tmp.o
COMMAND objcopy --localize-hidden cos_sdk_objs.tmp.o cos_sdk_objs.o
DEPENDS cos_sdk_objs
BYPRODUCTS cos_sdk_objs.o
COMMAND_EXPAND_LISTS
)
oblib_add_extra_objects(${CMAKE_CURRENT_BINARY_DIR}/cos_sdk_objs.o)
oblib_add_library(cos_sdk cos_sdk_objs.o)

21
deps/oblib/src/lib/restore/cos/cos_c_sdk/LICENSE vendored
View File

@ -1,21 +0,0 @@
MIT License
Copyright (c) 2017 Arvinzhu2017
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1527
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_api.h vendored
View File

File diff suppressed because it is too large Load Diff

600
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_auth.c vendored
View File

@ -1,600 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_auth.h"
#include "cos_log.h"
#include "cos_utility.h"
#if 0
static const char *g_s_cos_sub_resource_list[] = {
"acl",
"uploadId",
"uploads",
"partNumber",
"response-content-type",
"response-content-language",
"response-expires",
"response-cache-control",
"response-content-disposition",
"response-content-encoding",
"append",
"position",
"lifecycle",
"delete",
"live",
"status",
"comp",
"vod",
"startTime",
"endTime",
"x-cos-process",
"security-token",
NULL,
};
#endif
int cos_get_string_to_sign(cos_pool_t *p,
http_method_e method,
const cos_string_t *secret_id,
const cos_string_t *secret_key,
const cos_string_t *canon_res,
const cos_table_t *headers,
const cos_table_t *params,
const int64_t expire,
cos_string_t *signstr)
{
cos_buf_t *fmt_str;
cos_buf_t *sign_str;
const char *value;
apr_time_t now;
unsigned char time_str[64];
int time_str_len = 0;
unsigned char hexdigest[40];
unsigned char sign_key[40];
cos_str_null(signstr);
fmt_str = cos_create_buf(p, 1024);
if (NULL == fmt_str) {
cos_error_log("failed to call cos_create_buf.");
return COSE_OVER_MEMORY;
}
sign_str = cos_create_buf(p, 256);
if (NULL == sign_str) {
cos_error_log("failed to call cos_create_buf.");
return COSE_OVER_MEMORY;
}
// method
value = cos_http_method_to_string_lower(method);
cos_buf_append_string(p, fmt_str, value, strlen(value));
cos_buf_append_string(p, fmt_str, "\n", sizeof("\n")-1);
// canonicalized resource(URI)
cos_buf_append_string(p, fmt_str, canon_res->data, canon_res->len);
cos_buf_append_string(p, fmt_str, "\n", sizeof("\n")-1);
// query-parameters
cos_buf_append_string(p, fmt_str, "\n", sizeof("\n")-1);
// Host
cos_buf_append_string(p, fmt_str, "host=", sizeof("host=")-1);
if (headers != NULL && (value = apr_table_get(headers, COS_HOST)) != NULL) {
cos_buf_append_string(p, fmt_str, value, strlen(value));
}
cos_buf_append_string(p, fmt_str, "\n", sizeof("\n")-1);
// Format-String sha1hash
cos_get_sha1_hexdigest(hexdigest, fmt_str->pos, cos_buf_size(fmt_str));
// construct the string to sign
cos_buf_append_string(p, sign_str, "sha1\n", sizeof("sha1\n")-1);
now = apr_time_sec(apr_time_now());
time_str_len = apr_snprintf((char*)time_str, 64, "%"APR_INT64_T_FMT";%"APR_INT64_T_FMT, now, now + expire);
cos_buf_append_string(p, sign_str, (char*)time_str, time_str_len);
cos_buf_append_string(p, sign_str, "\n", sizeof("\n")-1);
cos_buf_append_string(p, sign_str, (const char*)hexdigest, sizeof(hexdigest));
cos_buf_append_string(p, sign_str, "\n", sizeof("\n")-1);
cos_get_hmac_sha1_hexdigest(sign_key, (unsigned char*)secret_key->data, secret_key->len, time_str, time_str_len);
cos_get_hmac_sha1_hexdigest(hexdigest, sign_key, sizeof(sign_key), sign_str->pos, cos_buf_size(sign_str));
value = apr_psprintf(p, "q-sign-algorithm=sha1&q-ak=%.*s&q-sign-time=%.*s&q-key-time=%.*s&q-header-list=host&q-url-param-list=&q-signature=%.*s",
secret_id->len, secret_id->data,
time_str_len, (char*)time_str,
time_str_len, (char*)time_str,
(int)sizeof(hexdigest), hexdigest);
// result
signstr->data = (char *)value;
signstr->len = strlen(value);
return COSE_OK;
}
void cos_sign_headers(cos_pool_t *p,
const cos_string_t *signstr,
const cos_string_t *access_key_id,
const cos_string_t *access_key_secret,
cos_table_t *headers)
{
apr_table_setn(headers, COS_AUTHORIZATION, signstr->data);
return;
}
int cos_get_signed_headers(cos_pool_t *p,
const cos_string_t *access_key_id,
const cos_string_t *access_key_secret,
const cos_string_t* canon_res,
cos_http_request_t *req)
{
int res;
cos_string_t signstr;
res = cos_get_string_to_sign(p, req->method, access_key_id, access_key_secret, canon_res,
req->headers, req->query_params, COS_AUTH_EXPIRE_DEFAULT, &signstr);
if (res != COSE_OK) {
return res;
}
cos_debug_log("signstr:%.*s.", signstr.len, signstr.data);
cos_sign_headers(p, &signstr, access_key_id, access_key_secret, req->headers);
return COSE_OK;
}
int cos_sign_request(cos_http_request_t *req,
const cos_config_t *config)
{
cos_string_t canon_res;
char canon_buf[COS_MAX_URI_LEN];
char datestr[COS_MAX_GMT_TIME_LEN];
const char *value;
int res = COSE_OK;
int len = 0;
len = strlen(req->resource);
if (len >= COS_MAX_URI_LEN - 1) {
cos_error_log("http resource too long, %s.", req->resource);
return COSE_INVALID_ARGUMENT;
}
canon_res.data = canon_buf;
canon_res.len = apr_snprintf(canon_buf, sizeof(canon_buf), "/%s", req->resource);
if ((value = apr_table_get(req->headers, COS_CANNONICALIZED_HEADER_DATE)) == NULL) {
cos_get_gmt_str_time(datestr);
apr_table_set(req->headers, COS_DATE, datestr);
}
if (req->host && !apr_table_get(req->headers, COS_HOST)) {
apr_table_set(req->headers, COS_HOST, req->host);
}
res = cos_get_signed_headers(req->pool, &config->access_key_id,
&config->access_key_secret, &canon_res, req);
return res;
}
#if 0
static int is_cos_sub_resource(const char *str);
static int is_cos_canonicalized_header(const char *str);
static int cos_get_canonicalized_headers(cos_pool_t *p,
const cos_table_t *headers, cos_buf_t *signbuf);
static int cos_get_canonicalized_resource(cos_pool_t *p,
const cos_table_t *params, cos_buf_t *signbuf);
static int cos_get_canonicalized_params(cos_pool_t *p,
const cos_table_t *params, cos_buf_t *signbuf);
static int is_cos_sub_resource(const char *str)
{
int i = 0;
for ( ; g_s_cos_sub_resource_list[i]; i++) {
if (apr_strnatcmp(g_s_cos_sub_resource_list[i], str) == 0) {
return 1;
}
}
return 0;
}
static int is_cos_canonicalized_header(const char *str)
{
size_t len = strlen(COS_CANNONICALIZED_HEADER_PREFIX);
return strncasecmp(str, COS_CANNONICALIZED_HEADER_PREFIX, len) == 0;
}
static int cos_get_canonicalized_headers(cos_pool_t *p,
const cos_table_t *headers,
cos_buf_t *signbuf)
{
int pos;
int meta_count = 0;
int i;
int len;
const cos_array_header_t *tarr;
const cos_table_entry_t *telts;
char **meta_headers;
const char *value;
cos_string_t tmp_str;
char *tmpbuf = (char*)malloc(COS_MAX_HEADER_LEN + 1);
if (NULL == tmpbuf) {
cos_error_log("malloc %d memory failed.", COS_MAX_HEADER_LEN + 1);
return COSE_OVER_MEMORY;
}
if (apr_is_empty_table(headers)) {
free(tmpbuf);
return COSE_OK;
}
// sort user meta header
tarr = cos_table_elts(headers);
telts = (cos_table_entry_t*)tarr->elts;
meta_headers = cos_pcalloc(p, tarr->nelts * sizeof(char*));
for (pos = 0; pos < tarr->nelts; ++pos) {
if (is_cos_canonicalized_header(telts[pos].key)) {
cos_string_t key = cos_string(telts[pos].key);
cos_string_tolower(&key);
meta_headers[meta_count++] = key.data;
}
}
if (meta_count == 0) {
free(tmpbuf);
return COSE_OK;
}
cos_gnome_sort((const char **)meta_headers, meta_count);
// sign string
for (i = 0; i < meta_count; ++i) {
value = apr_table_get(headers, meta_headers[i]);
cos_str_set(&tmp_str, value);
cos_strip_space(&tmp_str);
len = apr_snprintf(tmpbuf, COS_MAX_HEADER_LEN + 1, "%s:%.*s",
meta_headers[i], tmp_str.len, tmp_str.data);
if (len > COS_MAX_HEADER_LEN) {
free(tmpbuf);
cos_error_log("user meta header too many, %d > %d.",
len, COS_MAX_HEADER_LEN);
return COSE_INVALID_ARGUMENT;
}
tmp_str.data = tmpbuf;
tmp_str.len = len;
cos_buf_append_string(p, signbuf, tmpbuf, len);
cos_buf_append_string(p, signbuf, "\n", sizeof("\n")-1);
}
free(tmpbuf);
return COSE_OK;
}
static int cos_get_canonicalized_resource(cos_pool_t *p,
const cos_table_t *params,
cos_buf_t *signbuf)
{
int pos;
int subres_count = 0;
int i;
int len;
char sep;
const char *value;
char tmpbuf[COS_MAX_QUERY_ARG_LEN+1];
char **subres_headers;
const cos_array_header_t *tarr;
const cos_table_entry_t *telts;
if (apr_is_empty_table(params)) {
return COSE_OK;
}
// sort sub resource param
tarr = cos_table_elts(params);
telts = (cos_table_entry_t*)tarr->elts;
subres_headers = cos_pcalloc(p, tarr->nelts * sizeof(char*));
for (pos = 0; pos < tarr->nelts; ++pos) {
if (is_cos_sub_resource(telts[pos].key)) {
subres_headers[subres_count++] = telts[pos].key;
}
}
if (subres_count == 0) {
return COSE_OK;
}
cos_gnome_sort((const char **)subres_headers, subres_count);
// sign string
sep = '?';
for (i = 0; i < subres_count; ++i) {
value = apr_table_get(params, subres_headers[i]);
if (value != NULL && *value != '\0') {
len = apr_snprintf(tmpbuf, sizeof(tmpbuf), "%c%s=%s",
sep, subres_headers[i], value);
} else {
len = apr_snprintf(tmpbuf, sizeof(tmpbuf), "%c%s",
sep, subres_headers[i]);
}
if (len >= COS_MAX_QUERY_ARG_LEN) {
cos_error_log("http query params too long, %s.", tmpbuf);
return COSE_INVALID_ARGUMENT;
}
cos_buf_append_string(p, signbuf, tmpbuf, len);
sep = '&';
}
return COSE_OK;
}
int get_cos_request_signature(const cos_request_options_t *options,
cos_http_request_t *req,
const cos_string_t *expires,
cos_string_t *signature)
{
cos_string_t canon_res;
char canon_buf[COS_MAX_URI_LEN];
const char *value;
cos_string_t signstr;
int res = COSE_OK;
int b64Len;
unsigned char hmac[20];
char b64[((20 + 1) * 4) / 3];
canon_res.data = canon_buf;
canon_res.len = apr_snprintf(canon_buf, sizeof(canon_buf), "/%s", req->resource);
apr_table_set(req->headers, COS_DATE, expires->data);
if ((res = cos_get_string_to_sign(options->pool, req->method, &options->config->access_key_id, &options->config->access_key_secret, &canon_res,
req->headers, req->query_params, &signstr))!= COSE_OK) {
return res;
}
HMAC_SHA1(hmac, (unsigned char *)options->config->access_key_secret.data,
options->config->access_key_secret.len,
(unsigned char *)signstr.data, signstr.len);
b64Len = cos_base64_encode(hmac, 20, b64);
value = apr_psprintf(options->pool, "%.*s", b64Len, b64);
cos_str_set(signature, value);
return res;
}
int cos_get_signed_url(const cos_request_options_t *options,
cos_http_request_t *req,
const cos_string_t *expires,
cos_string_t *signed_url)
{
char *signed_url_str;
cos_string_t querystr;
char uristr[3*COS_MAX_URI_LEN+1];
int res = COSE_OK;
cos_string_t signature;
const char *proto;
if (options->config->sts_token.data != NULL) {
apr_table_set(req->query_params, COS_SECURITY_TOKEN, options->config->sts_token.data);
}
res = get_cos_request_signature(options, req, expires, &signature);
if (res != COSE_OK) {
return res;
}
apr_table_set(req->query_params, COS_ACCESSKEYID, options->config->access_key_id.data);
apr_table_set(req->query_params, COS_EXPIRES, expires->data);
apr_table_set(req->query_params, COS_SIGNATURE, signature.data);
uristr[0] = '\0';
cos_str_null(&querystr);
res = cos_url_encode(uristr, req->uri, COS_MAX_URI_LEN);
if (res != COSE_OK) {
return res;
}
res = cos_query_params_to_string(options->pool, req->query_params, &querystr);
if (res != COSE_OK) {
return res;
}
proto = strlen(req->proto) != 0 ? req->proto : COS_HTTP_PREFIX;
signed_url_str = apr_psprintf(options->pool, "%s%s/%s%.*s",
proto, req->host, uristr,
querystr.len, querystr.data);
cos_str_set(signed_url, signed_url_str);
return res;
}
int cos_get_rtmp_signed_url(const cos_request_options_t *options,
cos_http_request_t *req,
const cos_string_t *expires,
const cos_string_t *play_list_name,
cos_table_t *params,
cos_string_t *signed_url)
{
char *signed_url_str;
cos_string_t querystr;
char uristr[3*COS_MAX_URI_LEN+1];
int res = COSE_OK;
cos_string_t signature;
int pos = 0;
const cos_array_header_t *tarr;
const cos_table_entry_t *telts;
if (NULL != params) {
tarr = cos_table_elts(params);
telts = (cos_table_entry_t*)tarr->elts;
for (pos = 0; pos < tarr->nelts; ++pos) {
apr_table_set(req->query_params, telts[pos].key, telts[pos].val);
}
}
apr_table_set(req->query_params, COS_PLAY_LIST_NAME, play_list_name->data);
res = get_cos_rtmp_request_signature(options, req, expires,&signature);
if (res != COSE_OK) {
return res;
}
apr_table_set(req->query_params, COS_ACCESSKEYID,
options->config->access_key_id.data);
apr_table_set(req->query_params, COS_EXPIRES, expires->data);
apr_table_set(req->query_params, COS_SIGNATURE, signature.data);
uristr[0] = '\0';
cos_str_null(&querystr);
res = cos_url_encode(uristr, req->uri, COS_MAX_URI_LEN);
if (res != COSE_OK) {
return res;
}
res = cos_query_params_to_string(options->pool, req->query_params, &querystr);
if (res != COSE_OK) {
return res;
}
signed_url_str = apr_psprintf(options->pool, "%s%s/%s%.*s",
req->proto, req->host, uristr,
querystr.len, querystr.data);
cos_str_set(signed_url, signed_url_str);
return res;
}
int get_cos_rtmp_request_signature(const cos_request_options_t *options,
cos_http_request_t *req,
const cos_string_t *expires,
cos_string_t *signature)
{
cos_string_t canon_res;
char canon_buf[COS_MAX_URI_LEN];
const char *value;
cos_string_t signstr;
int res = COSE_OK;
int b64Len;
unsigned char hmac[20];
char b64[((20 + 1) * 4) / 3];
canon_res.data = canon_buf;
canon_res.len = apr_snprintf(canon_buf, sizeof(canon_buf), "/%s", req->resource);
if ((res = cos_get_rtmp_string_to_sign(options->pool, expires, &canon_res,
req->query_params, &signstr))!= COSE_OK) {
return res;
}
HMAC_SHA1(hmac, (unsigned char *)options->config->access_key_secret.data,
options->config->access_key_secret.len,
(unsigned char *)signstr.data, signstr.len);
b64Len = cos_base64_encode(hmac, 20, b64);
value = apr_psprintf(options->pool, "%.*s", b64Len, b64);
cos_str_set(signature, value);
return res;
}
int cos_get_rtmp_string_to_sign(cos_pool_t *p,
const cos_string_t *expires,
const cos_string_t *canon_res,
const cos_table_t *params,
cos_string_t *signstr)
{
int res;
cos_buf_t *signbuf;
cos_str_null(signstr);
signbuf = cos_create_buf(p, 1024);
// expires
cos_buf_append_string(p, signbuf, expires->data, expires->len);
cos_buf_append_string(p, signbuf, "\n", sizeof("\n")-1);
// canonicalized params
if ((res = cos_get_canonicalized_params(p, params, signbuf)) != COSE_OK) {
return res;
}
// canonicalized resource
cos_buf_append_string(p, signbuf, canon_res->data, canon_res->len);
// result
signstr->data = (char *)signbuf->pos;
signstr->len = cos_buf_size(signbuf);
return COSE_OK;
}
static int cos_get_canonicalized_params(cos_pool_t *p,
const cos_table_t *params,
cos_buf_t *signbuf)
{
int pos;
int meta_count = 0;
int i;
int len;
const cos_array_header_t *tarr;
const cos_table_entry_t *telts;
char **meta_headers;
const char *value;
cos_string_t tmp_str;
char *tmpbuf = (char*)malloc(COS_MAX_HEADER_LEN + 1);
if (NULL == tmpbuf) {
cos_error_log("malloc %d memory failed.", COS_MAX_HEADER_LEN + 1);
return COSE_OVER_MEMORY;
}
if (apr_is_empty_table(params)) {
free(tmpbuf);
return COSE_OK;
}
// sort user meta header
tarr = cos_table_elts(params);
telts = (cos_table_entry_t*)tarr->elts;
meta_headers = cos_pcalloc(p, tarr->nelts * sizeof(char*));
for (pos = 0; pos < tarr->nelts; ++pos) {
cos_string_t key = cos_string(telts[pos].key);
meta_headers[meta_count++] = key.data;
}
if (meta_count == 0) {
free(tmpbuf);
return COSE_OK;
}
cos_gnome_sort((const char **)meta_headers, meta_count);
// sign string
for (i = 0; i < meta_count; ++i) {
value = apr_table_get(params, meta_headers[i]);
cos_str_set(&tmp_str, value);
cos_strip_space(&tmp_str);
len = apr_snprintf(tmpbuf, COS_MAX_HEADER_LEN + 1, "%s:%.*s",
meta_headers[i], tmp_str.len, tmp_str.data);
if (len > COS_MAX_HEADER_LEN) {
free(tmpbuf);
cos_error_log("rtmp parameters too many, %d > %d.",
len, COS_MAX_HEADER_LEN);
return COSE_INVALID_ARGUMENT;
}
tmp_str.data = tmpbuf;
tmp_str.len = len;
cos_buf_append_string(p, signbuf, tmpbuf, len);
cos_buf_append_string(p, signbuf, "\n", sizeof("\n")-1);
}
free(tmpbuf);
return COSE_OK;
}
#endif

91
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_auth.h vendored
View File

@ -1,91 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIB_COS_AUTH_H
#define LIB_COS_AUTH_H
#include "cos_sys_util.h"
#include "cos_string.h"
#include "cos_http_io.h"
#include "cos_define.h"
COS_CPP_START
/**
* @brief sign cos headers
**/
void cos_sign_headers(cos_pool_t *p,
const cos_string_t *signstr,
const cos_string_t *access_key_id,
const cos_string_t *access_key_secret,
cos_table_t *headers);
/**
* @brief get string to signature
**/
int cos_get_string_to_sign(cos_pool_t *p,
http_method_e method,
const cos_string_t *secret_id,
const cos_string_t *secret_key,
const cos_string_t *canon_res,
const cos_table_t *headers,
const cos_table_t *params,
const int64_t expire,
cos_string_t *signstr);
/**
* @brief get signed cos request headers
**/
int cos_get_signed_headers(cos_pool_t *p, const cos_string_t *access_key_id,
const cos_string_t *access_key_secret,
const cos_string_t* canon_res, cos_http_request_t *req);
/**
* @brief sign cos request
**/
int cos_sign_request(cos_http_request_t *req, const cos_config_t *config);
/**
* @brief generate cos request Signature
**/
int get_cos_request_signature(const cos_request_options_t *options, cos_http_request_t *req,
const cos_string_t *expires, cos_string_t *signature);
/**
* @brief get cos signed url
**/
int cos_get_signed_url(const cos_request_options_t *options, cos_http_request_t *req,
const cos_string_t *expires, cos_string_t *auth_url);
/**
* @brief get rtmp string to signature
**/
int cos_get_rtmp_string_to_sign(cos_pool_t *p, const cos_string_t *expires,
const cos_string_t *canon_res, const cos_table_t *params,
cos_string_t *signstr);
/**
* @brief generate cos rtmp request signature
**/
int get_cos_rtmp_request_signature(const cos_request_options_t *options, cos_http_request_t *req,
const cos_string_t *expires, cos_string_t *signature);
/**
* @brief get cos rtmp signed url
**/
int cos_get_rtmp_signed_url(const cos_request_options_t *options, cos_http_request_t *req,
const cos_string_t *expires, const cos_string_t *play_list_name, cos_table_t *params,
cos_string_t *signed_url);
COS_CPP_END
#endif

1482
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_bucket.c vendored
View File

File diff suppressed because it is too large Load Diff

203
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_buf.c vendored
View File

@ -1,203 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_buf.h"
#include "cos_log.h"
#include <apr_file_io.h>
cos_buf_t *cos_create_buf(cos_pool_t *p, int size)
{
cos_buf_t* b;
b = cos_palloc(p, sizeof(cos_buf_t) + size);
if (b == NULL) {
return NULL;
}
b->pos = (uint8_t *)b + sizeof(cos_buf_t);
b->start = b->pos;
b->last = b->start;
b->end = b->last + size;
cos_list_init(&b->node);
return b;
}
cos_buf_t *cos_buf_pack(cos_pool_t *p, const void *data, int size)
{
cos_buf_t* b;
b = cos_palloc(p, sizeof(cos_buf_t));
if (b == NULL) {
return NULL;
}
b->pos = (uint8_t *)data;
b->start = b->pos;
b->last = b->start + size;
b->end = b->last;
cos_list_init(&b->node);
return b;
}
int64_t cos_buf_list_len(cos_list_t *list)
{
cos_buf_t *b;
int64_t len = 0;
cos_list_for_each_entry(cos_buf_t, b, list, node) {
len += cos_buf_size(b);
}
return len;
}
char *cos_buf_list_content(cos_pool_t *p, cos_list_t *list)
{
int64_t body_len;
char *buf;
int64_t pos = 0;
int64_t size = 0;
cos_buf_t *content;
body_len = cos_buf_list_len(list);
buf = cos_pcalloc(p, (size_t)(body_len + 1));
buf[body_len] = '\0';
cos_list_for_each_entry(cos_buf_t, content, list, node) {
size = cos_buf_size(content);
memcpy(buf + pos, content->pos, (size_t)(size));
pos += size;
}
return buf;
}
cos_file_buf_t *cos_create_file_buf(cos_pool_t *p)
{
return (cos_file_buf_t*)cos_pcalloc(p, sizeof(cos_file_buf_t));
}
int cos_open_file_for_read(cos_pool_t *p, const char *path, cos_file_buf_t *fb)
{
int s;
char buf[256];
apr_finfo_t finfo;
if ((s = apr_file_open(&fb->file, path, APR_READ, APR_UREAD | APR_GREAD, p)) != APR_SUCCESS) {
cos_error_log("apr_file_open failure, code:%d %s.", s, apr_strerror(s, buf, sizeof(buf)));
assert(fb->file == NULL);
return COSE_OPEN_FILE_ERROR;
}
if ((s = apr_file_info_get(&finfo, APR_FINFO_NORM, fb->file)) != APR_SUCCESS) {
apr_file_close(fb->file);
cos_error_log("apr_file_open failure, code:%d %s.", s, apr_strerror(s, buf, sizeof(buf)));
return COSE_FILE_INFO_ERROR;
}
fb->file_pos = 0;
fb->file_last = finfo.size;
fb->owner = 1;
return COSE_OK;
}
int cos_open_file_for_all_read(cos_pool_t *p, const char *path, cos_file_buf_t *fb)
{
return cos_open_file_for_read(p, path, fb);
}
int cos_open_file_for_range_read(cos_pool_t *p, const char *path,
int64_t file_pos, int64_t file_last, cos_file_buf_t *fb)
{
int s;
s = cos_open_file_for_read(p, path, fb);
if (s == COSE_OK) {
if (file_pos > fb->file_pos) {
if (file_pos > fb->file_last) {
cos_warn_log("read range beyond file size, read start:%" APR_INT64_T_FMT ", file size:%" APR_INT64_T_FMT "\n",
file_pos, fb->file_last);
file_pos = fb->file_last;
}
fb->file_pos = file_pos;
}
if (file_last < fb->file_last) {
fb->file_last = file_last;
}
apr_file_seek(fb->file, APR_SET, (apr_off_t *)&fb->file_pos);
}
return s;
}
int cos_open_file_for_write(cos_pool_t *p, const char *path, cos_file_buf_t *fb)
{
int s;
char buf[256];
if ((s = apr_file_open(&fb->file, path, APR_CREATE | APR_WRITE | APR_TRUNCATE,
APR_UREAD | APR_UWRITE | APR_GREAD, p)) != APR_SUCCESS) {
cos_error_log("apr_file_open failure, code:%d %s.", s, apr_strerror(s, buf, sizeof(buf)));
assert(fb->file == NULL);
return COSE_OPEN_FILE_ERROR;
}
fb->owner = 1;
return COSE_OK;
}
int cos_open_file_for_range_write(cos_pool_t *p, const char *path, int64_t file_pos, int64_t file_last, cos_file_buf_t *fb)
{
int s;
char buf[256];
if ((s = apr_file_open(&fb->file, path, APR_CREATE | APR_WRITE,
APR_UREAD | APR_UWRITE | APR_GREAD, p)) != APR_SUCCESS) {
cos_error_log("apr_file_open failure, code:%d %s.", s, apr_strerror(s, buf, sizeof(buf)));
assert(fb->file == NULL);
return COSE_OPEN_FILE_ERROR;
}
fb->owner = 1;
fb->file_pos = file_pos;
fb->file_last = file_last;
apr_file_seek(fb->file, APR_SET, (apr_off_t *)&fb->file_pos);
return COSE_OK;
}
void cos_buf_append_string(cos_pool_t *p, cos_buf_t *b, const char *str, int len)
{
int size;
int nsize;
int remain;
char *buf;
if (len <= 0) return;
remain = b->end - b->last;
if (remain > len + 128) {
memcpy(b->last, str, len);
b->last += len;
} else {
size = cos_buf_size(b);
nsize = (size + len) * 2;
buf = cos_palloc(p, nsize);
memcpy(buf, b->pos, size);
memcpy(buf+size, str, len);
b->start = (uint8_t *)buf;
b->end = (uint8_t *)buf + nsize;
b->pos = (uint8_t *)buf;
b->last = (uint8_t *)buf + size + len;
}
}

74
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_buf.h vendored
View File

@ -1,74 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_BUF_H
#define LIBCOS_BUF_H
#include "cos_sys_define.h"
#include "cos_list.h"
COS_CPP_START
typedef struct {
cos_list_t node;
uint8_t *pos;
uint8_t *last;
uint8_t *start;
uint8_t *end;
} cos_buf_t;
typedef struct {
cos_list_t node;
int64_t file_pos;
int64_t file_last;
apr_file_t *file;
uint32_t owner:1;
} cos_file_buf_t;
cos_buf_t *cos_create_buf(cos_pool_t *p, int size);
#define cos_buf_size(b) (b->last - b->pos)
cos_file_buf_t *cos_create_file_buf(cos_pool_t *p);
cos_buf_t *cos_buf_pack(cos_pool_t *p, const void *data, int size);
int64_t cos_buf_list_len(cos_list_t *list);
char *cos_buf_list_content(cos_pool_t *p, cos_list_t *list);
void cos_buf_append_string(cos_pool_t *p, cos_buf_t *b, const char *str, int len);
/**
* @param fb file_pos, file_last equal file_size.
* @return COSE_OK success, other failure.
*/
int cos_open_file_for_read(cos_pool_t *p, const char *path, cos_file_buf_t *fb);
int cos_open_file_for_all_read(cos_pool_t *p, const char *path, cos_file_buf_t *fb);
int cos_open_file_for_range_read(cos_pool_t *p, const char *path,
int64_t file_pos, int64_t file_last,
cos_file_buf_t *fb);
/**
* create the file if not there, truncate if file exists.
* @param fb not check file_pos, file_last.
* @return COSE_OK success, other failure.
*/
int cos_open_file_for_write(cos_pool_t *p, const char *path, cos_file_buf_t *fb);
int cos_open_file_for_range_write(cos_pool_t *p, const char *path, int64_t file_pos, int64_t file_last, cos_file_buf_t *fb);
COS_CPP_END
#endif

261
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_crc64.c vendored
View File

@ -1,261 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_crc64.h"
/* 64-bit CRC polynomial with these coefficients, but reversed:
64, 62, 57, 55, 54, 53, 52, 47, 46, 45, 40, 39, 38, 37, 35, 33, 32,
31, 29, 27, 24, 23, 22, 21, 19, 17, 13, 12, 10, 9, 7, 4, 1, 0 */
#define POLY UINT64_C(0xc96c5795d7870f42)
/* Tables for CRC calculation -- filled in by initialization functions that are
called once. These could be replaced by constant tables generated in the
same way. There are two tables, one for each endianess. Since these are
static, i.e. local, one should be compiled out of existence if the compiler
can evaluate the endianess check in crc64() at compile time. */
static uint64_t crc64_little_table[8][256];
static uint64_t crc64_big_table[8][256];
/* Fill in the CRC-64 constants table. */
static void crc64_init(uint64_t table[][256])
{
unsigned n, k;
uint64_t crc;
/* generate CRC-64's for all single byte sequences */
for (n = 0; n < 256; n++) {
crc = n;
for (k = 0; k < 8; k++)
crc = crc & 1 ? POLY ^ (crc >> 1) : crc >> 1;
table[0][n] = crc;
}
/* generate CRC-64's for those followed by 1 to 7 zeros */
for (n = 0; n < 256; n++) {
crc = table[0][n];
for (k = 1; k < 8; k++) {
crc = table[0][crc & 0xff] ^ (crc >> 8);
table[k][n] = crc;
}
}
}
/* This function is called once to initialize the CRC-64 table for use on a
little-endian architecture. */
static void crc64_little_init(void)
{
crc64_init(crc64_little_table);
}
/* Reverse the bytes in a 64-bit word. */
static APR_INLINE uint64_t rev8(uint64_t a)
{
uint64_t m;
m = UINT64_C(0xff00ff00ff00ff);
a = ((a >> 8) & m) | (a & m) << 8;
m = UINT64_C(0xffff0000ffff);
a = ((a >> 16) & m) | (a & m) << 16;
return a >> 32 | a << 32;
}
/* This function is called once to initialize the CRC-64 table for use on a
big-endian architecture. */
static void crc64_big_init(void)
{
unsigned k, n;
crc64_init(crc64_big_table);
for (k = 0; k < 8; k++)
for (n = 0; n < 256; n++)
crc64_big_table[k][n] = rev8(crc64_big_table[k][n]);
}
/* Run the init() function exactly once. If pthread.h is not included, then
this macro will use a simple static state variable for the purpose, which is
not thread-safe. The init function must be of the type void init(void). */
#ifdef PTHREAD_ONCE_INIT
# define ONCE(init) \
do { \
static pthread_once_t once = PTHREAD_ONCE_INIT; \
pthread_once(&once, init); \
} while (0)
#else
# define ONCE(init) \
do { \
static volatile int once = 1; \
if (once) { \
if (once++ == 1) { \
init(); \
once = 0; \
} \
else \
while (once) \
; \
} \
} while (0)
#endif
/* Calculate a CRC-64 eight bytes at a time on a little-endian architecture. */
static APR_INLINE uint64_t crc64_little(uint64_t crc, void *buf, size_t len)
{
unsigned char *next = buf;
ONCE(crc64_little_init);
crc = ~crc;
while (len && ((uintptr_t)next & 7) != 0) {
crc = crc64_little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
len--;
}
while (len >= 8) {
crc ^= *(uint64_t *)next;
crc = crc64_little_table[7][crc & 0xff] ^
crc64_little_table[6][(crc >> 8) & 0xff] ^
crc64_little_table[5][(crc >> 16) & 0xff] ^
crc64_little_table[4][(crc >> 24) & 0xff] ^
crc64_little_table[3][(crc >> 32) & 0xff] ^
crc64_little_table[2][(crc >> 40) & 0xff] ^
crc64_little_table[1][(crc >> 48) & 0xff] ^
crc64_little_table[0][crc >> 56];
next += 8;
len -= 8;
}
while (len) {
crc = crc64_little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
len--;
}
return ~crc;
}
/* Calculate a CRC-64 eight bytes at a time on a big-endian architecture. */
static APR_INLINE uint64_t crc64_big(uint64_t crc, void *buf, size_t len)
{
unsigned char *next = buf;
ONCE(crc64_big_init);
crc = ~rev8(crc);
while (len && ((uintptr_t)next & 7) != 0) {
crc = crc64_big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
len--;
}
while (len >= 8) {
crc ^= *(uint64_t *)next;
crc = crc64_big_table[0][crc & 0xff] ^
crc64_big_table[1][(crc >> 8) & 0xff] ^
crc64_big_table[2][(crc >> 16) & 0xff] ^
crc64_big_table[3][(crc >> 24) & 0xff] ^
crc64_big_table[4][(crc >> 32) & 0xff] ^
crc64_big_table[5][(crc >> 40) & 0xff] ^
crc64_big_table[6][(crc >> 48) & 0xff] ^
crc64_big_table[7][crc >> 56];
next += 8;
len -= 8;
}
while (len) {
crc = crc64_big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
len--;
}
return ~rev8(crc);
}
/* Return the CRC-64 of buf[0..len-1] with initial crc, processing eight bytes
at a time. This selects one of two routines depending on the endianess of
the architecture. A good optimizing compiler will determine the endianess
at compile time if it can, and get rid of the unused code and table. If the
endianess can be changed at run time, then this code will handle that as
well, initializing and using two tables, if called upon to do so. */
uint64_t cos_crc64(uint64_t crc, void *buf, size_t len)
{
uint64_t n = 1;
return *(char *)&n ? crc64_little(crc, buf, len) :
crc64_big(crc, buf, len);
}
#define GF2_DIM 64 /* dimension of GF(2) vectors (length of CRC) */
static uint64_t gf2_matrix_times(uint64_t *mat, uint64_t vec)
{
uint64_t sum;
sum = 0;
while (vec) {
if (vec & 1)
sum ^= *mat;
vec >>= 1;
mat++;
}
return sum;
}
static void gf2_matrix_square(uint64_t *square, uint64_t *mat)
{
unsigned n;
for (n = 0; n < GF2_DIM; n++)
square[n] = gf2_matrix_times(mat, mat[n]);
}
/* Return the CRC-64 of two sequential blocks, where crc1 is the CRC-64 of the
first block, crc2 is the CRC-64 of the second block, and len2 is the length
of the second block. */
uint64_t cos_crc64_combine(uint64_t crc1, uint64_t crc2, uintmax_t len2)
{
unsigned n;
uint64_t row;
uint64_t even[GF2_DIM]; /* even-power-of-two zeros operator */
uint64_t odd[GF2_DIM]; /* odd-power-of-two zeros operator */
/* degenerate case */
if (len2 == 0)
return crc1;
/* put operator for one zero bit in odd */
odd[0] = POLY; /* CRC-64 polynomial */
row = 1;
for (n = 1; n < GF2_DIM; n++) {
odd[n] = row;
row <<= 1;
}
/* put operator for two zero bits in even */
gf2_matrix_square(even, odd);
/* put operator for four zero bits in odd */
gf2_matrix_square(odd, even);
/* apply len2 zeros to crc1 (first square will put the operator for one
zero byte, eight zero bits, in even) */
do {
/* apply zeros operator for this bit of len2 */
gf2_matrix_square(even, odd);
if (len2 & 1)
crc1 = gf2_matrix_times(even, crc1);
len2 >>= 1;
/* if no more bits set, then done */
if (len2 == 0)
break;
/* another iteration of the loop with odd and even swapped */
gf2_matrix_square(odd, even);
if (len2 & 1)
crc1 = gf2_matrix_times(odd, crc1);
len2 >>= 1;
/* if no more bits set, then done */
} while (len2 != 0);
/* return combined crc */
crc1 ^= crc2;
return crc1;
}

26
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_crc64.h vendored
View File

@ -1,26 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_CRC_H
#define LIBCOS_CRC_H
#include "cos_sys_define.h"
COS_CPP_START
uint64_t cos_crc64(uint64_t crc, void *buf, size_t len);
uint64_t cos_crc64_combine(uint64_t crc1, uint64_t crc2, uintmax_t len2);
COS_CPP_END
#endif

97
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_define.c vendored
View File

@ -1,97 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_define.h"
const char COS_CANNONICALIZED_HEADER_PREFIX[] = "x-cos-";
const char COS_CANNONICALIZED_HEADER_DATE[] = "x-cos-date";
const char COS_CANNONICALIZED_HEADER_ACL[] = "x-cos-acl";
const char COS_CANNONICALIZED_HEADER_COPY_SOURCE[] = "x-cos-copy-source";
const char COS_GRANT_READ[] = "x-cos-grant-read";
const char COS_GRANT_WRITE[] = "x-cos-grant-write";
const char COS_GRANT_FULL_CONTROL[] = "x-cos-grant-full-control";
const char COS_CONTENT_MD5[] = "Content-MD5";
const char COS_CONTENT_TYPE[] = "Content-Type";
const char COS_CONTENT_LENGTH[] = "Content-Length";
const char COS_DATE[] = "Date";
const char COS_AUTHORIZATION[] = "Authorization";
const char COS_ACCESSKEYID[] = "COSAccessKeyId";
const char COS_EXPECT[] = "Expect";
const char COS_TRANSFER_ENCODING[] = "Transfer-Encoding";
const char COS_HOST[] = "Host";
const char COS_RANGE[] = "Range";
const char COS_EXPIRES[] = "Expires";
const char COS_SIGNATURE[] = "Signature";
const char COS_ACL[] = "acl";
const char COS_ENCODING_TYPE[] = "encoding-type";
const char COS_PREFIX[] = "prefix";
const char COS_DELIMITER[] = "delimiter";
const char COS_MARKER[] = "marker";
const char COS_MAX_KEYS[] = "max-keys";
const char COS_RESTORE[] = "restore";
const char COS_UPLOADS[] = "uploads";
const char COS_UPLOAD_ID[] = "uploadId";
const char COS_MAX_PARTS[] = "max-parts";
const char COS_PART_NUMBER_MARKER[] = "part-number-marker";
const char COS_KEY_MARKER[] = "key-marker";
const char COS_UPLOAD_ID_MARKER[] = "upload-id-marker";
const char COS_MAX_UPLOADS[] = "max-uploads";
const char COS_PARTNUMBER[] = "partNumber";
const char COS_APPEND[] = "append";
const char COS_POSITION[] = "position";
const char COS_MULTIPART_CONTENT_TYPE[] = "application/x-www-form-urlencoded";
const char COS_COPY_SOURCE[] = "x-cos-copy-source";
const char COS_COPY_SOURCE_RANGE[] = "x-cos-copy-source-range";
const char COS_SECURITY_TOKEN[] = "security-token";
const char COS_STS_SECURITY_TOKEN[] = "x-cos-security-token";
const char COS_REPLACE_OBJECT_META[] = "x-cos-replace-object-meta";
const char COS_OBJECT_TYPE[] = "x-cos-object-type";
const char COS_NEXT_APPEND_POSITION[] = "x-cos-next-append-position";
const char COS_HASH_CRC64_ECMA[] = "x-cos-hash-crc64ecma";
const char COS_CONTENT_SHA1[] = "x-cos-content-sha1";
const char COS_CALLBACK[] = "x-cos-callback";
const char COS_CALLBACK_VAR[] = "x-cos-callback-var";
const char COS_PROCESS[] = "x-cos-process";
const char COS_LIFECYCLE[] = "lifecycle";
const char COS_CORS[] = "cors";
const char COS_REPLICATION[] = "replication";
const char COS_VERSIONING[] = "versioning";
const char COS_WEBSITE[] = "website";
const char COS_DOMAIN[] = "domain";
const char COS_DELETE[] = "delete";
const char COS_LOGGING[] = "logging";
const char COS_INVENTORY[] = "inventory";
const char COS_TAGGING[] = "tagging";
const char COS_YES[] = "yes";
const char COS_OBJECT_TYPE_NORMAL[] = "normal";
const char COS_OBJECT_TYPE_APPENDABLE[] = "appendable";
const char COS_LIVE_CHANNEL[] = "live";
const char COS_LIVE_CHANNEL_STATUS[] = "status";
const char COS_COMP[] = "comp";
const char COS_LIVE_CHANNEL_STAT[] = "stat";
const char COS_LIVE_CHANNEL_HISTORY[] = "history";
const char COS_LIVE_CHANNEL_VOD[] = "vod";
const char COS_LIVE_CHANNEL_START_TIME[] = "startTime";
const char COS_LIVE_CHANNEL_END_TIME[] = "endTime";
const char COS_PLAY_LIST_NAME[] = "playlistName";
const char LIVE_CHANNEL_STATUS_DISABLED[] = "disabled";
const char LIVE_CHANNEL_STATUS_ENABLED[] = "enabled";
const char LIVE_CHANNEL_STATUS_IDLE[] = "idle";
const char LIVE_CHANNEL_STATUS_LIVE[] = "live";
const char LIVE_CHANNEL_DEFAULT_TYPE[] = "HLS";
const char LIVE_CHANNEL_DEFAULT_PLAYLIST[] = "playlist.m3u8";
const int LIVE_CHANNEL_DEFAULT_FRAG_DURATION = 5;
const int LIVE_CHANNEL_DEFAULT_FRAG_COUNT = 3;
const int COS_MAX_PART_NUM = 10000;
const int COS_PER_RET_NUM = 1000;
const int MAX_SUFFIX_LEN = 1024;
const char COS_INTELLIGENTTIERING[] = "intelligenttiering";

517
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_define.h vendored
View File

@ -1,517 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_DEFINE_H
#define LIBCOS_DEFINE_H
#include "cos_string.h"
#include "cos_list.h"
#include "cos_transport.h"
#ifdef __cplusplus
#define COS_CPP_START extern "C" {
#define COS_CPP_END }
#else
#define COS_CPP_START
#define COS_CPP_END
#endif
#define cos_xml_error_status_set(STATUS, RES) do { \
cos_status_set(STATUS, RES, COS_XML_PARSE_ERROR_CODE, NULL); \
} while(0)
#define cos_file_error_status_set(STATUS, RES) do { \
cos_status_set(STATUS, RES, COS_OPEN_FILE_ERROR_CODE, NULL); \
} while(0)
#define cos_inconsistent_error_status_set(STATUS, RES) do { \
cos_status_set(STATUS, RES, COS_INCONSISTENT_ERROR_CODE, NULL); \
} while(0)
extern const char COS_CANNONICALIZED_HEADER_ACL[];
extern const char COS_CANNONICALIZED_HEADER_SOURCE[];
extern const char COS_CANNONICALIZED_HEADER_PREFIX[];
extern const char COS_CANNONICALIZED_HEADER_DATE[];
extern const char COS_CANNONICALIZED_HEADER_COPY_SOURCE[];
extern const char COS_GRANT_READ[];
extern const char COS_GRANT_WRITE[];
extern const char COS_GRANT_FULL_CONTROL[];
extern const char COS_CONTENT_MD5[];
extern const char COS_CONTENT_TYPE[];
extern const char COS_CONTENT_LENGTH[];
extern const char COS_DATE[];
extern const char COS_AUTHORIZATION[];
extern const char COS_ACCESSKEYID[];
extern const char COS_EXPECT[];
extern const char COS_TRANSFER_ENCODING[];
extern const char COS_HOST[];
extern const char COS_EXPIRES[];
extern const char COS_SIGNATURE[];
extern const char COS_ACL[];
extern const char COS_ENCODING_TYPE[];
extern const char COS_PREFIX[];
extern const char COS_DELIMITER[];
extern const char COS_MARKER[];
extern const char COS_MAX_KEYS[];
extern const char COS_RESTORE[];
extern const char COS_UPLOADS[];
extern const char COS_UPLOAD_ID[];
extern const char COS_MAX_PARTS[];
extern const char COS_KEY_MARKER[];
extern const char COS_UPLOAD_ID_MARKER[];
extern const char COS_MAX_UPLOADS[];
extern const char COS_PARTNUMBER[];
extern const char COS_PART_NUMBER_MARKER[];
extern const char COS_APPEND[];
extern const char COS_POSITION[];
extern const char COS_MULTIPART_CONTENT_TYPE[];
extern const char COS_COPY_SOURCE[];
extern const char COS_COPY_SOURCE_RANGE[];
extern const char COS_SECURITY_TOKEN[];
extern const char COS_STS_SECURITY_TOKEN[];
extern const char COS_REPLACE_OBJECT_META[];
extern const char COS_OBJECT_TYPE[];
extern const char COS_NEXT_APPEND_POSITION[];
extern const char COS_HASH_CRC64_ECMA[];
extern const char COS_CALLBACK[];
extern const char COS_CALLBACK_VAR[];
extern const char COS_PROCESS[];
extern const char COS_LIFECYCLE[];
extern const char COS_CORS[];
extern const char COS_VERSIONING[];
extern const char COS_REPLICATION[];
extern const char COS_WEBSITE[];
extern const char COS_DOMAIN[];
extern const char COS_LOGGING[];
extern const char COS_INVENTORY[];
extern const char COS_TAGGING[];
extern const char COS_DELETE[];
extern const char COS_YES[];
extern const char COS_OBJECT_TYPE_NORMAL[];
extern const char COS_OBJECT_TYPE_APPENDABLE[];
extern const char COS_LIVE_CHANNEL[];
extern const char COS_LIVE_CHANNEL_STATUS[];
extern const char COS_COMP[];
extern const char COS_LIVE_CHANNEL_STAT[];
extern const char COS_LIVE_CHANNEL_HISTORY[];
extern const char COS_LIVE_CHANNEL_VOD[];
extern const char COS_LIVE_CHANNEL_START_TIME[];
extern const char COS_LIVE_CHANNEL_END_TIME[];
extern const char COS_PLAY_LIST_NAME[];
extern const char LIVE_CHANNEL_STATUS_DISABLED[];
extern const char LIVE_CHANNEL_STATUS_ENABLED[];
extern const char LIVE_CHANNEL_STATUS_IDLE[];
extern const char LIVE_CHANNEL_STATUS_LIVE[];
extern const char LIVE_CHANNEL_DEFAULT_TYPE[];
extern const char LIVE_CHANNEL_DEFAULT_PLAYLIST[];
extern const int LIVE_CHANNEL_DEFAULT_FRAG_DURATION;
extern const int LIVE_CHANNEL_DEFAULT_FRAG_COUNT;
extern const int COS_MAX_PART_NUM;
extern const int COS_PER_RET_NUM;
extern const int MAX_SUFFIX_LEN;
extern const char COS_CONTENT_SHA1[];
extern const char COS_RANGE[];
extern const char COS_INTELLIGENTTIERING[];
typedef struct cos_lib_curl_initializer_s cos_lib_curl_initializer_t;
/**
* cos_acl is an ACL that can be specified when an object is created or
* updated. Each canned ACL has a predefined value when expanded to a full
* set of COS ACL Grants.
* Private canned ACL gives the owner FULL_CONTROL and no other permissions
* are issued
* Public Read canned ACL gives the owner FULL_CONTROL and all users Read
* permission
* Public Read Write canned ACL gives the owner FULL_CONTROL and all users
* Read and Write permission
**/
typedef enum {
COS_ACL_PRIVATE = 0, /*< private */
COS_ACL_PUBLIC_READ = 1, /*< public read */
COS_ACL_PUBLIC_READ_WRITE = 2, /*< public read write */
COS_ACL_DEFAULT = 3 /*< default */
} cos_acl_e;
typedef struct {
cos_string_t endpoint;
cos_string_t access_key_id;
cos_string_t access_key_secret;
cos_string_t appid;
cos_string_t sts_token;
int is_cname;
cos_string_t proxy_host;
int proxy_port;
cos_string_t proxy_user;
cos_string_t proxy_passwd;
} cos_config_t;
typedef struct {
cos_config_t *config;
cos_http_controller_t *ctl; /*< cos http controller, more see cos_transport.h */
cos_pool_t *pool;
} cos_request_options_t;
typedef struct {
cos_list_t node;
cos_string_t type;
cos_string_t id;
cos_string_t name;
cos_string_t permission;
} cos_acl_grantee_content_t;
typedef struct {
cos_string_t owner_id;
cos_string_t owner_name;;
cos_list_t grantee_list;
} cos_acl_params_t;
typedef struct {
cos_string_t etag;
cos_string_t last_modify;;
} cos_copy_object_params_t;
typedef struct {
cos_list_t node;
cos_string_t key;
cos_string_t last_modified;
cos_string_t etag;
cos_string_t size;
cos_string_t owner_id;
cos_string_t owner_display_name;
cos_string_t storage_class;
} cos_list_object_content_t;
typedef struct {
cos_list_t node;
cos_string_t prefix;
} cos_list_object_common_prefix_t;
typedef struct {
cos_list_t node;
cos_string_t key;
cos_string_t upload_id;
cos_string_t initiated;
} cos_list_multipart_upload_content_t;
typedef struct {
cos_list_t node;
cos_string_t part_number;
cos_string_t size;
cos_string_t etag;
cos_string_t last_modified;
} cos_list_part_content_t;
typedef struct {
cos_list_t node;
cos_string_t part_number;
cos_string_t etag;
} cos_complete_part_content_t;
typedef struct {
int part_num;
char *etag;
} cos_upload_part_t;
typedef struct {
cos_list_t node;
cos_string_t bucket_name;
cos_string_t location;
cos_string_t creation_date;
} cos_get_service_content_t;
typedef struct {
int all_region;
cos_string_t owner_id;
cos_string_t owner_display_name;
cos_list_t bucket_list;
} cos_get_service_params_t;
typedef struct {
cos_string_t encoding_type;
cos_string_t prefix;
cos_string_t marker;
cos_string_t delimiter;
int max_ret;
int truncated;
cos_string_t next_marker;
cos_list_t object_list;
cos_list_t common_prefix_list;
} cos_list_object_params_t;
typedef struct {
cos_string_t encoding_type;
cos_string_t part_number_marker;
int max_ret;
int truncated;
cos_string_t next_part_number_marker;
cos_list_t part_list;
} cos_list_upload_part_params_t;
typedef struct {
cos_string_t encoding_type;
cos_string_t prefix;
cos_string_t key_marker;
cos_string_t upload_id_marker;
cos_string_t delimiter;
int max_ret;
int truncated;
cos_string_t next_key_marker;
cos_string_t next_upload_id_marker;
cos_list_t upload_list;
} cos_list_multipart_upload_params_t;
typedef struct {
cos_string_t copy_source;
cos_string_t dest_bucket;
cos_string_t dest_object;
cos_string_t upload_id;
int part_num;
int64_t range_start;
int64_t range_end;
cos_copy_object_params_t *rsp_content;
} cos_upload_part_copy_params_t;
typedef struct {
cos_string_t filename; /**< file range read filename */
int64_t file_pos; /**< file range read start position */
int64_t file_last; /**< file range read last position */
} cos_upload_file_t;
typedef struct {
int days;
cos_string_t date;
cos_string_t storage_class;
} cos_lifecycle_expire_t;
typedef struct {
int days;
cos_string_t date;
cos_string_t storage_class;
} cos_lifecycle_transition_t;
typedef struct {
int days;
} cos_lifecycle_abort_t;
typedef struct {
cos_list_t node;
cos_string_t id;
cos_string_t prefix;
cos_string_t status;
cos_lifecycle_expire_t expire;
cos_lifecycle_transition_t transition;
cos_lifecycle_abort_t abort;
} cos_lifecycle_rule_content_t;
typedef struct {
cos_string_t status;
} cos_versioning_content_t;
typedef struct {
cos_list_t node;
cos_string_t id;
cos_string_t allowed_origin;
cos_string_t allowed_method;
cos_string_t allowed_header;
cos_string_t expose_header;
int max_age_seconds;
} cos_cors_rule_content_t;
typedef struct {
cos_string_t role;
cos_list_t rule_list;
} cos_replication_params_t;
typedef struct {
cos_list_t node;
cos_string_t id;
cos_string_t status;
cos_string_t prefix;
cos_string_t dst_bucket;
cos_string_t storage_class;
} cos_replication_rule_content_t;
typedef struct {
cos_list_t node;
cos_string_t key;
} cos_object_key_t;
typedef struct {
char *suffix;
char *type;
} cos_content_type_t;
typedef struct {
int64_t part_size; // bytes, default 1MB
int32_t thread_num; // default 1
int enable_checkpoint; // default disable, false
cos_string_t checkpoint_path; // dafault ./filepath.ucp or ./filepath.dcp
} cos_resumable_clt_params_t;
typedef struct {
int days;
cos_string_t tier;
} cos_object_restore_params_t;
typedef struct {
cos_string_t type;
int32_t frag_duration;
int32_t frag_count;
cos_string_t play_list_name;
}cos_live_channel_target_t;
typedef struct {
cos_string_t name;
cos_string_t description;
cos_string_t status;
cos_live_channel_target_t target;
} cos_live_channel_configuration_t;
typedef struct {
cos_list_t node;
cos_string_t publish_url;
} cos_live_channel_publish_url_t;
typedef struct {
cos_list_t node;
cos_string_t play_url;
} cos_live_channel_play_url_t;
typedef struct {
int32_t width;
int32_t height;
int32_t frame_rate;
int32_t band_width;
cos_string_t codec;
} cos_video_stat_t;
typedef struct {
int32_t band_width;
int32_t sample_rate;
cos_string_t codec;
} cos_audio_stat_t;
typedef struct {
cos_string_t pushflow_status;
cos_string_t connected_time;
cos_string_t remote_addr;
cos_video_stat_t video_stat;
cos_audio_stat_t audio_stat;
} cos_live_channel_stat_t;
typedef struct {
cos_list_t node;
cos_string_t name;
cos_string_t description;
cos_string_t status;
cos_string_t last_modified;
cos_list_t publish_url_list;
cos_list_t play_url_list;
} cos_live_channel_content_t;
typedef struct {
cos_string_t prefix;
cos_string_t marker;
int max_keys;
int truncated;
cos_string_t next_marker;
cos_list_t live_channel_list;
} cos_list_live_channel_params_t;
typedef struct {
cos_list_t node;
cos_string_t start_time;
cos_string_t end_time;
cos_string_t remote_addr;
} cos_live_record_content_t;
typedef struct {
cos_string_t index;
cos_string_t redirect_protocol;
cos_string_t error_document;
cos_list_t rule_list;
} cos_website_params_t;
typedef struct {
cos_list_t node;
cos_string_t condition_errcode;
cos_string_t condition_prefix;
cos_string_t redirect_protocol;
cos_string_t redirect_replace_key;
cos_string_t redirect_replace_key_prefix;
} cos_website_rule_content_t;
typedef struct {
cos_string_t status;
cos_string_t name;
cos_string_t type;
cos_string_t forced_replacement;
} cos_domain_params_t;
typedef struct {
cos_string_t target_bucket;
cos_string_t target_prefix;
} cos_logging_params_t;
typedef struct {
cos_string_t format;
cos_string_t account_id;
cos_string_t bucket;
cos_string_t prefix;
int encryption;
} cos_inventory_destination_t;
typedef struct {
cos_list_t node;
cos_string_t field;
} cos_inventory_optional_t;
typedef struct {
cos_list_t node;
cos_string_t id;
cos_string_t is_enabled;
cos_string_t frequency;
cos_string_t filter_prefix;
cos_string_t included_object_versions;
cos_inventory_destination_t destination;
cos_list_t fields;
} cos_inventory_params_t;
typedef struct {
cos_list_t inventorys;
int is_truncated;
cos_string_t continuation_token;
cos_string_t next_continuation_token;
} cos_list_inventory_params_t;
typedef struct {
cos_list_t node;
cos_string_t key;
cos_string_t value;
} cos_tagging_tag_t;
typedef struct {
cos_list_t node;
} cos_tagging_params_t;
typedef struct {
cos_string_t status;
int days;
} cos_intelligenttiering_params_t;
#define COS_AUTH_EXPIRE_DEFAULT 300
#endif

62
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_fstack.c vendored
View File

@ -1,62 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_fstack.h"
cos_array_header_t *cos_fstack_create(cos_pool_t *p, int size)
{
return apr_array_make(p, size, sizeof(cos_fstack_item_t));
}
void cos_fstack_push(cos_array_header_t *fstack, void *data, cos_func_u func, int order)
{
cos_fstack_item_t *item;
item = (cos_fstack_item_t*)apr_array_push(fstack);
item->data = data;
item->func = func;
item->order = order;
}
cos_fstack_item_t *cos_fstack_pop(cos_array_header_t *fstack)
{
cos_fstack_item_t *item;
item = (cos_fstack_item_t*)apr_array_pop(fstack);
if (item == NULL) {
return NULL;
}
switch (item->order) {
case 1:
item->func.func1(item->data);
break;
case 2:
item->func.func2();
break;
case 3:
item->func.func3(item->data);
break;
case 4:
item->func.func4();
break;
default:
break;
}
return item;
}
void cos_fstack_destory(cos_array_header_t *fstack)
{
while (cos_fstack_pop(fstack) != NULL);
}

49
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_fstack.h vendored
View File

@ -1,49 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_FSTACK_H
#define LIBCOS_FSTACK_H
#include "cos_sys_define.h"
COS_CPP_START
typedef void (*cos_func1_pt)(void*);
typedef void (*cos_func2_pt)();
typedef int (*cos_func3_pt)(void*);
typedef int (*cos_func4_pt)();
typedef union cos_func_u {
cos_func1_pt func1;
cos_func2_pt func2;
cos_func3_pt func3;
cos_func4_pt func4;
} cos_func_u;
typedef struct cos_fstack_item_t {
void *data;
cos_func_u func;
int order;
} cos_fstack_item_t;
cos_array_header_t *cos_fstack_create(cos_pool_t *p, int size);
cos_fstack_item_t *cos_fstack_pop(cos_array_header_t *fstack);
void cos_fstack_destory(cos_array_header_t *fstack);
void cos_fstack_push(cos_array_header_t *fstack, void *data, cos_func_u func, int order);
COS_CPP_END
#endif

375
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_http_io.c vendored
View File

@ -1,375 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_log.h"
#include "cos_http_io.h"
#include "cos_sys_define.h"
#include <apr_thread_mutex.h>
#include <apr_file_io.h>
cos_pool_t *cos_global_pool = NULL;
apr_file_t *cos_stderr_file = NULL;
cos_http_request_options_t *cos_default_http_request_options = NULL;
cos_http_transport_options_t *cos_default_http_transport_options = NULL;
cos_http_transport_create_pt cos_http_transport_create = cos_curl_http_transport_create;
cos_http_transport_perform_pt cos_http_transport_perform = cos_curl_http_transport_perform;
static apr_thread_mutex_t* requestStackMutexG = NULL;
apr_thread_mutex_t* downloadMutex = NULL;
static CURL *requestStackG[COS_REQUEST_STACK_SIZE];
static int requestStackCountG;
static char cos_user_agent[256];
static cos_http_transport_options_t *cos_http_transport_options_create(cos_pool_t *p);
CURL *cos_request_get()
{
CURL *request = NULL;
apr_thread_mutex_lock(requestStackMutexG);
if (requestStackCountG > 0) {
request = requestStackG[--requestStackCountG];
}
apr_thread_mutex_unlock(requestStackMutexG);
// If we got one, deinitialize it for re-use
if (request) {
curl_easy_reset(request);
}
else {
request = curl_easy_init();
}
return request;
}
void request_release(CURL *request)
{
apr_thread_mutex_lock(requestStackMutexG);
// If the request stack is full, destroy this one
// else put this one at the front of the request stack; we do this because
// we want the most-recently-used curl handle to be re-used on the next
// request, to maximize our chances of re-using a TCP connection before it
// times out
if (requestStackCountG == COS_REQUEST_STACK_SIZE) {
apr_thread_mutex_unlock(requestStackMutexG);
curl_easy_cleanup(request);
}
else {
requestStackG[requestStackCountG++] = request;
apr_thread_mutex_unlock(requestStackMutexG);
}
}
void cos_set_default_request_options(cos_http_request_options_t *op)
{
cos_default_http_request_options = op;
}
void cos_set_default_transport_options(cos_http_transport_options_t *op)
{
cos_default_http_transport_options = op;
}
cos_http_request_options_t *cos_http_request_options_create(cos_pool_t *p)
{
cos_http_request_options_t *options;
options = (cos_http_request_options_t *)cos_pcalloc(p, sizeof(cos_http_request_options_t));
options->speed_limit = COS_MIN_SPEED_LIMIT;
options->speed_time = COS_MIN_SPEED_TIME;
options->connect_timeout = COS_CONNECT_TIMEOUT;
options->dns_cache_timeout = COS_DNS_CACHE_TIMOUT;
options->max_memory_size = COS_MAX_MEMORY_SIZE;
options->enable_crc = COS_TRUE;
options->enable_md5 = COS_TRUE;
options->proxy_auth = NULL;
options->proxy_host = NULL;
return options;
}
cos_http_transport_options_t *cos_http_transport_options_create(cos_pool_t *p)
{
return (cos_http_transport_options_t *)cos_pcalloc(p, sizeof(cos_http_transport_options_t));
}
cos_http_controller_t *cos_http_controller_create(cos_pool_t *p, int owner)
{
int s;
cos_http_controller_t *ctl;
if(p == NULL) {
if ((s = cos_pool_create(&p, NULL)) != APR_SUCCESS) {
cos_fatal_log("cos_pool_create failure.");
return NULL;
}
}
ctl = (cos_http_controller_t *)cos_pcalloc(p, sizeof(cos_http_controller_ex_t));
ctl->pool = p;
ctl->owner = owner;
ctl->options = cos_default_http_request_options;
return ctl;
}
cos_http_request_t *cos_http_request_create(cos_pool_t *p)
{
cos_http_request_t *req;
req = (cos_http_request_t *)cos_pcalloc(p, sizeof(cos_http_request_t));
req->method = HTTP_GET;
req->headers = cos_table_make(p, 5);
req->query_params = cos_table_make(p, 3);
cos_list_init(&req->body);
req->type = BODY_IN_MEMORY;
req->body_len = 0;
req->pool = p;
req->read_body = cos_read_http_body_memory;
return req;
}
cos_http_response_t *cos_http_response_create(cos_pool_t *p)
{
cos_http_response_t *resp;
resp = (cos_http_response_t *)cos_pcalloc(p, sizeof(cos_http_response_t));
resp->status = -1;
resp->headers = cos_table_make(p, 10);
cos_list_init(&resp->body);
resp->type = BODY_IN_MEMORY;
resp->body_len = 0;
resp->pool = p;
resp->write_body = cos_write_http_body_memory;
return resp;
}
int cos_read_http_body_memory(cos_http_request_t *req, char *buffer, int len)
{
int wsize;
int bytes = 0;
cos_buf_t *b;
cos_buf_t *n;
cos_list_for_each_entry_safe(cos_buf_t, b, n, &req->body, node) {
wsize = cos_buf_size(b);
if (wsize == 0) {
cos_list_del(&b->node);
continue;
}
wsize = cos_min(len - bytes, wsize);
if (wsize == 0) {
break;
}
memcpy(buffer + bytes, b->pos, wsize);
b->pos += wsize;
bytes += wsize;
if (b->pos == b->last) {
cos_list_del(&b->node);
}
}
return bytes;
}
int cos_read_http_body_file(cos_http_request_t *req, char *buffer, int len)
{
int s;
char buf[256];
apr_size_t nbytes = len;
apr_size_t bytes_left;
if (req->file_buf == NULL || req->file_buf->file == NULL) {
cos_error_log("request body arg invalid file_buf NULL.");
return COSE_INVALID_ARGUMENT;
}
if (req->file_buf->file_pos >= req->file_buf->file_last) {
cos_debug_log("file read finish.");
return 0;
}
bytes_left = (apr_size_t)(req->file_buf->file_last - req->file_buf->file_pos);
if (nbytes > bytes_left) {
nbytes = bytes_left;
}
if ((s = apr_file_read(req->file_buf->file, buffer, &nbytes)) != APR_SUCCESS) {
cos_error_log("apr_file_read filure, code:%d %s.", s, apr_strerror(s, buf, sizeof(buf)));
return COSE_FILE_READ_ERROR;
}
req->file_buf->file_pos += nbytes;
return nbytes;
}
int cos_write_http_body_memory(cos_http_response_t *resp, const char *buffer, int len)
{
cos_buf_t *b;
b = cos_create_buf(resp->pool, len);
memcpy(b->pos, buffer, len);
b->last += len;
cos_list_add_tail(&b->node, &resp->body);
resp->body_len += len;
return len;
}
int cos_write_http_body_file(cos_http_response_t *resp, const char *buffer, int len)
{
int elen;
int s;
char buf[256];
apr_size_t nbytes = len;
if (resp->file_buf == NULL) {
resp->file_buf = cos_create_file_buf(resp->pool);
}
if (resp->file_buf->file == NULL) {
if (resp->file_path == NULL) {
cos_error_log("resp body file arg NULL.");
return COSE_INVALID_ARGUMENT;
}
cos_trace_log("open file %s.", resp->file_path);
if ((elen = cos_open_file_for_write(resp->pool, resp->file_path, resp->file_buf)) != COSE_OK) {
return elen;
}
}
assert(resp->file_buf->file != NULL);
if ((s = apr_file_write(resp->file_buf->file, buffer, &nbytes)) != APR_SUCCESS) {
cos_error_log("apr_file_write fialure, code:%d %s.", s, apr_strerror(s, buf, sizeof(buf)));
return COSE_FILE_WRITE_ERROR;
}
resp->file_buf->file_last += nbytes;
resp->body_len += nbytes;
return nbytes;
}
int cos_write_http_body_file_part(cos_http_response_t *resp, const char *buffer, int len)
{
int s;
char buf[256];
apr_size_t nbytes = len;
if (resp->file_buf == NULL) {
cos_error_log("file_buf is NULL.");
return COSE_INVALID_ARGUMENT;
}
assert(resp->file_buf->file != NULL);
if ((s = apr_file_write(resp->file_buf->file, buffer, &nbytes)) != APR_SUCCESS) {
cos_error_log("apr_file_write fialure, code:%d %s.", s, apr_strerror(s, buf, sizeof(buf)));
return COSE_FILE_WRITE_ERROR;
}
resp->file_buf->file_last += nbytes;
resp->body_len += nbytes;
return nbytes;
}
int cos_http_io_initialize(const char *user_agent_info, int flags)
{
CURLcode ecode;
int s;
char buf[256];
cos_http_request_options_t *req_options;
cos_http_transport_options_t *trans_options;
if ((ecode = curl_global_init(CURL_GLOBAL_ALL &
~((flags & COS_INIT_WINSOCK) ? 0: CURL_GLOBAL_WIN32))) != CURLE_OK)
{
cos_error_log("curl_global_init failure, code:%d %s.\n", ecode, curl_easy_strerror(ecode));
return COSE_INTERNAL_ERROR;
}
if ((s = apr_initialize()) != APR_SUCCESS) {
cos_error_log("apr_initialize failue.\n");
return COSE_INTERNAL_ERROR;
}
if (!user_agent_info || !*user_agent_info) {
user_agent_info = "Unknown";
}
if ((s = cos_pool_create(&cos_global_pool, NULL)) != APR_SUCCESS) {
cos_error_log("cos_pool_create failure, code:%d %s.\n", s, apr_strerror(s, buf, sizeof(buf)));
return COSE_INTERNAL_ERROR;
}
if ((s = apr_thread_mutex_create(&requestStackMutexG, APR_THREAD_MUTEX_DEFAULT, cos_global_pool)) != APR_SUCCESS) {
cos_error_log("apr_thread_mutex_create failure, code:%d %s.\n", s, apr_strerror(s, buf, sizeof(buf)));
return COSE_INTERNAL_ERROR;
}
requestStackCountG = 0;
if ((s = apr_thread_mutex_create(&downloadMutex, APR_THREAD_MUTEX_DEFAULT, cos_global_pool)) != APR_SUCCESS) {
cos_error_log("apr_thread_mutex_create failure, code:%d %s.\n", s, apr_strerror(s, buf, sizeof(buf)));
return COSE_INTERNAL_ERROR;
}
apr_snprintf(cos_user_agent, sizeof(cos_user_agent)-1, "%s(Compatible %s)",
COS_VER, user_agent_info);
req_options = cos_http_request_options_create(cos_global_pool);
trans_options = cos_http_transport_options_create(cos_global_pool);
trans_options->user_agent = cos_user_agent;
cos_set_default_request_options(req_options);
cos_set_default_transport_options(trans_options);
return COSE_OK;
}
void cos_http_io_deinitialize()
{
apr_thread_mutex_destroy(requestStackMutexG);
apr_thread_mutex_destroy(downloadMutex);
while (requestStackCountG--) {
curl_easy_cleanup(requestStackG[requestStackCountG]);
}
if (cos_stderr_file != NULL) {
apr_file_close(cos_stderr_file);
cos_stderr_file = NULL;
}
if (cos_global_pool != NULL) {
cos_pool_destroy(cos_global_pool);
cos_global_pool = NULL;
}
apr_terminate();
}
int cos_http_send_request(cos_http_controller_t *ctl, cos_http_request_t *req, cos_http_response_t *resp)
{
cos_http_transport_t *t;
t = cos_http_transport_create(ctl->pool);
t->req = req;
t->resp = resp;
t->controller = (cos_http_controller_ex_t *)ctl;
return cos_http_transport_perform(t);
}

69
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_http_io.h vendored
View File

@ -1,69 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_HTTP_IO_H
#define LIBCOS_HTTP_IO_H
#include "cos_transport.h"
#include "cos_define.h"
COS_CPP_START
cos_http_controller_t *cos_http_controller_create(cos_pool_t *p, int owner);
/* http io error message*/
static APR_INLINE const char *cos_http_controller_get_reason(cos_http_controller_t *ctl)
{
cos_http_controller_ex_t *ctle = (cos_http_controller_ex_t *)ctl;
return ctle->reason;
}
CURL *cos_request_get();
void request_release(CURL *request);
int cos_http_io_initialize(const char *user_agent_info, int flag);
void cos_http_io_deinitialize();
int cos_http_send_request(cos_http_controller_t *ctl, cos_http_request_t *req, cos_http_response_t *resp);
void cos_set_default_request_options(cos_http_request_options_t *op);
void cos_set_default_transport_options(cos_http_transport_options_t *op);
cos_http_request_options_t *cos_http_request_options_create(cos_pool_t *p);
cos_http_request_t *cos_http_request_create(cos_pool_t *p);
cos_http_response_t *cos_http_response_create(cos_pool_t *p);
int cos_read_http_body_memory(cos_http_request_t *req, char *buffer, int len);
int cos_write_http_body_memory(cos_http_response_t *resp, const char *buffer, int len);
int cos_read_http_body_file(cos_http_request_t *req, char *buffer, int len);
int cos_write_http_body_file(cos_http_response_t *resp, const char *buffer, int len);
int cos_write_http_body_file_part(cos_http_response_t *resp, const char *buffer, int len);
typedef cos_http_transport_t *(*cos_http_transport_create_pt)(cos_pool_t *p);
typedef int (*cos_http_transport_perform_pt)(cos_http_transport_t *t);
extern cos_pool_t *cos_global_pool;
extern apr_file_t *cos_stderr_file;
extern cos_http_request_options_t *cos_default_http_request_options;
extern cos_http_transport_options_t *cos_default_http_transport_options;
extern cos_http_transport_create_pt cos_http_transport_create;
extern cos_http_transport_perform_pt cos_http_transport_perform;
COS_CPP_END
#endif

113
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_list.h vendored
View File

@ -1,113 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_LIST_H
#define LIBCOS_LIST_H
#include <apr_general.h>
// from kernel list
typedef struct cos_list_s cos_list_t;
struct cos_list_s {
cos_list_t *next, *prev;
};
#define cos_list_head_init(name) {&(name), &(name)}
#define cos_list_init(ptr) do { \
(ptr)->next = (ptr); \
(ptr)->prev = (ptr); \
} while (0)
static APR_INLINE void __cos_list_add(cos_list_t *list, cos_list_t *prev, cos_list_t *next)
{
next->prev = list;
list->next = next;
list->prev = prev;
prev->next = list;
}
// list head to add it before
static APR_INLINE void cos_list_add_tail(cos_list_t *list, cos_list_t *head)
{
__cos_list_add(list, head->prev, head);
}
static APR_INLINE void __cos_list_del(cos_list_t *prev, cos_list_t *next)
{
next->prev = prev;
prev->next = next;
}
// deletes entry from list
static APR_INLINE void cos_list_del(cos_list_t *entry)
{
__cos_list_del(entry->prev, entry->next);
cos_list_init(entry);
}
// tests whether a list is empty
static APR_INLINE int cos_list_empty(const cos_list_t *head)
{
return (head->next == head);
}
// move list to new_list
static APR_INLINE void cos_list_movelist(cos_list_t *list, cos_list_t *new_list)
{
if (!cos_list_empty(list)) {
new_list->prev = list->prev;
new_list->next = list->next;
new_list->prev->next = new_list;
new_list->next->prev = new_list;
cos_list_init(list);
} else {
cos_list_init(new_list);
}
}
// get last
#define cos_list_get_last(list, type, member) \
cos_list_empty(list) ? NULL : cos_list_entry((list)->prev, type, member)
// get first
#define cos_list_get_first(list, type, member) \
cos_list_empty(list) ? NULL : cos_list_entry((list)->next, type, member)
#define cos_list_entry(ptr, type, member) \
(type *)( (char *)ptr - APR_OFFSETOF(type, member) )
// traversing
#define cos_list_for_each_entry(postp, pos, head, member) \
for (pos = cos_list_entry((head)->next, postp, member); \
&pos->member != (head); \
pos = cos_list_entry(pos->member.next, postp, member))
#define cos_list_for_each_entry_reverse(postp, pos, head, member) \
for (pos = cos_list_entry((head)->prev, postp, member); \
&pos->member != (head); \
pos = cos_list_entry(pos->member.prev, postp, member))
#define cos_list_for_each_entry_safe(postp, pos, n, head, member) \
for (pos = cos_list_entry((head)->next, postp, member), \
n = cos_list_entry(pos->member.next, postp, member); \
&pos->member != (head); \
pos = n, n = cos_list_entry(n->member.next, postp, member))
#define cos_list_for_each_entry_safe_reverse(postp, pos, n, head, member) \
for (pos = cos_list_entry((head)->prev, postp, member), \
n = cos_list_entry(pos->member.prev, postp, member); \
&pos->member != (head); \
pos = n, n = cos_list_entry(n->member.prev, postp, member))
#endif

85
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_log.c vendored
View File

@ -1,85 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_log.h"
#include "apr_portable.h"
cos_log_print_pt cos_log_print = cos_log_print_default;
cos_log_format_pt cos_log_format = cos_log_format_default;
cos_log_level_e cos_log_level = COS_LOG_WARN;
extern apr_file_t *cos_stderr_file;
void cos_log_set_print(cos_log_print_pt p)
{
cos_log_print = p;
}
void cos_log_set_format(cos_log_format_pt p)
{
cos_log_format = p;
}
void cos_log_set_level(cos_log_level_e level)
{
cos_log_level = level;
}
void cos_log_set_output(apr_file_t *output)
{
cos_stderr_file = output;
}
void cos_log_print_default(const char *message, int len)
{
if (cos_stderr_file == NULL) {
fprintf(stderr, "%s", message);
} else {
apr_size_t bnytes = len;
apr_file_write(cos_stderr_file, message, &bnytes);
}
}
void cos_log_format_default(int level,
const char *file,
int line,
const char *function,
const char *fmt, ...)
{
int len;
apr_time_t t;
int s;
apr_time_exp_t tm;
va_list args;
char buffer[4096];
t = apr_time_now();
if ((s = apr_time_exp_lt(&tm, t)) != APR_SUCCESS) {
return;
}
len = apr_snprintf(buffer, 4090, "[%04d-%02d-%02d %02d:%02d:%02d.%03d] %" APR_INT64_T_FMT " %s:%d ",
tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
tm.tm_hour, tm.tm_min, tm.tm_sec, tm.tm_usec/1000,
(int64_t)apr_os_thread_current(), file, line);
va_start(args, fmt);
len += vsnprintf(buffer + len, 4090 - len, fmt, args);
va_end(args);
while (buffer[len -1] == '\n') len--;
buffer[len++] = '\n';
buffer[len] = '\0';
cos_log_print(buffer, len);
}

91
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_log.h vendored
View File

@ -1,91 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_LOG_H
#define LIBCOS_LOG_H
#include "cos_sys_define.h"
COS_CPP_START
typedef void (*cos_log_print_pt)(const char *message, int len);
typedef void (*cos_log_format_pt)(int level,
const char *file,
int line,
const char *function,
const char *fmt, ...)
__attribute__ ((__format__ (__printf__, 5, 6)));
void cos_log_set_print(cos_log_print_pt p);
void cos_log_set_format(cos_log_format_pt p);
typedef enum {
COS_LOG_OFF = 1,
COS_LOG_FATAL,
COS_LOG_ERROR,
COS_LOG_WARN,
COS_LOG_INFO,
COS_LOG_DEBUG,
COS_LOG_TRACE,
COS_LOG_ALL
} cos_log_level_e;
#ifdef WIN32
#define cos_fatal_log(format, ...) if(cos_log_level>=COS_LOG_FATAL) \
cos_log_format(COS_LOG_FATAL, __FILE__, __LINE__, __FUNCTION__, format, ##__VA_ARGS__)
#define cos_error_log(format, ...) if(cos_log_level>=COS_LOG_ERROR) \
cos_log_format(COS_LOG_ERROR, __FILE__, __LINE__, __FUNCTION__, format, ##__VA_ARGS__)
#define cos_warn_log(format, ...) if(cos_log_level>=COS_LOG_WARN) \
cos_log_format(COS_LOG_WARN, __FILE__, __LINE__, __FUNCTION__, format, ##__VA_ARGS__)
#define cos_info_log(format, ...) if(cos_log_level>=COS_LOG_INFO) \
cos_log_format(COS_LOG_INFO, __FILE__, __LINE__, __FUNCTION__, format, ##__VA_ARGS__)
#define cos_debug_log(format, ...) if(cos_log_level>=COS_LOG_DEBUG) \
cos_log_format(COS_LOG_DEBUG, __FILE__, __LINE__, __FUNCTION__, format, ##__VA_ARGS__)
#define cos_trace_log(format, ...) if(cos_log_level>=COS_LOG_TRACE) \
cos_log_format(COS_LOG_TRACE, __FILE__, __LINE__, __FUNCTION__, format, ##__VA_ARGS__)
#else
#define cos_fatal_log(format, args...) if(cos_log_level>=COS_LOG_FATAL) \
cos_log_format(COS_LOG_FATAL, __FILE__, __LINE__, __FUNCTION__, format, ## args)
#define cos_error_log(format, args...) if(cos_log_level>=COS_LOG_ERROR) \
cos_log_format(COS_LOG_ERROR, __FILE__, __LINE__, __FUNCTION__, format, ## args)
#define cos_warn_log(format, args...) if(cos_log_level>=COS_LOG_WARN) \
cos_log_format(COS_LOG_WARN, __FILE__, __LINE__, __FUNCTION__, format, ## args)
#define cos_info_log(format, args...) if(cos_log_level>=COS_LOG_INFO) \
cos_log_format(COS_LOG_INFO, __FILE__, __LINE__, __FUNCTION__, format, ## args)
#define cos_debug_log(format, args...) if(cos_log_level>=COS_LOG_DEBUG) \
cos_log_format(COS_LOG_DEBUG, __FILE__, __LINE__, __FUNCTION__, format, ## args)
#define cos_trace_log(format, args...) if(cos_log_level>=COS_LOG_TRACE) \
cos_log_format(COS_LOG_TRACE, __FILE__, __LINE__, __FUNCTION__, format, ## args)
#endif
void cos_log_set_level(cos_log_level_e level);
void cos_log_set_output(apr_file_t *output);
void cos_log_print_default(const char *message, int len);
void cos_log_format_default(int level,
const char *file,
int line,
const char *function,
const char *fmt, ...)
__attribute__ ((__format__ (__printf__, 5, 6)));
extern cos_log_level_e cos_log_level;
extern cos_log_format_pt cos_log_format;
extern cos_log_format_pt cos_log_format;
COS_CPP_END
#endif

726
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_multipart.c vendored
View File

@ -1,726 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_log.h"
#include "cos_sys_define.h"
#include "cos_sys_util.h"
#include "cos_string.h"
#include "cos_status.h"
#include "cos_auth.h"
#include "cos_utility.h"
#include "cos_xml.h"
#include "cos_api.h"
cos_status_t *cos_init_multipart_upload(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
cos_string_t *upload_id,
cos_table_t *headers,
cos_table_t **resp_headers)
{
int res = COSE_OK;
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
cos_table_t *query_params = NULL;
//init query_params
query_params = cos_table_create_if_null(options, query_params, 1);
apr_table_add(query_params, COS_UPLOADS, "");
//init headers
headers = cos_table_create_if_null(options, headers, 1);
set_content_type(NULL, object->data, headers);
cos_set_multipart_content_type(headers);
cos_init_object_request(options, bucket, object, HTTP_POST,
&req, query_params, headers, NULL, 0, &resp);
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
if (!cos_status_is_ok(s)) {
return s;
}
res = cos_upload_id_parse_from_body(options->pool, &resp->body, upload_id);
if (res != COSE_OK) {
cos_xml_error_status_set(s, res);
}
return s;
}
cos_status_t *cos_abort_multipart_upload(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
cos_string_t *upload_id,
cos_table_t **resp_headers)
{
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
cos_table_t *query_params = NULL;
cos_table_t *headers = NULL;
//init query_params
query_params = cos_table_create_if_null(options, query_params, 1);
apr_table_add(query_params, COS_UPLOAD_ID, upload_id->data);
//init headers
headers = cos_table_create_if_null(options, headers, 0);
cos_init_object_request(options, bucket, object, HTTP_DELETE,
&req, query_params, headers, NULL, 0, &resp);
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
return s;
}
cos_status_t *cos_list_upload_part(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
const cos_string_t *upload_id,
cos_list_upload_part_params_t *params,
cos_table_t **resp_headers)
{
int res = COSE_OK;
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
cos_table_t *query_params = NULL;
cos_table_t *headers = NULL;
//init query_params
query_params = cos_table_create_if_null(options, query_params, 4);
apr_table_add(query_params, COS_UPLOAD_ID, upload_id->data);
if (!cos_is_null_string(&params->encoding_type)) apr_table_add(query_params, COS_ENCODING_TYPE, params->encoding_type.data);
cos_table_add_int(query_params, COS_MAX_PARTS, params->max_ret);
if (!cos_is_null_string(&params->part_number_marker)) apr_table_add(query_params, COS_PART_NUMBER_MARKER, params->part_number_marker.data);
//init headers
headers = cos_table_create_if_null(options, headers, 0);
cos_init_object_request(options, bucket, object, HTTP_GET,
&req, query_params, headers, NULL, 0, &resp);
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
if (!cos_status_is_ok(s)) {
return s;
}
res = cos_list_parts_parse_from_body(options->pool, &resp->body,
&params->part_list, &params->next_part_number_marker,
&params->truncated);
if (res != COSE_OK) {
cos_xml_error_status_set(s, res);
}
return s;
}
cos_status_t *cos_list_multipart_upload(const cos_request_options_t *options,
const cos_string_t *bucket,
cos_list_multipart_upload_params_t *params,
cos_table_t **resp_headers)
{
int res = COSE_OK;
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
cos_table_t *query_params = NULL;
cos_table_t *headers = NULL;
//init query_params
query_params = cos_table_create_if_null(options, query_params, 7);
apr_table_add(query_params, COS_UPLOADS, "");
if (!cos_is_null_string(&params->encoding_type)) apr_table_add(query_params, COS_ENCODING_TYPE, params->encoding_type.data);
if (!cos_is_null_string(&params->prefix)) apr_table_add(query_params, COS_PREFIX, params->prefix.data);
if (!cos_is_null_string(&params->delimiter)) apr_table_add(query_params, COS_DELIMITER, params->delimiter.data);
if (!cos_is_null_string(&params->key_marker)) apr_table_add(query_params, COS_KEY_MARKER, params->key_marker.data);
if (!cos_is_null_string(&params->upload_id_marker)) apr_table_add(query_params, COS_UPLOAD_ID_MARKER, params->upload_id_marker.data);
cos_table_add_int(query_params, COS_MAX_UPLOADS, params->max_ret);
//init headers
headers = cos_table_create_if_null(options, headers, 0);
cos_init_bucket_request(options, bucket, HTTP_GET, &req,
query_params, headers, &resp);
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
if (!cos_status_is_ok(s)) {
return s;
}
res = cos_list_multipart_uploads_parse_from_body(options->pool, &resp->body,
&params->upload_list, &params->next_key_marker,
&params->next_upload_id_marker, &params->truncated);
if (res != COSE_OK) {
cos_xml_error_status_set(s, res);
}
return s;
}
cos_status_t *cos_complete_multipart_upload(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
const cos_string_t *upload_id,
cos_list_t *part_list,
cos_table_t *headers,
cos_table_t **resp_headers)
{
return cos_do_complete_multipart_upload(options, bucket, object, upload_id, part_list,
headers, NULL, resp_headers, NULL);
}
cos_status_t *cos_do_complete_multipart_upload(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
const cos_string_t *upload_id,
cos_list_t *part_list,
cos_table_t *headers,
cos_table_t *params,
cos_table_t **resp_headers,
cos_list_t *resp_body)
{
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
apr_table_t *query_params = NULL;
cos_list_t body;
//init query_params
query_params = cos_table_create_if_null(options, params, 1);
apr_table_add(query_params, COS_UPLOAD_ID, upload_id->data);
//init headers
headers = cos_table_create_if_null(options, headers, 1);
set_content_type(NULL, object->data, headers);
cos_set_multipart_content_type(headers);
//apr_table_add(headers, COS_REPLACE_OBJECT_META, COS_YES);
cos_init_object_request(options, bucket, object, HTTP_POST,
&req, query_params, headers, NULL, 0, &resp);
build_complete_multipart_upload_body(options->pool, part_list, &body);
cos_write_request_body_from_buffer(&body, req);
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
cos_fill_read_response_body(resp, resp_body);
return s;
}
cos_status_t *cos_upload_part_from_buffer(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
const cos_string_t *upload_id,
int part_num,
cos_list_t *buffer,
cos_table_t **resp_headers)
{
return cos_do_upload_part_from_buffer(options, bucket, object, upload_id, part_num,
buffer, NULL, NULL, NULL, resp_headers, NULL);
}
cos_status_t *cos_do_upload_part_from_buffer(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
const cos_string_t *upload_id,
int part_num,
cos_list_t *buffer,
cos_progress_callback progress_callback,
cos_table_t *headers,
cos_table_t *params,
cos_table_t **resp_headers,
cos_list_t *resp_body)
{
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
cos_table_t *query_params = NULL;
//init query_params
query_params = cos_table_create_if_null(options, params, 2);
apr_table_add(query_params, COS_UPLOAD_ID, upload_id->data);
cos_table_add_int(query_params, COS_PARTNUMBER, part_num);
//init headers
headers = cos_table_create_if_null(options, headers, 0);
cos_add_content_md5_from_buffer(options, buffer, headers);
cos_init_object_request(options, bucket, object, HTTP_PUT, &req, query_params,
headers, progress_callback, 0, &resp);
cos_write_request_body_from_buffer(buffer, req);
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
cos_fill_read_response_body(resp, resp_body);
if (is_enable_crc(options) && has_crc_in_response(resp)) {
cos_check_crc_consistent(req->crc64, resp->headers, s);
}
return s;
}
cos_status_t *cos_upload_part_from_file(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
const cos_string_t *upload_id,
int part_num,
cos_upload_file_t *upload_file,
cos_table_t **resp_headers)
{
return cos_do_upload_part_from_file(options, bucket, object, upload_id, part_num,
upload_file, NULL, NULL, NULL, resp_headers, NULL);
}
cos_status_t *cos_do_upload_part_from_file(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
const cos_string_t *upload_id,
int part_num,
cos_upload_file_t *upload_file,
cos_progress_callback progress_callback,
cos_table_t *headers,
cos_table_t *params,
cos_table_t **resp_headers,
cos_list_t *resp_body)
{
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
cos_table_t *query_params = NULL;
int res = COSE_OK;
s = cos_status_create(options->pool);
//init query_params
query_params = cos_table_create_if_null(options, params, 2);
apr_table_add(query_params, COS_UPLOAD_ID, upload_id->data);
cos_table_add_int(query_params, COS_PARTNUMBER, part_num);
//init headers
headers = cos_table_create_if_null(options, headers, 0);
cos_add_content_md5_from_file_range(options, upload_file, headers);
cos_init_object_request(options, bucket, object, HTTP_PUT, &req,
query_params, headers, progress_callback, 0, &resp);
res = cos_write_request_body_from_upload_file(options->pool, upload_file, req);
if (res != COSE_OK) {
cos_file_error_status_set(s, res);
return s;
}
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
cos_fill_read_response_body(resp, resp_body);
if (is_enable_crc(options) && has_crc_in_response(resp)) {
cos_check_crc_consistent(req->crc64, resp->headers, s);
}
return s;
}
cos_status_t *cos_upload_part_copy(const cos_request_options_t *options,
cos_upload_part_copy_params_t *params,
cos_table_t *headers,
cos_table_t **resp_headers)
{
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
cos_table_t *query_params = NULL;
char *copy_source_range = NULL;
int res;
s = cos_status_create(options->pool);
//init query_params
query_params = cos_table_create_if_null(options, query_params, 2);
apr_table_add(query_params, COS_UPLOAD_ID, params->upload_id.data);
cos_table_add_int(query_params, COS_PARTNUMBER, params->part_num);
//init headers
headers = cos_table_create_if_null(options, headers, 2);
apr_table_add(headers, COS_COPY_SOURCE, params->copy_source.data);
if (-1 != params->range_start && -1 != params->range_end) {
copy_source_range = apr_psprintf(options->pool,
"bytes=%" APR_INT64_T_FMT "-%" APR_INT64_T_FMT,
params->range_start, params->range_end);
apr_table_add(headers, COS_COPY_SOURCE_RANGE, copy_source_range);
}
cos_init_object_request(options, &params->dest_bucket, &params->dest_object,
HTTP_PUT, &req, query_params, headers, NULL, 0, &resp);
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
if (!cos_status_is_ok(s)) {
return s;
}
if (NULL != params->rsp_content) {
res = cos_copy_object_parse_from_body(options->pool, &resp->body, params->rsp_content);
if (res != COSE_OK) {
cos_xml_error_status_set(s, res);
}
}
return s;
}
cos_status_t *cos_get_sorted_uploaded_part(cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
const cos_string_t *upload_id,
cos_list_t *complete_part_list,
int *part_count)
{
cos_pool_t *subpool = NULL;
cos_pool_t *parent_pool = NULL;
cos_status_t *s = NULL;
cos_status_t *ret = NULL;
cos_upload_part_t *part_arr = NULL;
int part_index = 0;
int index = 0;
int uploaded_part_count = 0;
cos_list_upload_part_params_t *params = NULL;
cos_list_part_content_t *part_content = NULL;
cos_complete_part_content_t *complete_content = NULL;
cos_table_t *list_part_resp_headers = NULL;
char *part_num_str = NULL;
parent_pool = options->pool;
part_arr = cos_palloc(parent_pool, COS_MAX_PART_NUM * sizeof(cos_upload_part_t *));
params = cos_create_list_upload_part_params(parent_pool);
while (params->truncated) {
cos_pool_create(&subpool, parent_pool);
options->pool = subpool;
s = cos_list_upload_part(options, bucket, object,
upload_id, params, &list_part_resp_headers);
if (!cos_status_is_ok(s)) {
ret = cos_status_dup(parent_pool, s);
cos_pool_destroy(subpool);
options->pool = parent_pool;
return ret;
}
if (!params->truncated) {
ret = cos_status_dup(parent_pool, s);
}
cos_list_for_each_entry(cos_list_part_content_t, part_content, &params->part_list, node) {
cos_upload_part_t upload_part;
upload_part.etag = part_content->etag.data;
upload_part.part_num = atoi(part_content->part_number.data);
part_arr[part_index++] = upload_part;
uploaded_part_count++;
}
cos_list_init(&params->part_list);
if (params->next_part_number_marker.data != NULL) {
cos_str_set(&params->part_number_marker,
params->next_part_number_marker.data);
}
//sort multipart upload part content
qsort(part_arr, uploaded_part_count, sizeof(part_arr[0]), part_sort_cmp);
for (index = 0; index < part_index; ++index) {
complete_content = cos_create_complete_part_content(parent_pool);
part_num_str = apr_psprintf(parent_pool, "%d", part_arr[index].part_num);
cos_str_set(&complete_content->part_number, part_num_str);
cos_str_set(&complete_content->etag, part_arr[index].etag);
cos_list_add_tail(&complete_content->node, complete_part_list);
}
part_index = 0;
cos_pool_destroy(subpool);
}
*part_count = uploaded_part_count;
options->pool = parent_pool;
return ret;
}
cos_status_t *cos_upload_file(cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
cos_string_t *upload_id,
cos_string_t *filepath,
int64_t part_size,
cos_table_t *headers)
{
cos_pool_t *subpool = NULL;
cos_pool_t *parent_pool = NULL;
int64_t start_pos;
int64_t end_pos;
int part_num;
int part_count = 0;
int res = COSE_OK;
cos_status_t *s = NULL;
cos_status_t *ret = NULL;
cos_file_buf_t *fb = NULL;
cos_upload_file_t *upload_file = NULL;
cos_table_t *upload_part_resp_headers = NULL;
char *part_num_str = NULL;
char *etag = NULL;
cos_list_t complete_part_list;
cos_complete_part_content_t *complete_content = NULL;
cos_table_t *complete_resp_headers = NULL;
cos_list_init(&complete_part_list);
parent_pool = options->pool;
//get upload_id and uploaded part
if (NULL == upload_id->data) {
cos_table_t *init_multipart_headers = NULL;
cos_table_t *init_multipart_resp_headers = NULL;
init_multipart_headers = cos_table_make(parent_pool, 0);
s = cos_init_multipart_upload(options, bucket, object,
upload_id, init_multipart_headers, &init_multipart_resp_headers);
if (!cos_status_is_ok(s)) {
ret = cos_status_dup(parent_pool, s);
return ret;
}
} else {
s = cos_get_sorted_uploaded_part(options, bucket, object, upload_id,
&complete_part_list, &part_count);
if (!cos_status_is_ok(s)) {
ret = cos_status_dup(parent_pool, s);
return ret;
}
}
//get part size
fb = cos_create_file_buf(parent_pool);
res = cos_open_file_for_read(parent_pool, filepath->data, fb);
if (res != COSE_OK) {
s = cos_status_create(parent_pool);
cos_file_error_status_set(s, res);
options->pool = parent_pool;
return s;
}
cos_get_part_size(fb->file_last, &part_size);
//upload part from file
upload_file = cos_create_upload_file(parent_pool);
cos_str_set(&upload_file->filename, filepath->data);
start_pos = part_size * part_count;
end_pos = start_pos + part_size;
part_num = part_count + 1;
while (1) {
cos_pool_create(&subpool, parent_pool);
options->pool = subpool;
upload_file->file_pos = start_pos;
upload_file->file_last = end_pos;
s = cos_upload_part_from_file(options, bucket, object, upload_id,
part_num, upload_file, &upload_part_resp_headers);
if (!cos_status_is_ok(s)) {
ret = cos_status_dup(parent_pool, s);
cos_pool_destroy(subpool);
options->pool = parent_pool;
return ret;
}
complete_content = cos_create_complete_part_content(parent_pool);
part_num_str = apr_psprintf(parent_pool, "%d", part_num);
cos_str_set(&complete_content->part_number, part_num_str);
etag = apr_pstrdup(parent_pool,
(char*)apr_table_get(upload_part_resp_headers, "ETag"));
cos_str_set(&complete_content->etag, etag);
cos_list_add_tail(&complete_content->node, &complete_part_list);
cos_pool_destroy(subpool);
if (end_pos >= fb->file_last) {
break;
}
start_pos += part_size;
end_pos += part_size;
if (end_pos > fb->file_last)
end_pos = fb->file_last;
part_num += 1;
}
//complete multipart
cos_pool_create(&subpool, parent_pool);
options->pool = subpool;
headers = cos_table_create_if_null(options, headers, 0);
s = cos_complete_multipart_upload(options, bucket, object, upload_id,
&complete_part_list, headers, &complete_resp_headers);
ret = cos_status_dup(parent_pool, s);
cos_pool_destroy(subpool);
options->pool = parent_pool;
return ret;
}
cos_status_t *cos_upload_object_by_part_copy
(
cos_request_options_t *options,
const cos_string_t *copy_source,
const cos_string_t *dest_bucket,
const cos_string_t *dest_object,
int64_t part_size
)
{
cos_pool_t *subpool = NULL;
cos_pool_t *parent_pool = NULL;
cos_status_t *s = NULL;
cos_status_t *ret = NULL;
char *part_num_str = NULL;
char *etag = NULL;
cos_list_t complete_part_list;
cos_complete_part_content_t *complete_content = NULL;
int64_t total_size = 0;
cos_list_init(&complete_part_list);
parent_pool = options->pool;
cos_pool_create(&subpool, options->pool);
options->pool = subpool;
//get object size
cos_table_t *head_resp_headers = NULL;
s = cos_head_object(options, dest_bucket, dest_object, NULL, &head_resp_headers);
if (!cos_status_is_ok(s)) {
ret = cos_status_dup(parent_pool, s);
cos_pool_destroy(subpool);
options->pool = parent_pool;
return ret;
}
total_size = atol((char*)apr_table_get(head_resp_headers, COS_CONTENT_LENGTH));
//set part copy param
cos_upload_part_copy_params_t *upload_part_copy_params = cos_create_upload_part_copy_params(parent_pool);
cos_str_set(&upload_part_copy_params->copy_source, copy_source->data);
cos_str_set(&upload_part_copy_params->dest_bucket, dest_bucket->data);
cos_str_set(&upload_part_copy_params->dest_object, dest_object->data);
upload_part_copy_params->range_start = 0;
upload_part_copy_params->range_end = 0;
upload_part_copy_params->part_num = 1;
//get upload_id
cos_table_t *init_multipart_headers = NULL;
cos_table_t *init_multipart_resp_headers = NULL;
cos_string_t upload_id;
init_multipart_headers = cos_table_make(subpool, 0);
s = cos_init_multipart_upload(options, dest_bucket, dest_object,
&upload_id, init_multipart_headers, &init_multipart_resp_headers);
if (!cos_status_is_ok(s)) {
ret = cos_status_dup(parent_pool, s);
cos_pool_destroy(subpool);
options->pool = parent_pool;
return ret;
}
cos_str_set(&upload_part_copy_params->upload_id, apr_pstrdup(parent_pool, upload_id.data));
cos_pool_destroy(subpool);
//upload part by copy
while (1) {
cos_pool_create(&subpool, parent_pool);
options->pool = subpool;
upload_part_copy_params->range_end = cos_min(upload_part_copy_params->range_start + part_size - 1, total_size - 1);
s = cos_upload_part_copy(options, upload_part_copy_params, NULL, NULL);
if (!cos_status_is_ok(s)) {
ret = cos_status_dup(parent_pool, s);
cos_pool_destroy(subpool);
options->pool = parent_pool;
return ret;
}
complete_content = cos_create_complete_part_content(parent_pool);
part_num_str = apr_psprintf(parent_pool, "%d", upload_part_copy_params->part_num);
cos_str_set(&complete_content->part_number, part_num_str);
etag = apr_pstrdup(parent_pool, upload_part_copy_params->rsp_content->etag.data);
cos_str_set(&complete_content->etag, etag);
cos_list_add_tail(&complete_content->node, &complete_part_list);
cos_pool_destroy(subpool);
if (upload_part_copy_params->range_end + 1 >= total_size) {
break;
}
upload_part_copy_params->range_start += part_size;
upload_part_copy_params->part_num++;
}
//complete multipart
cos_pool_create(&subpool, parent_pool);
options->pool = subpool;
s = cos_complete_multipart_upload(options, dest_bucket, dest_object, &upload_part_copy_params->upload_id,
&complete_part_list, NULL, NULL);
ret = cos_status_dup(parent_pool, s);
cos_pool_destroy(subpool);
options->pool = parent_pool;
return ret;
}
cos_status_t *cos_download_part_to_file(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
cos_upload_file_t *download_file,
cos_table_t **resp_headers)
{
return cos_do_download_part_to_file(options, bucket, object,
download_file, NULL, NULL, NULL, resp_headers);
}
cos_status_t *cos_do_download_part_to_file(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
cos_upload_file_t *download_file,
cos_progress_callback progress_callback,
cos_table_t *headers,
cos_table_t *params,
cos_table_t **resp_headers)
{
cos_status_t *s = NULL;
cos_http_request_t *req = NULL;
cos_http_response_t *resp = NULL;
int res = COSE_OK;
char range_buf[64];
headers = cos_table_create_if_null(options, headers, 1);
params = cos_table_create_if_null(options, params, 0);
apr_snprintf(range_buf, sizeof(range_buf), "bytes=%"APR_INT64_T_FMT"-%"APR_INT64_T_FMT, download_file->file_pos, download_file->file_last-1);
apr_table_add(headers, COS_RANGE, range_buf);
cos_init_object_request(options, bucket, object, HTTP_GET,
&req, params, headers, progress_callback, 0, &resp);
s = cos_status_create(options->pool);
res = cos_init_read_response_body_to_file_part(options->pool, download_file, resp);
if (res != COSE_OK) {
cos_file_error_status_set(s, res);
return s;
}
s = cos_process_request(options, req, resp);
cos_fill_read_response_header(resp, resp_headers);
if (is_enable_crc(options) && has_crc_in_response(resp) &&
!has_range_or_process_in_request(req)) {
cos_check_crc_consistent(resp->crc64, resp->headers, s);
}
return s;
}

1014
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_object.c vendored
View File

File diff suppressed because it is too large Load Diff

1220
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_resumable.c vendored
View File

File diff suppressed because it is too large Load Diff

210
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_resumable.h vendored
View File

@ -1,210 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_RESUMABLE_H
#define LIBCOS_RESUMABLE_H
#include "cos_sys_define.h"
#include "apr_atomic.h"
#include "apr_queue.h"
#include "apr_thread_pool.h"
COS_CPP_START
#define COS_CP_UPLOAD 1
#define COS_CP_DOWNLOAD 2
typedef struct {
int32_t index; // the index of part, start from 0
int64_t offset; // the offset point of part
int64_t size; // the size of part
int completed; // COS_TRUE completed, COS_FALSE uncompleted
cos_string_t etag; // the etag of part, for upload
} cos_checkpoint_part_t;
typedef struct {
cos_string_t md5; // the md5 of checkout content
int cp_type; // 1 upload, 2 download
apr_file_t *thefile; // the handle of checkpoint file
cos_string_t file_path; // local file path
int64_t file_size; // local file size, for upload
apr_time_t file_last_modified; // local file last modified time, for upload
cos_string_t file_md5; // the md5 of the local file content, for upload, reserved
cos_string_t object_name; // object name
int64_t object_size; // object size, for download
cos_string_t object_last_modified; // object last modified time, for download
cos_string_t object_etag; // object etag, for download
cos_string_t upload_id; // upload id
int part_num; // the total number of parts
int64_t part_size; // the part size, byte
cos_checkpoint_part_t *parts; // the parts of local or object, from 0
} cos_checkpoint_t;
typedef struct {
cos_checkpoint_part_t *part;
cos_status_t *s;
cos_string_t etag;
} cos_part_task_result_t;
typedef struct {
cos_request_options_t options;
cos_string_t *bucket;
cos_string_t *object;
cos_string_t *upload_id;
cos_string_t *filepath;
cos_checkpoint_part_t *part;
cos_part_task_result_t *result;
apr_uint32_t *launched; // the number of launched part tasks, use atomic
apr_uint32_t *failed; // the number of failed part tasks, use atomic
apr_uint32_t *completed; // the number of completed part tasks, use atomic
apr_queue_t *failed_parts; // the queue of failed parts tasks, thread safe
apr_queue_t *completed_parts; // the queue of completed parts tasks, thread safe
} cos_upload_thread_params_t;
typedef struct {
cos_request_options_t options;
cos_string_t *bucket;
cos_string_t *object;
cos_string_t *upload_id;
cos_string_t *copy_source;
cos_checkpoint_part_t *part;
cos_part_task_result_t *result;
apr_uint32_t *launched; // the number of launched part tasks, use atomic
apr_uint32_t *failed; // the number of failed part tasks, use atomic
apr_uint32_t *completed; // the number of completed part tasks, use atomic
apr_queue_t *failed_parts; // the queue of failed parts tasks, thread safe
apr_queue_t *completed_parts; // the queue of completed parts tasks, thread safe
} cos_upload_copy_thread_params_t;
typedef cos_upload_thread_params_t cos_transport_thread_params_t;
int32_t cos_get_thread_num(cos_resumable_clt_params_t *clt_params);
void cos_get_checkpoint_path(cos_resumable_clt_params_t *clt_params, const cos_string_t *filepath,
cos_pool_t *pool, cos_string_t *checkpoint_path);
int cos_get_file_info(const cos_string_t *filepath, cos_pool_t *pool, apr_finfo_t *finfo);
int cos_does_file_exist(const cos_string_t *filepath, cos_pool_t *pool);
int cos_open_checkpoint_file(cos_pool_t *pool, cos_string_t *checkpoint_path, cos_checkpoint_t *checkpoint);
int cos_open_checkpoint_file(cos_pool_t *pool, cos_string_t *checkpoint_path, cos_checkpoint_t *checkpoint);
int cos_get_part_num(int64_t file_size, int64_t part_size);
void cos_build_parts(int64_t file_size, int64_t part_size, cos_checkpoint_part_t *parts);
void cos_build_thread_params(cos_transport_thread_params_t *thr_params, int part_num,
cos_pool_t *parent_pool, cos_request_options_t *options,
cos_string_t *bucket, cos_string_t *object, cos_string_t *filepath,
cos_string_t *upload_id, cos_checkpoint_part_t *parts,
cos_part_task_result_t *result);
void cos_build_copy_thread_params(cos_upload_copy_thread_params_t *thr_params, int part_num,
cos_pool_t *parent_pool, cos_request_options_t *options,
cos_string_t *bucket, cos_string_t *object, cos_string_t *copy_source,
cos_string_t *upload_id, cos_checkpoint_part_t *parts,
cos_part_task_result_t *result);
void cos_destroy_thread_pool(cos_transport_thread_params_t *thr_params, int part_num);
void cos_set_task_tracker(cos_transport_thread_params_t *thr_params, int part_num,
apr_uint32_t *launched, apr_uint32_t *failed, apr_uint32_t *completed,
apr_queue_t *failed_parts, apr_queue_t *completed_parts);
int cos_verify_checkpoint_md5(cos_pool_t *pool, const cos_checkpoint_t *checkpoint);
void cos_build_upload_checkpoint(cos_pool_t *pool, cos_checkpoint_t *checkpoint, cos_string_t *file_path,
apr_finfo_t *finfo, cos_string_t *upload_id, int64_t part_size);
int cos_dump_checkpoint(cos_pool_t *pool, const cos_checkpoint_t *checkpoint);
int cos_load_checkpoint(cos_pool_t *pool, const cos_string_t *filepath, cos_checkpoint_t *checkpoint);
int cos_is_upload_checkpoint_valid(cos_pool_t *pool, cos_checkpoint_t *checkpoint, apr_finfo_t *finfo);
void cos_update_checkpoint(cos_pool_t *pool, cos_checkpoint_t *checkpoint, int32_t part_index, cos_string_t *etag);
void cos_get_checkpoint_undo_parts(cos_checkpoint_t *checkpoint, int *part_num, cos_checkpoint_part_t *parts);
void * APR_THREAD_FUNC upload_part(apr_thread_t *thd, void *data);
cos_status_t *cos_resumable_upload_file_without_cp(cos_request_options_t *options,
cos_string_t *bucket,
cos_string_t *object,
cos_string_t *filepath,
cos_table_t *headers,
cos_table_t *params,
int32_t thread_num,
int64_t part_size,
apr_finfo_t *finfo,
cos_progress_callback progress_callback,
cos_table_t **resp_headers,
cos_list_t *resp_body);
cos_status_t *cos_resumable_upload_file_with_cp(cos_request_options_t *options,
cos_string_t *bucket,
cos_string_t *object,
cos_string_t *filepath,
cos_table_t *headers,
cos_table_t *params,
int32_t thread_num,
int64_t part_size,
cos_string_t *checkpoint_path,
apr_finfo_t *finfo,
cos_progress_callback progress_callback,
cos_table_t **resp_headers,
cos_list_t *resp_body);
void * APR_THREAD_FUNC upload_part_copy(apr_thread_t *thd, void *data);
cos_status_t *cos_upload_object_by_part_copy_mt
(
cos_request_options_t *options,
cos_string_t *src_bucket,
cos_string_t *src_object,
cos_string_t *src_endpoint,
cos_string_t *dest_bucket,
cos_string_t *dest_object,
int64_t part_size,
int32_t thread_num,
cos_progress_callback progress_callback
);
void * APR_THREAD_FUNC download_part(apr_thread_t *thd, void *data);
int64_t cos_get_safe_size_for_download(int64_t part_size);
cos_status_t *cos_resumable_download_file_without_cp(cos_request_options_t *options,
cos_string_t *bucket,
cos_string_t *object,
cos_string_t *filepath,
cos_table_t *headers,
cos_table_t *params,
int32_t thread_num,
int64_t part_size,
cos_progress_callback progress_callback);
COS_CPP_END
#endif

118
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_status.c vendored
View File

@ -1,118 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_log.h"
#include "cos_sys_util.h"
#include "cos_status.h"
const char COS_XML_PARSE_ERROR_CODE[] = "ParseXmlError";
const char COS_OPEN_FILE_ERROR_CODE[] = "OpenFileFail";
const char COS_WRITE_FILE_ERROR_CODE[] = "WriteFileFail";
const char COS_HTTP_IO_ERROR_CODE[] = "HttpIoError";
const char COS_UNKNOWN_ERROR_CODE[] = "UnknownError";
const char COS_CLIENT_ERROR_CODE[] = "ClientError";
const char COS_UTF8_ENCODE_ERROR_CODE[] = "Utf8EncodeFail";
const char COS_URL_ENCODE_ERROR_CODE[] = "UrlEncodeFail";
const char COS_INCONSISTENT_ERROR_CODE[] = "InconsistentError";
const char COS_CREATE_QUEUE_ERROR_CODE[] = "CreateQueueFail";
const char COS_CREATE_THREAD_POOL_ERROR_CODE[] = "CreateThreadPoolFail";
const char COS_LACK_OF_CONTENT_LEN_ERROR_CODE[] = "LackOfContentLength";
cos_status_t *cos_status_create(cos_pool_t *p)
{
return (cos_status_t *)cos_pcalloc(p, sizeof(cos_status_t));
}
cos_status_t *cos_status_dup(cos_pool_t *p, cos_status_t *src)
{
cos_status_t *dst = cos_status_create(p);
dst->code = src->code;
dst->error_code = apr_pstrdup(p, src->error_code);
dst->error_msg = apr_pstrdup(p, src->error_msg);
return dst;
}
int cos_should_retry(cos_status_t *s) {
int cos_error_code = 0;
if (s == NULL || s->code / 100 == 2) {
return COS_FALSE;
}
if (s->code / 100 == 5) {
return COS_TRUE;
}
if (s->error_code != NULL) {
cos_error_code = atoi(s->error_code);
if (cos_error_code == COSE_CONNECTION_FAILED || cos_error_code == COSE_REQUEST_TIMEOUT ||
cos_error_code == COSE_FAILED_CONNECT || cos_error_code == COSE_SERVICE_ERROR) {
return COS_TRUE;
}
}
return COS_FALSE;
}
cos_status_t *cos_status_parse_from_body(cos_pool_t *p, cos_list_t *bc, int code, cos_status_t *s)
{
int res;
mxml_node_t *root, *node;
mxml_node_t *code_node, *message_node;
const char *node_content;
if (s == NULL) {
s = cos_status_create(p);
}
s->code = code;
if (cos_http_is_ok(code)) {
return s;
}
if (cos_list_empty(bc)) {
s->error_code = (char *)COS_UNKNOWN_ERROR_CODE;
return s;
}
if ((res = cos_parse_xml_body(bc, &root)) != COSE_OK) {
s->error_code = (char *)COS_UNKNOWN_ERROR_CODE;
return s;
}
node = mxmlFindElement(root, root, "Error",NULL, NULL,MXML_DESCEND);
if (NULL == node) {
char *xml_content = cos_buf_list_content(p, bc);
cos_error_log("Xml format invalid, root node name is not Error.\n");
cos_error_log("Xml Content:%s\n", xml_content);
s->error_code = (char *)COS_UNKNOWN_ERROR_CODE;
mxmlDelete(root);
return s;
}
code_node = mxmlFindElement(node, root, "Code",NULL, NULL,MXML_DESCEND);
node_content = mxmlGetOpaque(code_node);
if (node_content != NULL) {
s->error_code = apr_pstrdup(p, node_content);
}
message_node = mxmlFindElement(node, root, "Message",NULL, NULL,MXML_DESCEND);
node_content = mxmlGetOpaque(message_node);
if (node_content != NULL) {
s->error_msg = apr_pstrdup(p, node_content);
}
mxmlDelete(root);
return s;
}

71
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_status.h vendored
View File

@ -1,71 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_STATUS_H
#define LIBCOS_STATUS_H
#include "cos_sys_define.h"
#include "cos_list.h"
COS_CPP_START
typedef struct cos_status_s cos_status_t;
struct cos_status_s {
int code; // > 0 http code
char *error_code; // can't modify
char *error_msg; // can't modify
char *req_id; // can't modify
};
static APR_INLINE int cos_status_is_ok(cos_status_t *s)
{
return s->code > 0 && s->code / 100 == 2;
}
static APR_INLINE int cos_http_is_ok(int st)
{
return st / 100 == 2;
}
#define cos_status_set(s, c, ec, es) \
(s)->code = c; (s)->error_code = (char *)ec; (s)->error_msg = (char *)es
/**
* @brief determine whether the request should be retried
* @param[in] s the return status of api, such as cos_put_object_from_buffer
* @return int COS_FALSE indicates no retries, COS_TRUE retry
*/
int cos_should_retry(cos_status_t *s);
cos_status_t *cos_status_create(cos_pool_t *p);
cos_status_t *cos_status_dup(cos_pool_t *p, cos_status_t *src);
cos_status_t *cos_status_parse_from_body(cos_pool_t *p, cos_list_t *bc, int code, cos_status_t *s);
extern const char COS_XML_PARSE_ERROR_CODE[];
extern const char COS_OPEN_FILE_ERROR_CODE[];
extern const char COS_WRITE_FILE_ERROR_CODE[];
extern const char COS_HTTP_IO_ERROR_CODE[];
extern const char COS_UNKNOWN_ERROR_CODE[];
extern const char COS_CLIENT_ERROR_CODE[];
extern const char COS_UTF8_ENCODE_ERROR_CODE[];
extern const char COS_URL_ENCODE_ERROR_CODE[];
extern const char COS_INCONSISTENT_ERROR_CODE[];
extern const char COS_CREATE_QUEUE_ERROR_CODE[];
extern const char COS_CREATE_THREAD_POOL_ERROR_CODE[];
extern const char COS_LACK_OF_CONTENT_LEN_ERROR_CODE[];
COS_CPP_END
#endif

66
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_string.c vendored
View File

@ -1,66 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_string.h"
typedef int (*cos_is_char_pt)(char c);
static void cos_strip_str_func(cos_string_t *str, cos_is_char_pt func);
char *cos_pstrdup(cos_pool_t *p, const cos_string_t *s)
{
return apr_pstrndup(p, s->data, s->len);
}
static void cos_strip_str_func(cos_string_t *str, cos_is_char_pt func)
{
char *data = str->data;
int len = str->len;
int offset = 0;
if (len == 0) return;
while (len > 0 && func(data[len - 1])) {
--len;
}
for (; offset < len && func(data[offset]); ++offset) {
// empty;
}
str->data = data + offset;
str->len = len - offset;
}
void cos_unquote_str(cos_string_t *str)
{
cos_strip_str_func(str, cos_is_quote);
}
void cos_strip_space(cos_string_t *str)
{
cos_strip_str_func(str, cos_is_space);
}
void cos_trip_space_and_cntrl(cos_string_t *str)
{
cos_strip_str_func(str, cos_is_space_or_cntrl);
}
int cos_ends_with(const cos_string_t *str, const cos_string_t *suffix)
{
if (!str || !suffix) {
return 0;
}
return (str->len >= suffix->len) && strncmp(str->data + str->len - suffix->len, suffix->data, suffix->len) == 0;
}

88
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_string.h vendored
View File

@ -1,88 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_STRING_H
#define LIBCOS_STRING_H
#include "cos_sys_define.h"
COS_CPP_START
typedef struct {
int len;
char *data;
} cos_string_t;
#define cos_string(str) { sizeof(str) - 1, (char *) str }
#define cos_null_string { 0, NULL }
#define cos_str_set(str, text) \
(str)->len = strlen(text); (str)->data = (char *) text
#define cos_str_null(str) (str)->len = 0; (str)->data = NULL
#define cos_tolower(c) (char) ((c >= 'A' && c <= 'Z') ? (c | 0x20) : c)
#define cos_toupper(c) (char) ((c >= 'a' && c <= 'z') ? (c & ~0x20) : c)
static APR_INLINE void cos_string_tolower(cos_string_t *str)
{
int i = 0;
while (i < str->len) {
str->data[i] = cos_tolower(str->data[i]);
++i;
}
}
static APR_INLINE char *cos_strlchr(char *p, char *last, char c)
{
while (p < last) {
if (*p == c) {
return p;
}
p++;
}
return NULL;
}
static APR_INLINE int cos_is_quote(char c)
{
return c == '\"';
}
static APR_INLINE int cos_is_space(char c)
{
return ((c == ' ') || (c == '\t'));
}
static APR_INLINE int cos_is_space_or_cntrl(char c)
{
return c <= ' ';
}
static APR_INLINE int cos_is_null_string(cos_string_t *str)
{
if (str == NULL || str->data == NULL || str->len == 0) {
return COS_TRUE;
}
return COS_FALSE;
}
void cos_strip_space(cos_string_t *str);
void cos_trip_space_and_cntrl(cos_string_t *str);
void cos_unquote_str(cos_string_t *str);
char *cos_pstrdup(cos_pool_t *p, const cos_string_t *s);
int cos_ends_with(const cos_string_t *str, const cos_string_t *suffix);
COS_CPP_END
#endif

150
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_sys_define.h vendored
View File

@ -1,150 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_SYS_DEFINE_H
#define LIBCOS_SYS_DEFINE_H
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <ctype.h>
#include <time.h>
#include <apr_time.h>
#include <apr_strings.h>
#include <apr_pools.h>
#include <apr_tables.h>
#include <apr_file_io.h>
#include <curl/curl.h>
#ifdef __cplusplus
# define COS_CPP_START extern "C" {
# define COS_CPP_END }
#else
# define COS_CPP_START
# define COS_CPP_END
#endif
typedef enum {
HTTP_GET,
HTTP_HEAD,
HTTP_PUT,
HTTP_POST,
HTTP_DELETE
} http_method_e;
typedef enum {
COSE_OK = 0,
COSE_OUT_MEMORY = -1000,
COSE_OVER_MEMORY = -999,
COSE_FAILED_CONNECT = -998,
COSE_ABORT_CALLBACK = -997,
COSE_INTERNAL_ERROR = -996,
COSE_REQUEST_TIMEOUT = -995,
COSE_INVALID_ARGUMENT = -994,
COSE_INVALID_OPERATION = -993,
COSE_CONNECTION_FAILED = -992,
COSE_FAILED_INITIALIZE = -991,
COSE_NAME_LOOKUP_ERROR = -990,
COSE_FAILED_VERIFICATION = -989,
COSE_WRITE_BODY_ERROR = -988,
COSE_READ_BODY_ERROR = -987,
COSE_SERVICE_ERROR = -986,
COSE_OPEN_FILE_ERROR = -985,
COSE_FILE_SEEK_ERROR = -984,
COSE_FILE_INFO_ERROR = -983,
COSE_FILE_READ_ERROR = -982,
COSE_FILE_WRITE_ERROR = -981,
COSE_XML_PARSE_ERROR = -980,
COSE_UTF8_ENCODE_ERROR = -979,
COSE_CRC_INCONSISTENT_ERROR = -978,
COSE_FILE_FLUSH_ERROR = -977,
COSE_FILE_TRUNC_ERROR = -976,
COSE_UNKNOWN_ERROR = -100
} cos_error_code_e;
typedef apr_pool_t cos_pool_t;
typedef apr_table_t cos_table_t;
typedef apr_table_entry_t cos_table_entry_t;
typedef apr_array_header_t cos_array_header_t;
#define cos_table_elts(t) apr_table_elts(t)
#define cos_is_empty_table(t) apr_is_empty_table(t)
#define cos_table_make(p, n) apr_table_make(p, n)
#define cos_table_add_int(t, key, value) do { \
char value_str[64]; \
apr_snprintf(value_str, sizeof(value_str), "%d", value);\
apr_table_add(t, key, value_str); \
} while(0)
#define cos_table_add_int64(t, key, value) do { \
char value_str[64]; \
apr_snprintf(value_str, sizeof(value_str), "%" APR_INT64_T_FMT, value);\
apr_table_add(t, key, value_str); \
} while(0)
#define cos_table_set_int64(t, key, value) do { \
char value_str[64]; \
apr_snprintf(value_str, sizeof(value_str), "%" APR_INT64_T_FMT, value);\
apr_table_set(t, key, value_str); \
} while(0)
#define cos_pool_create(n, p) apr_pool_create(n, p)
#define cos_pool_destroy(p) apr_pool_destroy(p)
#define cos_palloc(p, s) apr_palloc(p, s)
#define cos_pcalloc(p, s) apr_pcalloc(p, s)
#define COS_RETRY_TIME 2
#define COS_INIT_WINSOCK 1
#define COS_MD5_STRING_LEN 32
#define COS_MAX_URI_LEN 2048
#define COS_MAX_HEADER_LEN 8192
#define COS_MAX_QUERY_ARG_LEN 1024
#define COS_MAX_GMT_TIME_LEN 128
#define COS_MAX_XML_NODE_VALUE_LEN 1024
#define COS_MAX_INT64_STRING_LEN 64
#define COS_CONNECT_TIMEOUT 10
#define COS_DNS_CACHE_TIMOUT 60
#define COS_MIN_SPEED_LIMIT 8
#define COS_MIN_SPEED_TIME 120
#define COS_MAX_MEMORY_SIZE 1024*1024*1024L
#define COS_MAX_PART_SIZE 512*1024*1024L
#define COS_DEFAULT_PART_SIZE 1024*1024L
#define COS_REQUEST_STACK_SIZE 32
#define cos_abs(value) (((value) >= 0) ? (value) : - (value))
#define cos_max(val1, val2) (((val1) < (val2)) ? (val2) : (val1))
#define cos_min(val1, val2) (((val1) > (val2)) ? (val2) : (val1))
#define LF (char) 10
#define CR (char) 13
#define CRLF "\x0d\x0a"
#define COS_VERSION "5.0.8"
#define COS_VER "cos-sdk-c/" COS_VERSION
#define COS_HTTP_PREFIX "http://"
#define COS_HTTPS_PREFIX "https://"
#define COS_RTMP_PREFIX "rtmp://"
#define COS_TEMP_FILE_SUFFIX ".tmp"
#define COS_FALSE 0
#define COS_TRUE 1
#endif

593
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_sys_util.c vendored
View File

@ -1,593 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_sys_util.h"
#include "cos_log.h"
static const char *g_s_wday[] = {
"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"
};
static const char *g_s_mon[] = {
"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
};
static const char g_s_gmt_format[] = "%s, %.2d %s %.4d %.2d:%.2d:%.2d GMT";
int cos_parse_xml_body(cos_list_t *bc, mxml_node_t **root)
{
cos_buf_t *b;
size_t len;
*root = NULL;
len = (size_t)cos_buf_list_len(bc);
{
int nsize = 0;
char *buffer = (char*)malloc(sizeof(char)*(len+1));
memset(buffer, 0, len + 1);
cos_list_for_each_entry(cos_buf_t, b, bc, node) {
memcpy(buffer + nsize, (char *)b->pos, cos_buf_size(b));
nsize += cos_buf_size(b);
}
*root = mxmlLoadString(NULL, buffer, MXML_OPAQUE_CALLBACK);
free(buffer);
if (NULL == *root) {
return COSE_INTERNAL_ERROR;
}
}
return COSE_OK;
}
int cos_convert_to_gmt_time(char* date, const char* format, apr_time_exp_t *tm)
{
int size = apr_snprintf(date, COS_MAX_GMT_TIME_LEN, format,
g_s_wday[tm->tm_wday], tm->tm_mday, g_s_mon[tm->tm_mon], 1900 + tm->tm_year, tm->tm_hour, tm->tm_min, tm->tm_sec);
if (size >= 0 && size < COS_MAX_GMT_TIME_LEN) {
return COSE_OK;
} else {
return COSE_INTERNAL_ERROR;
}
}
int cos_get_gmt_str_time(char datestr[COS_MAX_GMT_TIME_LEN])
{
int s;
apr_time_t now;
char buf[128];
apr_time_exp_t result;
now = apr_time_now();
if ((s = apr_time_exp_gmt(&result, now)) != APR_SUCCESS) {
cos_error_log("apr_time_exp_gmt fialure, code:%d %s.", s, apr_strerror(s, buf, sizeof(buf)));
return COSE_INTERNAL_ERROR;
}
if ((s = cos_convert_to_gmt_time(datestr, g_s_gmt_format, &result))
!= COSE_OK) {
cos_error_log("cos_convert_to_GMT failure, code:%d.", s);
}
return s;
}
int cos_url_encode(char *dest, const char *src, int maxSrcSize)
{
static const char *hex = "0123456789ABCDEF";
int len = 0;
unsigned char c;
while (*src) {
if (++len > maxSrcSize) {
*dest = 0;
return COSE_INVALID_ARGUMENT;
}
c = *src;
if (isalnum(c) || (c == '-') || (c == '_') || (c == '.') || (c == '~')) {
*dest++ = c;
} else if (*src == ' ') {
*dest++ = '%';
*dest++ = '2';
*dest++ = '0';
} else {
*dest++ = '%';
*dest++ = hex[c >> 4];
*dest++ = hex[c & 15];
}
src++;
}
*dest = 0;
return COSE_OK;
}
int cos_query_params_to_string(cos_pool_t *p, cos_table_t *query_params, cos_string_t *querystr)
{
int rs;
int pos;
int len;
char sep = '?';
char ebuf[COS_MAX_QUERY_ARG_LEN*3+1];
char abuf[COS_MAX_QUERY_ARG_LEN*6+128];
int max_len;
const cos_array_header_t *tarr;
const cos_table_entry_t *telts;
cos_buf_t *querybuf;
if (apr_is_empty_table(query_params)) {
return COSE_OK;
}
max_len = sizeof(abuf)-1;
querybuf = cos_create_buf(p, 256);
cos_str_null(querystr);
tarr = cos_table_elts(query_params);
telts = (cos_table_entry_t*)tarr->elts;
for (pos = 0; pos < tarr->nelts; ++pos) {
if ((rs = cos_url_encode(ebuf, telts[pos].key, COS_MAX_QUERY_ARG_LEN)) != COSE_OK) {
cos_error_log("query params args too big, key:%s.", telts[pos].key);
return COSE_INVALID_ARGUMENT;
}
len = apr_snprintf(abuf, max_len, "%c%s", sep, ebuf);
if (telts[pos].val != NULL && *telts[pos].val != '\0') {
if ((rs = cos_url_encode(ebuf, telts[pos].val, COS_MAX_QUERY_ARG_LEN)) != COSE_OK) {
cos_error_log("query params args too big, value:%s.", telts[pos].val);
return COSE_INVALID_ARGUMENT;
}
len += apr_snprintf(abuf+len, max_len-len, "=%s", ebuf);
if (len >= COS_MAX_QUERY_ARG_LEN) {
cos_error_log("query params args too big, %s.", abuf);
return COSE_INVALID_ARGUMENT;
}
}
cos_buf_append_string(p, querybuf, abuf, len);
sep = '&';
}
// result
querystr->data = (char *)querybuf->pos;
querystr->len = cos_buf_size(querybuf);
return COSE_OK;
}
#if 0
void cos_gnome_sort(const char **headers, int size)
{
const char *tmp;
int i = 0, last_highest = 0;
while (i < size) {
if ((i == 0) || apr_strnatcasecmp(headers[i-1], headers[i]) < 0) {
i = ++last_highest;
} else {
tmp = headers[i];
headers[i] = headers[i - 1];
headers[--i] = tmp;
}
}
}
const char* cos_http_method_to_string(http_method_e method)
{
switch (method) {
case HTTP_GET:
return "GET";
case HTTP_HEAD:
return "HEAD";
case HTTP_PUT:
return "PUT";
case HTTP_POST:
return "POST";
case HTTP_DELETE:
return "DELETE";
default:
return "UNKNOWN";
}
}
#endif
const char* cos_http_method_to_string_lower(http_method_e method)
{
switch (method) {
case HTTP_GET:
return "get";
case HTTP_HEAD:
return "head";
case HTTP_PUT:
return "put";
case HTTP_POST:
return "post";
case HTTP_DELETE:
return "delete";
default:
return "unknown";
}
}
int cos_base64_encode(const unsigned char *in, int inLen, char *out)
{
static const char *ENC =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
char *original_out = out;
while (inLen) {
// first 6 bits of char 1
*out++ = ENC[*in >> 2];
if (!--inLen) {
// last 2 bits of char 1, 4 bits of 0
*out++ = ENC[(*in & 0x3) << 4];
*out++ = '=';
*out++ = '=';
break;
}
// last 2 bits of char 1, first 4 bits of char 2
*out++ = ENC[((*in & 0x3) << 4) | (*(in + 1) >> 4)];
in++;
if (!--inLen) {
// last 4 bits of char 2, 2 bits of 0
*out++ = ENC[(*in & 0xF) << 2];
*out++ = '=';
break;
}
// last 4 bits of char 2, first 2 bits of char 3
*out++ = ENC[((*in & 0xF) << 2) | (*(in + 1) >> 6)];
in++;
// last 6 bits of char 3
*out++ = ENC[*in & 0x3F];
in++, inLen--;
}
return (out - original_out);
}
// HMAC-SHA-1:
//
// K - is key padded with zeros to 512 bits
// m - is message
// OPAD - 0x5c5c5c...
// IPAD - 0x363636...
//
// HMAC(K,m) = SHA1((K ^ OPAD) . SHA1((K ^ IPAD) . m))
void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len,
const unsigned char *message, int message_len)
{
unsigned char kopad[64], kipad[64];
int i;
unsigned char digest[APR_SHA1_DIGESTSIZE];
apr_sha1_ctx_t context;
if (key_len > 64) {
key_len = 64;
}
for (i = 0; i < key_len; i++) {
kopad[i] = key[i] ^ 0x5c;
kipad[i] = key[i] ^ 0x36;
}
for ( ; i < 64; i++) {
kopad[i] = 0 ^ 0x5c;
kipad[i] = 0 ^ 0x36;
}
apr_sha1_init(&context);
apr_sha1_update(&context, (const char *)kipad, 64);
apr_sha1_update(&context, (const char *)message, (unsigned int)message_len);
apr_sha1_final(digest, &context);
apr_sha1_init(&context);
apr_sha1_update(&context, (const char *)kopad, 64);
apr_sha1_update(&context, (const char *)digest, 20);
apr_sha1_final(hmac, &context);
}
unsigned char* cos_md5(cos_pool_t* pool, const char *in, apr_size_t in_len) {
unsigned char* out;
apr_md5_ctx_t context;
//APR_MD5_DIGESTSIZE: The MD5 digest size, value is 16
out = cos_palloc(pool, APR_MD5_DIGESTSIZE + 1);
if (!out) {
return NULL;
}
if (0 != apr_md5_init(&context)) {
return NULL;
}
if (0 != apr_md5_update(&context, in, in_len)) {
return NULL;
}
if (0 != apr_md5_final(out, &context)) {
return NULL;
}
out[APR_MD5_DIGESTSIZE] = '\0';
return out;
};
int cos_url_decode(const char *in, char *out)
{
static const char tbl[256] = {
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1,
-1,10,11,12,13,14,15,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,10,11,12,13,14,15,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1
};
char c, v1, v2;
if(in != NULL) {
while((c=*in++) != '\0') {
if(c == '%') {
if(!(v1=*in++) || (v1=tbl[(unsigned char)v1])<0 ||
!(v2=*in++) || (v2=tbl[(unsigned char)v2])<0) {
*out = '\0';
return -1;
}
c = (v1<<4)|v2;
} else if (c == '+') {
c = ' ';
}
*out++ = c;
}
}
*out = '\0';
return 0;
}
/*
* Convert a string to a long long integer.
*
* Ignores `locale' stuff. Assumes that the upper and lower case
* alphabets and digits are each contiguous.
*/
long long cos_strtoll(const char *nptr, char **endptr, int base)
{
const char *s;
/* LONGLONG */
long long int acc, cutoff;
int c;
int neg, any, cutlim;
/* endptr may be NULL */
#ifdef __GNUC__
/* This outrageous construct just to shut up a GCC warning. */
(void) &acc; (void) &cutoff;
#endif
/*
* Skip white space and pick up leading +/- sign if any.
* If base is 0, allow 0x for hex and 0 for octal, else
* assume decimal; if base is already 16, allow 0x.
*/
s = nptr;
do {
c = (unsigned char) *s++;
} while (isspace(c));
if (c == '-') {
neg = 1;
c = *s++;
} else {
neg = 0;
if (c == '+')
c = *s++;
}
if ((base == 0 || base == 16) &&
c == '0' && (*s == 'x' || *s == 'X')) {
c = s[1];
s += 2;
base = 16;
}
if (base == 0)
base = c == '0' ? 8 : 10;
/*
* Compute the cutoff value between legal numbers and illegal
* numbers. That is the largest legal value, divided by the
* base. An input number that is greater than this value, if
* followed by a legal input character, is too big. One that
* is equal to this value may be valid or not; the limit
* between valid and invalid numbers is then based on the last
* digit. For instance, if the range for long longs is
* [-9223372036854775808..9223372036854775807] and the input base
* is 10, cutoff will be set to 922337203685477580 and cutlim to
* either 7 (neg==0) or 8 (neg==1), meaning that if we have
* accumulated a value > 922337203685477580, or equal but the
* next digit is > 7 (or 8), the number is too big, and we will
* return a range error.
*
* Set any if any `digits' consumed; make it negative to indicate
* overflow.
*/
cutoff = neg ? LLONG_MIN : LLONG_MAX;
cutlim = (int)(cutoff % base);
cutoff /= base;
if (neg) {
if (cutlim > 0) {
cutlim -= base;
cutoff += 1;
}
cutlim = -cutlim;
}
for (acc = 0, any = 0;; c = (unsigned char) *s++) {
if (isdigit(c))
c -= '0';
else if (isalpha(c))
c -= isupper(c) ? 'A' - 10 : 'a' - 10;
else
break;
if (c >= base)
break;
if (any < 0)
continue;
if (neg) {
if (acc < cutoff || (acc == cutoff && c > cutlim)) {
any = -1;
acc = LLONG_MIN;
errno = ERANGE;
} else {
any = 1;
acc *= base;
acc -= c;
}
} else {
if (acc > cutoff || (acc == cutoff && c > cutlim)) {
any = -1;
acc = LLONG_MAX;
errno = ERANGE;
} else {
any = 1;
acc *= base;
acc += c;
}
}
}
if (endptr != 0)
/* LINTED interface specification */
*endptr = (char *)(any ? s - 1 : nptr);
return (acc);
}
int64_t cos_atoi64(const char *nptr)
{
return cos_strtoull(nptr, NULL, 10);
}
unsigned long long cos_strtoull(const char *nptr, char **endptr, int base)
{
const char *s;
unsigned long long acc, cutoff;
int c;
int neg, any, cutlim;
/*
* See strtoq for comments as to the logic used.
*/
s = nptr;
do {
c = (unsigned char) *s++;
} while (isspace(c));
if (c == '-') {
neg = 1;
c = *s++;
} else {
neg = 0;
if (c == '+')
c = *s++;
}
if ((base == 0 || base == 16) &&
c == '0' && (*s == 'x' || *s == 'X')) {
c = s[1];
s += 2;
base = 16;
}
if (base == 0)
base = c == '0' ? 8 : 10;
cutoff = ULLONG_MAX / (unsigned long long)base;
cutlim = ULLONG_MAX % (unsigned long long)base;
for (acc = 0, any = 0;; c = (unsigned char) *s++) {
if (isdigit(c))
c -= '0';
else if (isalpha(c))
c -= isupper(c) ? 'A' - 10 : 'a' - 10;
else
break;
if (c >= base)
break;
if (any < 0)
continue;
if (acc > cutoff || (acc == cutoff && c > cutlim)) {
any = -1;
acc = ULLONG_MAX;
errno = ERANGE;
} else {
any = 1;
acc *= (unsigned long long)base;
acc += c;
}
}
if (neg && any > 0)
#ifdef WIN32
#pragma warning(disable : 4146)
#endif
acc = -acc;
#ifdef WIN32
#pragma warning(default : 4146)
#endif
if (endptr != 0)
*endptr = (char *) (any ? s - 1 : nptr);
return (acc);
}
uint64_t cos_atoui64(const char *nptr)
{
return cos_strtoull(nptr, NULL, 10);
}
void cos_get_hex_from_digest(unsigned char hexdigest[40], unsigned char digest[20])
{
unsigned char hex_digits[16] = { '0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
int j = 0;
int i = 0;
for(; i < 20; i++)
{
hexdigest[j++] = hex_digits[(digest[i] >> 4) & 0x0f];
hexdigest[j++] = hex_digits[digest[i] & 0x0f];
}
}
void cos_get_hmac_sha1_hexdigest(unsigned char hexdigest[40], const unsigned char *key, int key_len,
const unsigned char *message, int message_len)
{
unsigned char hmac[20];
HMAC_SHA1(hmac, key, key_len, message, message_len);
cos_get_hex_from_digest(hexdigest, hmac);
}
void cos_get_sha1_hexdigest(unsigned char hexdigest[40], const unsigned char *message, int message_len)
{
unsigned char digest[20];
apr_sha1_ctx_t context;
apr_sha1_init(&context);
apr_sha1_update(&context, (const char *)message, (unsigned int)message_len);
apr_sha1_final(digest, &context);
cos_get_hex_from_digest(hexdigest, digest);
}

107
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_sys_util.h vendored
View File

@ -1,107 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_SYS_UTIL_H
#define LIBCOS_SYS_UTIL_H
#include "cos_buf.h"
#include "cos_string.h"
#include "cos_sys_define.h"
#include "cos_fstack.h"
#include <mxml.h>
#include <apr_md5.h>
#include <apr_sha1.h>
COS_CPP_START
int cos_parse_xml_body(cos_list_t *bc, mxml_node_t **root);
void cos_gnome_sort(const char **headers, int size);
int cos_convert_to_gmt_time(char* date, const char* format, apr_time_exp_t *tm);
int cos_get_gmt_str_time(char datestr[COS_MAX_GMT_TIME_LEN]);
/**
* URL-encodes a string from [src] into [dest]. [dest] must have at least
* 3x the number of characters that [source] has. At most [maxSrcSize] bytes
* from [src] are encoded; if more are present in [src], 0 is returned from
* urlEncode, else nonzero is returned.
*/
int cos_url_encode(char *dest, const char *src, int maxSrcSize);
const char* cos_http_method_to_string(http_method_e method);
const char* cos_http_method_to_string_lower(http_method_e method);
/**
* encode query string, check query args < COS_MAX_QUERY_ARG_LEN
* result string "?a&b=x"
*/
int cos_query_params_to_string(cos_pool_t *p, cos_table_t *query_params, cos_string_t *querystr);
/**
* base64 encode bytes. The output buffer must have at least
* ((4 * (inLen + 1)) / 3) bytes in it. Returns the number of bytes written
* to [out].
*/
int cos_base64_encode(const unsigned char *in, int inLen, char *out);
/**
* Compute HMAC-SHA-1 with key [key] and message [message], storing result
* in [hmac]
*/
void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len,
const unsigned char *message, int message_len);
unsigned char* cos_md5(cos_pool_t* pool, const char* in, apr_size_t in_len);
int cos_url_decode(const char *in, char *out);
/*
* Convert a string to a long long integer.
*
* Ignores `locale' stuff. Assumes that the upper and lower case
* alphabets and digits are each contiguous.
*/
long long cos_strtoll(const char *nptr, char **endptr, int base);
/*
* @brief Convert a string to int64_t.
**/
int64_t cos_atoi64(const char *nptr);
/*
* @brief Convert a string to an unsigned long long integer.
*
* Ignores `locale' stuff. Assumes that the upper and lower case
* alphabets and digits are each contiguous.
**/
unsigned long long cos_strtoull(const char *nptr, char **endptr, int base);
/*
* @brief Convert a string to uint64_t.
**/
uint64_t cos_atoui64(const char *nptr);
void cos_get_hex_from_digest(unsigned char hexdigest[40], unsigned char digest[20]);
void cos_get_hmac_sha1_hexdigest(unsigned char hexdigest[40], const unsigned char *key, int key_len,
const unsigned char *message, int message_len);
void cos_get_sha1_hexdigest(unsigned char hexdigest[40], const unsigned char *message, int message_len);
COS_CPP_END
#endif

508
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_transport.c vendored
View File

@ -1,508 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "cos_log.h"
#include "cos_sys_util.h"
#include "cos_string.h"
#include "cos_http_io.h"
#include "cos_transport.h"
#include "cos_crc64.h"
int cos_curl_code_to_status(CURLcode code);
static void cos_init_curl_headers(cos_curl_http_transport_t *t);
static void cos_transport_cleanup(cos_http_transport_t *t);
static int cos_init_curl_url(cos_curl_http_transport_t *t);
static void cos_curl_transport_headers_done(cos_curl_http_transport_t *t);
static int cos_curl_transport_setup(cos_curl_http_transport_t *t);
static void cos_curl_transport_finish(cos_curl_http_transport_t *t);
static void cos_move_transport_state(cos_curl_http_transport_t *t, cos_transport_state_e s);
static size_t cos_curl_default_header_callback(char *buffer, size_t size, size_t nitems, void *userdata);
static size_t cos_curl_default_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata);
static size_t cos_curl_default_read_callback(char *buffer, size_t size, size_t nitems, void *instream);
static void cos_init_curl_headers(cos_curl_http_transport_t *t)
{
int pos;
char *header;
const cos_array_header_t *tarr;
const cos_table_entry_t *telts;
union cos_func_u func;
if (t->req->method == HTTP_PUT || t->req->method == HTTP_POST) {
header = apr_psprintf(t->pool, "Content-Length: %" APR_INT64_T_FMT, t->req->body_len);
t->headers = curl_slist_append(t->headers, header);
}
tarr = cos_table_elts(t->req->headers);
telts = (cos_table_entry_t*)tarr->elts;
for (pos = 0; pos < tarr->nelts; ++pos) {
header = apr_psprintf(t->pool, "%s: %s", telts[pos].key, telts[pos].val);
t->headers = curl_slist_append(t->headers, header);
}
/* Disable these headers if they're not set explicitly */
if (NULL == apr_table_get(t->req->headers, COS_EXPECT)) {
header = apr_psprintf(t->pool, "%s: %s", COS_EXPECT, "");
t->headers = curl_slist_append(t->headers, header);
}
if (NULL == apr_table_get(t->req->headers, COS_TRANSFER_ENCODING)) {
header = apr_psprintf(t->pool, "%s: %s", COS_TRANSFER_ENCODING, "");
t->headers = curl_slist_append(t->headers, header);
}
func.func1 = (cos_func1_pt)curl_slist_free_all;
cos_fstack_push(t->cleanup, t->headers, func, 1);
}
static int cos_init_curl_url(cos_curl_http_transport_t *t)
{
int rs;
const char *proto;
cos_string_t querystr;
char uristr[3*COS_MAX_URI_LEN+1];
uristr[0] = '\0';
cos_str_null(&querystr);
if ((rs = cos_url_encode(uristr, t->req->uri, COS_MAX_URI_LEN)) != COSE_OK) {
t->controller->error_code = rs;
t->controller->reason = "uri invalid argument.";
return rs;
}
if ((rs = cos_query_params_to_string(t->pool, t->req->query_params, &querystr)) != COSE_OK) {
t->controller->error_code = rs;
t->controller->reason = "query params invalid argument.";
return rs;
}
proto = strlen(t->req->proto) != 0 ? t->req->proto : COS_HTTP_PREFIX;
/* use original host to build url */
if (NULL == t->controller->options->host_ip || 0 >= t->controller->options->host_port) {
if (querystr.len == 0) {
t->url = apr_psprintf(t->pool, "%s%s/%s",
proto,
t->req->host,
uristr);
} else {
t->url = apr_psprintf(t->pool, "%s%s/%s%.*s",
proto,
t->req->host,
uristr,
querystr.len,
querystr.data);
}
}
/* use specified ip-port to build url */
else {
if (querystr.len == 0) {
t->url = apr_psprintf(t->pool, "%s%s:%d/%s",
proto,
t->controller->options->host_ip,
t->controller->options->host_port,
uristr);
} else {
t->url = apr_psprintf(t->pool, "%s%s:%d/%s%.*s",
proto,
t->controller->options->host_ip,
t->controller->options->host_port,
uristr,
querystr.len,
querystr.data);
}
}
cos_info_log("url:%s", t->url);
return COSE_OK;
}
static void cos_transport_cleanup(cos_http_transport_t *t)
{
int s;
char buf[256];
if (t->req->file_buf != NULL && t->req->file_buf->owner) {
cos_trace_log("close request body file.");
if ((s = apr_file_close(t->req->file_buf->file)) != APR_SUCCESS) {
cos_warn_log("apr_file_close failure, %s.", apr_strerror(s, buf, sizeof(buf)));
}
t->req->file_buf = NULL;
}
if (t->resp->file_buf != NULL && t->resp->file_buf->owner) {
cos_trace_log("close response body file.");
if ((s = apr_file_close(t->resp->file_buf->file)) != APR_SUCCESS) {
cos_warn_log("apr_file_close failure, %s.", apr_strerror(s, buf, sizeof(buf)));
}
t->resp->file_buf = NULL;
}
}
cos_http_transport_t *cos_curl_http_transport_create(cos_pool_t *p)
{
cos_func_u func;
cos_curl_http_transport_t *t;
t = (cos_curl_http_transport_t *)cos_pcalloc(p, sizeof(cos_curl_http_transport_t));
t->pool = p;
t->options = cos_default_http_transport_options;
t->cleanup = cos_fstack_create(p, 5);
func.func1 = (cos_func1_pt)cos_transport_cleanup;
cos_fstack_push(t->cleanup, t, func, 1);
t->curl = cos_request_get();
func.func1 = (cos_func1_pt)request_release;
cos_fstack_push(t->cleanup, t->curl, func, 1);
t->header_callback = cos_curl_default_header_callback;
t->read_callback = cos_curl_default_read_callback;
t->write_callback = cos_curl_default_write_callback;
return (cos_http_transport_t *)t;
}
static void cos_move_transport_state(cos_curl_http_transport_t *t, cos_transport_state_e s)
{
if (t->state < s) {
t->state = s;
}
}
void cos_curl_response_headers_parse(cos_pool_t *p, cos_table_t *headers, char *buffer, int len)
{
char *pos;
cos_string_t str;
cos_string_t key;
cos_string_t value;
str.data = buffer;
str.len = len;
cos_trip_space_and_cntrl(&str);
pos = cos_strlchr(str.data, str.data + str.len, ':');
if (pos == NULL) {
return;
}
key.data = str.data;
key.len = pos - str.data;
pos += 1;
value.len = str.data + str.len - pos;
value.data = pos;
cos_strip_space(&value);
apr_table_addn(headers, cos_pstrdup(p, &key), cos_pstrdup(p, &value));
}
size_t cos_curl_default_header_callback(char *buffer, size_t size, size_t nitems, void *userdata)
{
int len;
cos_curl_http_transport_t *t;
t = (cos_curl_http_transport_t *)(userdata);
len = size * nitems;
if (t->controller->first_byte_time == 0) {
t->controller->first_byte_time = apr_time_now();
}
cos_curl_response_headers_parse(t->pool, t->resp->headers, buffer, len);
cos_move_transport_state(t, TRANS_STATE_HEADER);
return len;
}
static void cos_curl_transport_headers_done(cos_curl_http_transport_t *t)
{
long http_code;
CURLcode code;
const char *value;
if (t->controller->error_code != COSE_OK) {
cos_debug_log("has error %d.", t->controller->error_code);
return;
}
if (t->resp->status > 0) {
cos_trace_log("http response status %d.", t->resp->status);
return;
}
t->resp->status = 0;
if ((code = curl_easy_getinfo(t->curl, CURLINFO_RESPONSE_CODE, &http_code)) != CURLE_OK) {
t->controller->reason = apr_pstrdup(t->pool, curl_easy_strerror(code));
t->controller->error_code = COSE_INTERNAL_ERROR;
return;
} else {
t->resp->status = http_code;
}
value = apr_table_get(t->resp->headers, "Content-Length");
if (value != NULL) {
t->resp->content_length = cos_atoi64(value);
}
}
size_t cos_curl_default_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
int len;
int bytes;
cos_curl_http_transport_t *t;
t = (cos_curl_http_transport_t *)(userdata);
len = size * nmemb;
if (t->controller->first_byte_time == 0) {
t->controller->first_byte_time = apr_time_now();
}
cos_curl_transport_headers_done(t);
if (t->controller->error_code != COSE_OK) {
cos_debug_log("write callback abort");
return 0;
}
// On HTTP error, we expect to parse an HTTP error response
if (t->resp->status < 200 || t->resp->status > 299) {
bytes = cos_write_http_body_memory(t->resp, ptr, len);
assert(bytes == len);
cos_move_transport_state(t, TRANS_STATE_BODY_IN);
return bytes;
}
if (t->resp->type == BODY_IN_MEMORY && t->resp->body_len >= (int64_t)t->controller->options->max_memory_size) {
t->controller->reason = apr_psprintf(t->pool,
"receive body too big, current body size: %" APR_INT64_T_FMT ", max memory size: %" APR_INT64_T_FMT,
t->resp->body_len, t->controller->options->max_memory_size);
t->controller->error_code = COSE_OVER_MEMORY;
cos_error_log("error reason:%s, ", t->controller->reason);
return 0;
}
if ((bytes = t->resp->write_body(t->resp, ptr, len)) < 0) {
cos_debug_log("write body failure, %d.", bytes);
t->controller->error_code = COSE_WRITE_BODY_ERROR;
t->controller->reason = "write body failure.";
return 0;
}
if (bytes >= 0) {
// progress callback
if (NULL != t->resp->progress_callback) {
t->resp->progress_callback(t->resp->body_len, t->resp->content_length);
}
// crc
if (t->controller->options->enable_crc) {
t->resp->crc64 = cos_crc64(t->resp->crc64, ptr, bytes);
}
}
cos_move_transport_state(t, TRANS_STATE_BODY_IN);
return bytes;
}
size_t cos_curl_default_read_callback(char *buffer, size_t size, size_t nitems, void *instream)
{
int len;
int bytes;
cos_curl_http_transport_t *t;
t = (cos_curl_http_transport_t *)(instream);
len = size * nitems;
if (t->controller->error_code != COSE_OK) {
cos_debug_log("abort read callback.");
return CURL_READFUNC_ABORT;
}
if ((bytes = t->req->read_body(t->req, buffer, len)) < 0) {
cos_debug_log("read body failure, %d.", bytes);
t->controller->error_code = COSE_READ_BODY_ERROR;
t->controller->reason = "read body failure.";
return CURL_READFUNC_ABORT;
}
if (bytes >= 0) {
// progress callback
t->req->consumed_bytes += bytes;
if (NULL != t->req->progress_callback) {
t->req->progress_callback(t->req->consumed_bytes, t->req->body_len);
}
// crc
if (t->controller->options->enable_crc) {
t->req->crc64 = cos_crc64(t->req->crc64, buffer, bytes);
}
}
cos_move_transport_state(t, TRANS_STATE_BODY_OUT);
return bytes;
}
int cos_curl_code_to_status(CURLcode code)
{
switch (code) {
case CURLE_OUT_OF_MEMORY:
return COSE_OUT_MEMORY;
case CURLE_COULDNT_RESOLVE_PROXY:
case CURLE_COULDNT_RESOLVE_HOST:
return COSE_NAME_LOOKUP_ERROR;
case CURLE_COULDNT_CONNECT:
return COSE_FAILED_CONNECT;
case CURLE_WRITE_ERROR:
case CURLE_OPERATION_TIMEDOUT:
return COSE_CONNECTION_FAILED;
case CURLE_PARTIAL_FILE:
return COSE_OK;
case CURLE_SSL_CACERT:
return COSE_FAILED_VERIFICATION;
default:
return COSE_INTERNAL_ERROR;
}
}
static void cos_curl_transport_finish(cos_curl_http_transport_t *t)
{
cos_curl_transport_headers_done(t);
if (t->cleanup != NULL) {
cos_fstack_destory(t->cleanup);
t->cleanup = NULL;
}
}
int cos_curl_transport_setup(cos_curl_http_transport_t *t)
{
CURLcode code;
#define curl_easy_setopt_safe(opt, val) \
if ((code = curl_easy_setopt(t->curl, opt, val)) != CURLE_OK) { \
t->controller->reason = apr_pstrdup(t->pool, curl_easy_strerror(code)); \
t->controller->error_code = COSE_FAILED_INITIALIZE; \
cos_error_log("curl_easy_setopt failed, code:%d %s.", code, t->controller->reason); \
return COSE_FAILED_INITIALIZE; \
}
curl_easy_setopt_safe(CURLOPT_PRIVATE, t);
curl_easy_setopt_safe(CURLOPT_HEADERDATA, t);
curl_easy_setopt_safe(CURLOPT_HEADERFUNCTION, t->header_callback);
curl_easy_setopt_safe(CURLOPT_READDATA, t);
curl_easy_setopt_safe(CURLOPT_READFUNCTION, t->read_callback);
curl_easy_setopt_safe(CURLOPT_WRITEDATA, t);
curl_easy_setopt_safe(CURLOPT_WRITEFUNCTION, t->write_callback);
curl_easy_setopt_safe(CURLOPT_FILETIME, 1);
curl_easy_setopt_safe(CURLOPT_NOSIGNAL, 1);
curl_easy_setopt_safe(CURLOPT_NOPROGRESS, 1);
curl_easy_setopt_safe(CURLOPT_TCP_NODELAY, 1);
curl_easy_setopt_safe(CURLOPT_NETRC, CURL_NETRC_IGNORED);
// transport options
curl_easy_setopt_safe(CURLOPT_SSL_VERIFYHOST, 0);
curl_easy_setopt_safe(CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt_safe(CURLOPT_USERAGENT, t->options->user_agent);
// request options
curl_easy_setopt_safe(CURLOPT_DNS_CACHE_TIMEOUT, t->controller->options->dns_cache_timeout);
curl_easy_setopt_safe(CURLOPT_CONNECTTIMEOUT, t->controller->options->connect_timeout);
curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, t->controller->options->speed_limit);
curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, t->controller->options->speed_time);
cos_init_curl_headers(t);
curl_easy_setopt_safe(CURLOPT_HTTPHEADER, t->headers);
if (t->controller->options->proxy_host != NULL) {
// proxy
curl_easy_setopt_safe(CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
curl_easy_setopt_safe(CURLOPT_PROXY, t->controller->options->proxy_host);
// authorize
if (t->controller->options->proxy_auth != NULL) {
curl_easy_setopt_safe(CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
curl_easy_setopt_safe(CURLOPT_PROXYUSERPWD, t->controller->options->proxy_auth);
}
}
if (NULL == t->req->signed_url) {
if (cos_init_curl_url(t) != COSE_OK) {
return t->controller->error_code;
}
}
else {
t->url = t->req->signed_url;
}
curl_easy_setopt_safe(CURLOPT_URL, t->url);
switch (t->req->method) {
case HTTP_HEAD:
curl_easy_setopt_safe(CURLOPT_NOBODY, 1);
break;
case HTTP_PUT:
curl_easy_setopt_safe(CURLOPT_UPLOAD, 1);
break;
case HTTP_POST:
curl_easy_setopt_safe(CURLOPT_POST, 1);
break;
case HTTP_DELETE:
curl_easy_setopt_safe(CURLOPT_CUSTOMREQUEST, "DELETE");
break;
default: // HTTP_GET
break;
}
#undef curl_easy_setopt_safe
t->state = TRANS_STATE_INIT;
return COSE_OK;
}
int cos_curl_http_transport_perform(cos_http_transport_t *t_)
{
int ecode;
CURLcode code;
cos_curl_http_transport_t *t = (cos_curl_http_transport_t *)(t_);
ecode = cos_curl_transport_setup(t);
if (ecode != COSE_OK) {
return ecode;
}
t->controller->start_time = apr_time_now();
code = curl_easy_perform(t->curl);
t->controller->finish_time = apr_time_now();
cos_move_transport_state(t, TRANS_STATE_DONE);
if ((code != CURLE_OK) && (t->controller->error_code == COSE_OK)) {
ecode = cos_curl_code_to_status(code);
if (ecode != COSE_OK) {
t->controller->error_code = ecode;
t->controller->reason = apr_pstrdup(t->pool, curl_easy_strerror(code));
cos_error_log("transport failure curl code:%d error:%s", code, t->controller->reason);
}
}
cos_curl_transport_finish(t);
return t->controller->error_code;
}

167
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_transport.h vendored
View File

@ -1,167 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_TRANSPORT_H
#define LIBCOS_TRANSPORT_H
#include "cos_sys_define.h"
#include "cos_buf.h"
COS_CPP_START
typedef struct cos_http_request_s cos_http_request_t;
typedef struct cos_http_response_s cos_http_response_t;
typedef struct cos_http_transport_s cos_http_transport_t;
typedef struct cos_http_controller_s cos_http_controller_t;
typedef struct cos_http_request_options_s cos_http_request_options_t;
typedef struct cos_http_transport_options_s cos_http_transport_options_t;
typedef struct cos_curl_http_transport_s cos_curl_http_transport_t;
typedef int (*cos_read_http_body_pt)(cos_http_request_t *req, char *buffer, int len);
typedef int (*cos_write_http_body_pt)(cos_http_response_t *resp, const char *buffer, int len);
typedef void (*cos_progress_callback)(int64_t consumed_bytes, int64_t total_bytes);
void cos_curl_response_headers_parse(cos_pool_t *p, cos_table_t *headers, char *buffer, int len);
cos_http_transport_t *cos_curl_http_transport_create(cos_pool_t *p);
int cos_curl_http_transport_perform(cos_http_transport_t *t);
struct cos_http_request_options_s {
int speed_limit;
int speed_time;
int dns_cache_timeout;
int connect_timeout;
int64_t max_memory_size;
int enable_crc;
int enable_md5;
char *proxy_host;
char *proxy_auth;
char *host_ip;
int host_port;
};
struct cos_http_transport_options_s {
char *user_agent;
char *cacerts_path;
uint32_t ssl_verification_disabled:1;
};
#define COS_HTTP_BASE_CONTROLLER_DEFINE \
cos_http_request_options_t *options; \
cos_pool_t *pool; \
int64_t start_time; \
int64_t first_byte_time; \
int64_t finish_time; \
uint32_t owner:1; \
void *user_data;
struct cos_http_controller_s {
COS_HTTP_BASE_CONTROLLER_DEFINE
};
typedef struct cos_http_controller_ex_s {
COS_HTTP_BASE_CONTROLLER_DEFINE
// private
int error_code;
char *reason; // can't modify
} cos_http_controller_ex_t;
typedef enum {
BODY_IN_MEMORY = 0,
BODY_IN_FILE,
BODY_IN_CALLBACK
} cos_http_body_type_e;
struct cos_http_request_s {
char *host;
char *proto;
char *signed_url;
http_method_e method;
char *uri;
char *resource;
cos_table_t *headers;
cos_table_t *query_params;
cos_list_t body;
int64_t body_len;
char *file_path;
cos_file_buf_t *file_buf;
cos_pool_t *pool;
void *user_data;
cos_read_http_body_pt read_body;
cos_http_body_type_e type;
cos_progress_callback progress_callback;
uint64_t crc64;
int64_t consumed_bytes;
};
struct cos_http_response_s {
int status;
cos_table_t *headers;
cos_list_t body;
int64_t body_len;
char *file_path;
cos_file_buf_t* file_buf;
int64_t content_length;
cos_pool_t *pool;
void *user_data;
cos_write_http_body_pt write_body;
cos_http_body_type_e type;
cos_progress_callback progress_callback;
uint64_t crc64;
};
typedef enum {
TRANS_STATE_INIT,
TRANS_STATE_HEADER,
TRANS_STATE_BODY_IN,
TRANS_STATE_BODY_OUT,
TRANS_STATE_ABORT,
TRANS_STATE_DONE
} cos_transport_state_e;
#define COS_HTTP_BASE_TRANSPORT_DEFINE \
cos_http_request_t *req; \
cos_http_response_t *resp; \
cos_pool_t *pool; \
cos_transport_state_e state; \
cos_array_header_t *cleanup; \
cos_http_transport_options_t *options; \
cos_http_controller_ex_t *controller;
struct cos_http_transport_s {
COS_HTTP_BASE_TRANSPORT_DEFINE
};
struct cos_curl_http_transport_s {
COS_HTTP_BASE_TRANSPORT_DEFINE
CURL *curl;
char *url;
struct curl_slist *headers;
curl_read_callback header_callback;
curl_read_callback read_callback;
curl_write_callback write_callback;
};
COS_CPP_END
#endif

1464
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_utility.c vendored
View File

File diff suppressed because it is too large Load Diff

426
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_utility.h vendored
View File

@ -1,426 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_UTILITY_H
#define LIBCOS_UTILITY_H
#include "cos_string.h"
#include "cos_transport.h"
#include "cos_status.h"
#include "cos_define.h"
#include "cos_resumable.h"
COS_CPP_START
#define init_sts_token_header() do { \
if (options->config->sts_token.data != NULL) {\
apr_table_set(headers, COS_STS_SECURITY_TOKEN, options->config->sts_token.data);\
}\
} while(0)
/**
* @brief check hostname ends with specific cos domain suffix.
**/
int is_cos_domain(const cos_string_t *str);
/**
* @brief check hostname is ip.
**/
int is_valid_ip(const char *str);
/**
* @brief get cos acl str according cos_acl
* @param[in] cos_acl the cos bucket acl
* @return cos acl str
**/
const char *get_cos_acl_str(cos_acl_e cos_acl);
/**
* @brief create cos config including host, port, access_key_id, access_key_secret, is_cos_domain
**/
cos_config_t *cos_config_create(cos_pool_t *p);
/**
* @brief evaluate config to curl
**/
void cos_config_resolve(cos_pool_t *pool, cos_config_t *config, cos_http_controller_t *ctl);
/**
* @brief create cos request options
* @return cos request options
**/
cos_request_options_t *cos_request_options_create(cos_pool_t *p);
/**
* @brief init cos request
**/
void cos_init_request(const cos_request_options_t *options, http_method_e method,
cos_http_request_t **req, cos_table_t *params, cos_table_t *headers, cos_http_response_t **resp);
/**
* @brief init cos service request
**/
void cos_init_service_request(const cos_request_options_t *options, http_method_e method,
cos_http_request_t **req, cos_table_t *params, cos_table_t *headers, const int all_region, cos_http_response_t **resp);
/**
* @brief init cos bucket request
**/
void cos_init_bucket_request(const cos_request_options_t *options, const cos_string_t *bucket,
http_method_e method, cos_http_request_t **req, cos_table_t *params, cos_table_t *headers,
cos_http_response_t **resp);
/**
* @brief init cos object request
**/
void cos_init_object_request(const cos_request_options_t *options, const cos_string_t *bucket,
const cos_string_t *object, http_method_e method, cos_http_request_t **req,
cos_table_t *params, cos_table_t *headers, cos_progress_callback cb, uint64_t initcrc,
cos_http_response_t **resp);
/**
* @brief init cos live channel request
**/
void cos_init_live_channel_request(const cos_request_options_t *options,
const cos_string_t *bucket, const cos_string_t *live_channel,
http_method_e method, cos_http_request_t **req, cos_table_t *params,
cos_table_t *headers, cos_http_response_t **resp);
/**
* @brief init cos request with signed_url
**/
void cos_init_signed_url_request(const cos_request_options_t *options, const cos_string_t *signed_url,
http_method_e method, cos_http_request_t **req,
cos_table_t *params, cos_table_t *headers, cos_http_response_t **resp);
/**
* @brief cos send request
**/
cos_status_t *cos_send_request(cos_http_controller_t *ctl, cos_http_request_t *req,
cos_http_response_t *resp);
/**
* @brief process cos request including sign request, send request, get response
**/
cos_status_t *cos_process_request(const cos_request_options_t *options,
cos_http_request_t *req, cos_http_response_t *resp);
/**
* @brief process cos request with signed_url including send request, get response
**/
cos_status_t *cos_process_signed_request(const cos_request_options_t *options,
cos_http_request_t *req, cos_http_response_t *resp);
/**
* @brief get object uri using third-level domain if hostname is cos domain, otherwise second-level domain
**/
void cos_get_object_uri(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *object,
cos_http_request_t *req);
/**
* @brief bucket uri using third-level domain if hostname is cos domain, otherwise second-level domain
**/
void cos_get_bucket_uri(const cos_request_options_t *options,
const cos_string_t *bucket,
cos_http_request_t *req);
/**
* @brief service uri
**/
void cos_get_service_uri(const cos_request_options_t *options,
const int all_region,
cos_http_request_t *req);
/**
* @brief get rtmp uri using third-level domain if hostname is cos domain, otherwise second-level domain
**/
void cos_get_rtmp_uri(const cos_request_options_t *options,
const cos_string_t *bucket,
const cos_string_t *live_channel_id,
cos_http_request_t *req);
/**
* @brief write body content into cos request body from buffer
**/
void cos_write_request_body_from_buffer(cos_list_t *buffer, cos_http_request_t *req);
/**
* @brief write body content into cos request body from file
**/
int cos_write_request_body_from_file(cos_pool_t *p, const cos_string_t *filename, cos_http_request_t *req);
/**
* @brief write body content into cos request body from multipart upload file
**/
int cos_write_request_body_from_upload_file(cos_pool_t *p, cos_upload_file_t *upload_file, cos_http_request_t *req);
/**
* @brief read body content from cos response body to buffer
**/
void cos_fill_read_response_body(cos_http_response_t *resp, cos_list_t *buffer);
/**
* @brief read body content from cos response body to file
**/
int cos_init_read_response_body_to_file(cos_pool_t *p, const cos_string_t *filename, cos_http_response_t *resp);
/**
* @brief read response header if headers is not null
**/
void cos_fill_read_response_header(cos_http_response_t *resp, cos_table_t **headers);
/**
* @brief create cos api result content
* @return cos api result content
**/
void *cos_create_api_result_content(cos_pool_t *p, size_t size);
cos_acl_grantee_content_t *cos_create_acl_list_content(cos_pool_t *p);
cos_get_service_content_t *cos_create_get_service_content(cos_pool_t *p);
cos_list_object_content_t *cos_create_list_object_content(cos_pool_t *p);
cos_list_object_common_prefix_t *cos_create_list_object_common_prefix(cos_pool_t *p);
cos_list_part_content_t *cos_create_list_part_content(cos_pool_t *p);
cos_list_multipart_upload_content_t *cos_create_list_multipart_upload_content(cos_pool_t *p);
cos_complete_part_content_t *cos_create_complete_part_content(cos_pool_t *p);
/**
* @brief create cos api get service parameters
* @return cos api get service parameters
**/
cos_get_service_params_t *cos_create_get_service_params(cos_pool_t *p);
/**
* @brief create cos api list parameters
* @return cos api list parameters
**/
cos_list_object_params_t *cos_create_list_object_params(cos_pool_t *p);
cos_list_upload_part_params_t *cos_create_list_upload_part_params(cos_pool_t *p);
cos_list_multipart_upload_params_t *cos_create_list_multipart_upload_params(cos_pool_t *p);
cos_list_live_channel_params_t *cos_create_list_live_channel_params(cos_pool_t *p);
cos_acl_params_t *cos_create_acl_params(cos_pool_t *p);
cos_copy_object_params_t *cos_create_copy_object_params(cos_pool_t *p);
/**
* @brief create upload part copy params
* @return upload part copy params struct for upload part copy
**/
cos_upload_part_copy_params_t *cos_create_upload_part_copy_params(cos_pool_t *p);
/**
* @brief create upload file struct for range multipart upload
* @return upload file struct for range multipart upload
**/
cos_upload_file_t *cos_create_upload_file(cos_pool_t *p);
/**
* @brief get content-type for HTTP_POST request
* @return content-type for HTTP_POST request
**/
void cos_set_multipart_content_type(cos_table_t *headers);
/**
* @brief create lifecycle rule content
* @return lifecycle rule content
**/
cos_lifecycle_rule_content_t *cos_create_lifecycle_rule_content(cos_pool_t *p);
/**
* @brief create cors rule content
* @return cors rule content
**/
cos_cors_rule_content_t *cos_create_cors_rule_content(cos_pool_t *p);
/**
* @brief create versioning content
* @return bucket versioning content
**/
cos_versioning_content_t *cos_create_versioning_content(cos_pool_t *p);
/**
* @brief create replication rule content
* @return replication rule content
**/
cos_replication_rule_content_t *cos_create_replication_rule_content(cos_pool_t *p);
/**
* @brief create replication param
* @return replication param
**/
cos_replication_params_t *cos_create_replication_params(cos_pool_t *p);
/**
* @brief create website rule content
*/
cos_website_rule_content_t *cos_create_website_rule_content(cos_pool_t *P);
/**
* @brief create website params
*/
cos_website_params_t *cos_create_website_params(cos_pool_t *p);
/**
* @brief create domain params
*/
cos_domain_params_t *cos_create_domain_params(cos_pool_t *p);
// @brief create logging params
cos_logging_params_t *cos_create_logging_params(cos_pool_t *p);
// @brief create inventory params
cos_list_inventory_params_t *cos_create_list_inventory_params(cos_pool_t *p);
cos_inventory_params_t *cos_create_inventory_params(cos_pool_t *p);
cos_inventory_optional_t *cos_create_inventory_optional(cos_pool_t *p);
// @brief create tagging params
cos_tagging_params_t *cos_create_tagging_params(cos_pool_t *p);
cos_tagging_tag_t *cos_create_tagging_tag(cos_pool_t *p);
// @brief create intelligenttiering params
cos_intelligenttiering_params_t *cos_create_intelligenttiering_params(cos_pool_t *p);
cos_object_restore_params_t *cos_create_object_restore_params(cos_pool_t *p);
/**
* @brief create cos object content for delete objects
* @return cos object content
**/
cos_object_key_t *cos_create_cos_object_key(cos_pool_t *p);
/**
* @brief create cos live channel publish url content for delete objects
* @return cos live channel publish url content
**/
cos_live_channel_publish_url_t *cos_create_live_channel_publish_url(cos_pool_t *p);
/**
* @brief create cos live channel play url content for delete objects
* @return cos live channel play url content
**/
cos_live_channel_play_url_t *cos_create_live_channel_play_url(cos_pool_t *p);
/**
* @brief create cos list live channel content for delete objects
* @return cos list live channel content
**/
cos_live_channel_content_t *cos_create_list_live_channel_content(cos_pool_t *p);
/**
* @brief create cos live recored content for delete objects
* @return cos live record content
**/
cos_live_record_content_t *cos_create_live_record_content(cos_pool_t *p);
/**
* @brief create live channel configuration content
* @return live channel configuration content
**/
cos_live_channel_configuration_t *cos_create_live_channel_configuration_content(cos_pool_t *p);
/**
* @brief create cos checkpoint content
* @return cos checkpoint content
**/
cos_checkpoint_t *cos_create_checkpoint_content(cos_pool_t *p);
/**
* @brief create cos resumable clt params content
* @return cos checkpoint content
**/
cos_resumable_clt_params_t *cos_create_resumable_clt_params_content(cos_pool_t *p, int64_t part_size, int32_t thread_num,
int enable_checkpoint, const char *checkpoint_path);
/**
* @brief get part size for multipart upload
**/
void cos_get_part_size(int64_t filesize, int64_t *part_size);
/**
* @brief compare function for part sort
**/
int part_sort_cmp(const void *a, const void *b);
/**
* @brief set content type for object according to objectname
* @return cos content type
**/
char *get_content_type(const char *name);
char *get_content_type_by_suffix(const char *suffix);
/**
* @brief set content type for object according to filename
**/
void set_content_type(const char* filename, const char* key, cos_table_t *headers);
cos_table_t* cos_table_create_if_null(const cos_request_options_t *options,
cos_table_t *table, int table_size);
int is_enable_crc(const cos_request_options_t *options);
int is_enable_md5(const cos_request_options_t *options);
int has_crc_in_response(const cos_http_response_t *resp);
int has_range_or_process_in_request(const cos_http_request_t *req) ;
/**
* @brief check crc consistent between client and server
**/
int cos_check_crc_consistent(uint64_t crc, const apr_table_t *resp_headers, cos_status_t *s);
int cos_check_len_consistent(cos_list_t *buffer, const apr_table_t *resp_headers, cos_status_t *s);
int cos_get_temporary_file_name(cos_pool_t *p, const cos_string_t *filename, cos_string_t *temp_file_name);
int cos_temp_file_rename(cos_status_t *s, const char *from_path, const char *to_path, apr_pool_t *pool);
int cos_init_read_response_body_to_file_part(cos_pool_t *p,
cos_upload_file_t *download_file,
cos_http_response_t *resp);
/**
* @brief add Content-MD5 header, md5 calculated from buffer
**/
int cos_add_content_md5_from_buffer(const cos_request_options_t *options,
cos_list_t *buffer,
cos_table_t *headers);
/**
* @brief add Content-MD5 header, md5 calculated from file
**/
int cos_add_content_md5_from_file(const cos_request_options_t *options,
const cos_string_t *filename,
cos_table_t *headers);
/**
* @brief add Content-MD5 header, md5 calculated from file range
**/
int cos_add_content_md5_from_file_range(const cos_request_options_t *options,
cos_upload_file_t *upload_file,
cos_table_t *headers);
/**
* @brief set flag of adding Content-MD5 header
* @param[in] enable COS_TRUE: sdk will add Content-MD5 automatically; COS_FALSE:sdk does not add Content-MD5
**/
void cos_set_content_md5_enable(cos_http_controller_t *ctl, int enable);
/**
* @brief set route address param in request options
* @param[in] host_ip, string of route ip with '\0' ending, ip-port will not be applied if host_ip is NULL
* @param[in] host_port, the route port, ip-port will not be applied if host_port is a none-positive integer
**/
void cos_set_request_route(cos_http_controller_t *ctl, char *host_ip, int host_port);
COS_CPP_END
#endif

2731
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_xml.c vendored
View File

File diff suppressed because it is too large Load Diff

364
deps/oblib/src/lib/restore/cos/cos_c_sdk/cos_xml.h vendored
View File

@ -1,364 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef LIBCOS_XML_H
#define LIBCOS_XML_H
#include <mxml.h>
#include "cos_string.h"
#include "cos_transport.h"
#include "cos_status.h"
#include "cos_define.h"
#include "cos_resumable.h"
COS_CPP_START
/**
* @brief functions for xml body parse
**/
int get_xmldoc(cos_list_t *bc, mxml_node_t **root);
char *get_xmlnode_value(cos_pool_t *p, mxml_node_t * root, const char *xml_path);
/**
* @brief build xml body for complete_multipart_upload
**/
char *build_complete_multipart_upload_xml(cos_pool_t *p, cos_list_t *bc);
/**
* @brief build body for complete multipart upload
**/
void build_complete_multipart_upload_body(cos_pool_t *p, cos_list_t *part_list, cos_list_t *body);
/**
* @brief build xml body for put lifecycle
**/
char *build_lifecycle_xml(cos_pool_t *p, cos_list_t *lifecycle_rule_list);
/**
* @brief build body for put lifecycle
**/
void build_lifecycle_body(cos_pool_t *p, cos_list_t *lifecycle_rule_list, cos_list_t *body);
/**
* @brief build xml body for put cors
**/
char *build_cors_xml(cos_pool_t *p, cos_list_t *cors_rule_list);
/**
* @brief build body for put cors
**/
void build_cors_body(cos_pool_t *p, cos_list_t *cors_rule_list, cos_list_t *body);
/**
* @brief build xml body for put bucket replication
**/
char *build_replication_xml(cos_pool_t *p, cos_replication_params_t *replication_param);
/**
* @brief build body for put bucket replication
**/
void build_replication_body(cos_pool_t *p, cos_replication_params_t *replication_param, cos_list_t *body);
/**
* @brief build body for put bucket versioning
**/
void build_versioning_body(cos_pool_t *p, cos_versioning_content_t *versioning, cos_list_t *body);
/**
* @brief build xml body for put bucket versioning
**/
char *build_versioning_xml(cos_pool_t *p, cos_versioning_content_t *versioning);
/**
* @brief build a xml node
* eg: <xml>param->data</xml>
**/
void build_xml_node(mxml_node_t *pnode, const char *xml, cos_string_t *param);
/** @brief build a xml node with parent.
* eg: <pxml><cxml>pamam->data</cxml></pxml>
**/
void build_xml_node_with_parent(mxml_node_t *root, const char *pxml, const char *cxml, cos_string_t *param);
/**
* @brief build body for put bucket website
**/
void build_website_body(cos_pool_t *p, cos_website_params_t *website_params, cos_list_t *body);
/**
* @brief build xml body for put bucket website
**/
char *build_website_xml(cos_pool_t *p, cos_website_params_t *website_params);
/**
* @brief build body for put bucket domain
**/
void build_domain_body(cos_pool_t *p, cos_domain_params_t *domain_params, cos_list_t *body);
/**
* @brief build xml body for put bucket domain
**/
char *build_domain_xml(cos_pool_t *p, cos_domain_params_t *domain_params);
/**
* @brief build body for put bucket logging
*/
void build_logging_body(cos_pool_t *p, cos_logging_params_t *params, cos_list_t *body);
char *build_logging_xml(cos_pool_t *p, cos_logging_params_t *params);
/**
* @brief build body for put bucket logging
*/
void build_inventory_body(cos_pool_t *p, cos_inventory_params_t *params, cos_list_t *body);
char *build_inventory_xml(cos_pool_t *p, cos_inventory_params_t *params);
void build_tagging_body(cos_pool_t *p, cos_tagging_params_t *params, cos_list_t *body);
char *build_tagging_xml(cos_pool_t *p, cos_tagging_params_t *params);
void build_intelligenttiering_body(cos_pool_t *p, cos_intelligenttiering_params_t *params, cos_list_t *body);
char *build_intelligenttiering_xml(cos_pool_t *p, cos_intelligenttiering_params_t *params);
void build_object_restore_body(cos_pool_t *p, cos_object_restore_params_t *params, cos_list_t *body);
/**
* @brief build xml body for delete objects
**/
char *build_objects_xml(cos_pool_t *p, cos_list_t *object_list, const char *quiet);
/**
* @brief build body for delete objects
**/
void build_delete_objects_body(cos_pool_t *p, cos_list_t *object_list, int is_quiet,
cos_list_t *body);
mxml_node_t *set_xmlnode_value_str(mxml_node_t *parent, const char *name, const cos_string_t *value);
mxml_node_t *set_xmlnode_value_int(mxml_node_t *parent, const char *name, int value);
mxml_node_t *set_xmlnode_value_int64(mxml_node_t *parent, const char *name, int64_t value);
int get_xmlnode_value_str(cos_pool_t *p, mxml_node_t *xml_node, const char *xml_path, cos_string_t *value);
int get_xmlnode_value_int(cos_pool_t *p, mxml_node_t *xml_node, const char *xml_path, int *value);
int get_xmlnode_value_int64(cos_pool_t *p, mxml_node_t *xml_node, const char *xml_path, int64_t *value);
/**
* @brief build xml for checkpoint
**/
char *cos_build_checkpoint_xml(cos_pool_t *p, const cos_checkpoint_t *checkpoint);
/**
* @bried parse checkpoint from xml
**/
int cos_checkpoint_parse_from_body(cos_pool_t *p, const char *xml_body, cos_checkpoint_t *checkpoint);
/**
* @bried parse acl from xml body for get_bucket_acl
**/
int cos_acl_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_acl_params_t *content);
/**
* @bried parse acl from xml body for get_bucket_acl
**/
void cos_acl_grantee_content_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_acl_grantee_content_t *content);
/**
* @bried parse acl from xml body for get_bucket_acl
**/
void cos_acl_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path, cos_list_t *acl_list);
/**
* @bried parse acl from xml body for get_bucket_acl
**/
void cos_acl_owner_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_acl_params_t *content);
/**
* @bried parse result from xml body for copy_object
**/
int cos_copy_object_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_copy_object_params_t *content);
/**
* @brief parse upload_id from xml body for init multipart upload
**/
int cos_upload_id_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_string_t *upload_id);
/**
* @brief parse buckets list from xml body for get services
**/
int cos_get_service_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_get_service_params_t *params);
/**
* @brief parse objects from xml body for list objects
**/
void cos_list_objects_owner_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_list_object_content_t *content);
void cos_list_objects_content_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_list_object_content_t *content);
void cos_list_objects_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *object_list);
void cos_list_objects_prefix_parse(cos_pool_t *p, mxml_node_t *root,
cos_list_object_common_prefix_t *common_prefix);
void cos_list_objects_common_prefix_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *common_prefix_list);
int cos_list_objects_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_list_t *object_list,
cos_list_t *common_prefix_list, cos_string_t *marker, int *truncated);
/**
* @brief parse parts from xml body for list upload part
**/
void cos_list_parts_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *part_list);
void cos_list_parts_content_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_list_part_content_t *content);
int cos_list_parts_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_list_t *part_list,
cos_string_t *part_number_marker, int *truncated);
/**
* @brief parse uploads from xml body for list multipart upload
**/
void cos_list_multipart_uploads_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *upload_list);
void cos_list_multipart_uploads_content_parse(cos_pool_t *p, mxml_node_t *xml_node,
cos_list_multipart_upload_content_t *content);
int cos_list_multipart_uploads_parse_from_body(cos_pool_t *p, cos_list_t *bc,
cos_list_t *upload_list, cos_string_t *key_marker,
cos_string_t *upload_id_marker, int *truncated);
/**
* @brief parse cors rules from xml body
**/
int cos_cors_rules_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_list_t *cors_rule_list);
void cos_cors_rule_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path, cos_list_t *cors_rule_list);
void cos_cors_rule_content_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_cors_rule_content_t *content);
/**
* @brief parse bucket replication rules from xml body
**/
void cos_replication_rule_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_replication_rule_content_t *content);
void cos_replication_rules_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path, cos_list_t *rule_list);
int cos_replication_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_replication_params_t *content);
/**
* @brief parse versioning from body
**/
int cos_versioning_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_versioning_content_t *versioning);
/**
* @brief parse lifecycle rules from xml body
**/
void cos_lifecycle_rule_expire_parse(cos_pool_t *p, mxml_node_t *xml_node,
cos_lifecycle_rule_content_t *content);
void cos_lifecycle_rule_transition_parse(cos_pool_t *p, mxml_node_t *xml_node,
cos_lifecycle_rule_content_t *content);
void cos_lifecycle_rule_abort_parse(cos_pool_t *p, mxml_node_t *xml_node,
cos_lifecycle_rule_content_t *content);
void cos_lifecycle_rule_content_parse(cos_pool_t *p, mxml_node_t *xml_node,
cos_lifecycle_rule_content_t *content);
void cos_lifecycle_rule_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *lifecycle_rule_list);
int cos_lifecycle_rules_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_list_t *lifecycle_rule_list);
/**
* @brief parse from a xml node.
* eg: <xml>text</xml>
**/
void cos_common_parse_from_xml_node(cos_pool_t *p, mxml_node_t *pnode, mxml_node_t *root, const char *xml, cos_string_t *param);
/**
* @brief parse from a parent xml node.
* eg: <pxml><cxml>test<cxml></pxml>
**/
void cos_common_parse_from_parent_node(cos_pool_t *p, mxml_node_t *root, const char *pxml, const char *cxml, cos_string_t *param);
/**
* @brief parse website from body
**/
int cos_get_website_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_website_params_t *website);
/**
* @brief parse domain form body
**/
int cos_get_domain_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_domain_params_t *domain);
/**
* @brief parse logging from body
*/
int cos_get_logging_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_logging_params_t *logging);
// @brief parse inventory from body
int cos_get_inventory_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_inventory_params_t *params);
int cos_list_inventory_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_list_inventory_params_t *params);
int cos_get_tagging_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_tagging_params_t *params);
int cos_get_intelligenttiering_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_intelligenttiering_params_t *params);
/**
* @brief parse delete objects contents from xml body
**/
void cos_delete_objects_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *object_list);
void cos_object_key_parse(cos_pool_t *p, mxml_node_t * xml_node, cos_object_key_t *content);
int cos_delete_objects_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_list_t *object_list);
/**
* @brief build body for create live channel
**/
char *build_create_live_channel_xml(cos_pool_t *p, cos_live_channel_configuration_t *config);
void build_create_live_channel_body(cos_pool_t *p, cos_live_channel_configuration_t *config, cos_list_t *body);
/**
* @brief parse create live channel contents from xml body
**/
void cos_publish_url_parse(cos_pool_t *p, mxml_node_t *node, cos_live_channel_publish_url_t *content);
void cos_play_url_parse(cos_pool_t *p, mxml_node_t *node, cos_live_channel_play_url_t *content);
void cos_publish_urls_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *publish_xml_list);
void cos_play_urls_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *play_xml_list);
void cos_create_live_channel_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *publish_xml_path,
cos_list_t *publish_url_list, const char *play_xml_path, cos_list_t *play_url_list);
int cos_create_live_channel_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_list_t *publish_url_list,
cos_list_t *play_url_list);
/**
* @brief parse live channel info content from xml body
**/
void cos_live_channel_info_target_content_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_live_channel_target_t *target);
void cos_live_channel_info_content_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_live_channel_configuration_t *info);
int cos_live_channel_info_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_live_channel_configuration_t *info);
/**
* @brief parse live channel stat content from xml body
**/
void cos_live_channel_stat_video_content_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_video_stat_t *video_stat);
void cos_live_channel_stat_audio_content_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_audio_stat_t *audio_stat);
void cos_live_channel_stat_content_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path, cos_live_channel_stat_t *stat);
int cos_live_channel_stat_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_live_channel_stat_t *stat);
/**
* @brief parse live channel from xml body for list list channel
**/
void cos_list_live_channel_content_parse(cos_pool_t *p, mxml_node_t *xml_node, cos_live_channel_content_t *content);
void cos_list_live_channel_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *live_channel_list);
int cos_list_live_channel_parse_from_body(cos_pool_t *p, cos_list_t *bc,
cos_list_t *live_channel_list, cos_string_t *next_marker, int *truncated);
/**
* @brief parse live channel history content from xml body
**/
void cos_live_channel_history_content_parse(cos_pool_t *p, mxml_node_t * xml_node, cos_live_record_content_t *content);
void cos_live_channel_history_contents_parse(cos_pool_t *p, mxml_node_t *root, const char *xml_path,
cos_list_t *live_record_list);
int cos_live_channel_history_parse_from_body(cos_pool_t *p, cos_list_t *bc, cos_list_t *live_record_list);
COS_CPP_END
#endif

1048
deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp vendored
View File

File diff suppressed because it is too large Load Diff

264
deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h vendored
View File

@ -1,264 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef SRC_LIBRARY_SRC_LIB_OB_COS_WRAPPER_H_
#define SRC_LIBRARY_SRC_LIB_OB_COS_WRAPPER_H_
#include <string.h>
#include "ob_singleton.h"
#include <dirent.h>
namespace oceanbase
{
namespace common
{
// Naming qcloud_cos to avoid conflicts with the function called cos.
namespace qcloud_cos
{
#define OB_PUBLIC_API __attribute__ ((visibility ("default")))
class OB_PUBLIC_API ObCosEnv : public ObSingleton<ObCosEnv>
{
public:
struct Conf
{
static const int64_t MIN_COS_SLICE_SIZE = 32000;
static const int64_t MAX_COS_SLICE_SIZE = 10000000;
int64_t slice_size = 524288; // default 512K
};
ObCosEnv() : ObSingleton<ObCosEnv>(), is_inited_(false) {}
// global init cos env resource, must and only can be called once
int init();
void fin();
int set_slice_size(const int64_t size);
int64_t get_slice_size() const;
private:
bool is_inited_;
Conf conf_;
};
// COS domain name structure: bucket_name-appid.endpoint
// cos.ap-guangzhou.myqcloud.com is the endpoint.
struct OB_PUBLIC_API ObCosAccount
{
// max domain length
static constexpr int MAX_COS_DOMAIN_LENGTH = 1536;
// max endpoint length
static constexpr int MAX_COS_ENDPOINT_LENGTH = 128;
// max access id length
static constexpr int MAX_COS_ACCESS_ID_LENGTH = 128;
// max access key length
static constexpr int MAX_COS_ACCESS_KEY_LENGTH = 128;
// max appid length
static constexpr int MAX_COS_APPID_LENGTH = 128;
// cos endpoint
char endpoint[MAX_COS_ENDPOINT_LENGTH];
// your access id
char access_id[MAX_COS_ACCESS_ID_LENGTH];
// your access key
char access_key[MAX_COS_ACCESS_KEY_LENGTH];
// your appid
char appid[MAX_COS_APPID_LENGTH];
ObCosAccount()
{
memset(this, 0, sizeof(*this));
}
~ObCosAccount() {}
void clear()
{
memset(this, 0, sizeof(*this));
}
// parse endpoint, access id, access key and appid from storage_info
// You must call parse_from first before using any field.
int parse_from(const char *storage_info, uint32_t size);
private:
// make sure 'value' end with '\0'
int set_field(const char *value, char *field, uint32_t length);
};
// The string does not own the buffer.
struct OB_PUBLIC_API CosStringBuffer
{
const char *data;
int32_t size;
CosStringBuffer() : data(NULL), size(0) {}
CosStringBuffer(const char *ptr, int len) : data(ptr), size(len) {}
bool empty() const
{
return NULL == data || 0 >= size;
}
~CosStringBuffer() {}
};
struct OB_PUBLIC_API CosObjectMeta
{
int64_t file_length;
int64_t last_modified_ts;
int type;
CosObjectMeta()
{
reset();
}
~CosObjectMeta() {}
void reset()
{
memset(this, 0, sizeof(*this));
file_length = -1;
}
};
/*= Custom memory allocation functions */
typedef void* (*OB_COS_allocFunction) (void* opaque, size_t size);
typedef void (*OB_COS_freeFunction) (void* opaque, void* address);
typedef struct { OB_COS_allocFunction customAlloc; OB_COS_freeFunction customFree; void* opaque; } OB_COS_customMem;
class OB_PUBLIC_API ObCosWrapper
{
public:
struct Handle {};
// alloc_f is used to allocate the handle's memory.
// You need to call destroy_cos_handle when handle is not use
// any more.
static int create_cos_handle(
OB_COS_customMem &custom_mem,
const struct ObCosAccount &account,
Handle **h);
// You can not use handle any more after destroy_cos_handle is called.
static void destroy_cos_handle(Handle *h);
// Put an object to cos, the new object will overwrite the old one if object exist.
static int put(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &object_name,
const char *buffer,
const int64_t size);
// Get object meta
static int head_object_meta(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &object_name,
bool &is_exist,
CosObjectMeta &meta);
// Delete one object from cos.
static int del(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &object_name);
// Delete all objects that match the same dir_name prefix.
static int del_objects_in_dir(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &dir_name,
int64_t &deleted_cnt);
// Update object last modofied time.
static int update_object_modified_ts(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &object_name);
// Random read data of specific range from object.
// Buffer's memory is provided by user.
static int pread(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &object_name,
int64_t offset,
char *buf,
int64_t buf_size,
int64_t &read_size);
// Get whole object
// Buffer's memory is provided by user.
static int get_object(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &object_name,
char *buf,
int64_t buf_size,
int64_t &read_size);
struct CosListObjPara {
CosListObjPara() : arg(NULL), cur_full_path_slice_name(NULL),
full_path_size(0), cur_object_size(0), next_flag(false)
{
last_container_name.d_name[0] = '\0';
last_container_name.d_type = DT_REG;
last_container_name.d_reclen = sizeof(struct dirent);
}
void* arg;
char* cur_full_path_slice_name;
struct dirent last_container_name;
int64_t full_path_size;
int64_t cur_object_size;
bool next_flag;
};
typedef int (*handleObjectNameFunc)(CosListObjPara&);
typedef int (*handleDirectoryFunc)(void*, const char*, int64_t);
// List objects in the same directory, include all objects
// in the inner sub directories.
// dir_name must be end with "/\0".
static int list_objects(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &dir_name,
const CosStringBuffer &next_marker,
handleObjectNameFunc handle_object_name_f,
void *arg);
static int list_directories(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &dir_name,
const CosStringBuffer &next_marker,
const CosStringBuffer &delimiter,
handleDirectoryFunc handle_directory_name_f,
void *arg);
};
#undef OB_PUBLIC_API
}
}
}
#endif

50
deps/oblib/src/lib/restore/cos/ob_singleton.h vendored
View File

@ -1,50 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef SRC_LIBRARY_SRC_LIB_OB_SINGLETON_H_
#define SRC_LIBRARY_SRC_LIB_OB_SINGLETON_H_
namespace oceanbase
{
namespace common
{
namespace qcloud_cos
{
// A singleton base class offering an easy way to create singleton.
template <typename T>
class __attribute__ ((visibility ("default"))) ObSingleton
{
public:
// not thread safe
static T& get_instance()
{
static T instance;
return instance;
}
virtual ~ObSingleton() {}
protected:
ObSingleton() {}
private:
ObSingleton(const ObSingleton &);
ObSingleton& operator=(const ObSingleton&);
};
}
}
}
#endif

293
deps/oblib/src/lib/restore/cos/ob_storage_cos_obj.cpp vendored
View File

@ -1,293 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "lib/utility/ob_print_utils.h"
#include "ob_storage_cos_obj.h"
namespace oceanbase
{
namespace common
{
/**
* ------------------------------ObCosCtxAllocator---------------------
*/
ObCosCtxAllocator::ObCosCtxAllocator()
: allocator_(ObModIds::BACKUP, OB_MALLOC_NORMAL_BLOCK_SIZE)
{
}
ObCosCtxAllocator::~ObCosCtxAllocator()
{
}
void* ObCosCtxAllocator::alloc(size_t size)
{
return allocator_.alloc(size);
}
void ObCosCtxAllocator::free(void *addr)
{
allocator_.free(addr);
}
void ObCosCtxAllocator::reuse()
{
allocator_.reuse();
}
// Memory function used for cos context
static void *ob_cos_malloc(void *opaque, size_t size)
{
void *buf = NULL;
if (NULL != opaque) {
ObCosCtxAllocator *allocator = reinterpret_cast<ObCosCtxAllocator*> (opaque);
buf = allocator->alloc(size);
}
return buf;
}
static void ob_cos_free(void *opaque, void *address)
{
if (NULL != opaque) {
ObCosCtxAllocator *allocator = reinterpret_cast<ObCosCtxAllocator*> (opaque);
allocator->free(address);
}
}
/**
* ------------------------------ObCosBase---------------------
*/
int ObCosBase::build_account(const ObString &storage_info)
{
int ret = OB_SUCCESS;
if (NULL == account_) {
account_ = static_cast<qcloud_cos::ObCosAccount *>(cos_ctx_allocator_.alloc(sizeof(qcloud_cos::ObCosAccount)));
if (NULL == account_) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(WARN, "failed to alloc account", K(ret), K(storage_info));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(account_->parse_from(storage_info.ptr(), storage_info.length()))) {
OB_LOG(WARN, "failed to parse storage info", K(ret), K(storage_info));
}
}
return ret;
}
int ObCosObject::build_bucket_and_object_name(const ObString &uri)
{
int ret = OB_SUCCESS;
ObString::obstr_size_t bucket_start = 0;
ObString::obstr_size_t bucket_end = 0;
ObString::obstr_size_t object_start = 0;
char* bucket_name_buff = NULL;
char* object_name_buff = NULL;
if (!uri.prefix_match(OB_COS_PREFIX)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "uri is invalid ", KCSTRING(OB_COS_PREFIX), K(uri), K(ret));
} else {
bucket_start = static_cast<ObString::obstr_size_t>(strlen(OB_COS_PREFIX));
for (int64_t i = bucket_start; OB_SUCC(ret) && i < uri.length() - 2; i++) {
if ('/' == *(uri.ptr() + i) && '/' == *(uri.ptr() + i + 1)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "uri has two // ", K(uri), K(ret), K(i));
break;
}
}
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(bucket_name_buff = static_cast<char *>(allocator_.alloc(OB_MAX_URI_LENGTH)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(WARN, "failed to alloc bucket name buff", K(ret), K(uri));
} else if (OB_ISNULL(object_name_buff = static_cast<char *>(allocator_.alloc(OB_MAX_URI_LENGTH)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(WARN, "failed to alloc object name buff", K(ret), K(uri));
} else {
//make sure this last always 0
bucket_name_buff[0] = '\0';
object_name_buff[0] = '\0';
}
}
// parse bucket name
if (OB_SUCC(ret)) {
if (bucket_start >= uri.length()) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "bucket name is NULL", K(uri), K(ret));
} else {
for (bucket_end = bucket_start; OB_SUCC(ret) && bucket_end < uri.length(); ++bucket_end) {
if ('/' == *(uri.ptr() + bucket_end)) {
ObString::obstr_size_t bucket_length = bucket_end - bucket_start;
//must end with '\0'
if (OB_FAIL(databuff_printf(bucket_name_buff, OB_MAX_URI_LENGTH, "%.*s", bucket_length, uri.ptr() + bucket_start))) {
OB_LOG(WARN, "fail to deep copy bucket", K(uri), K(bucket_start), K(bucket_length), K(ret));
} else {
bucket_name_string_.assign_ptr(bucket_name_buff, strlen(bucket_name_buff) + 1);// must include '\0'
bucket_name_ = obstring_to_string_buffer(bucket_name_string_);
}
break;
}
}
if (OB_SUCC(ret) && bucket_end == uri.length()) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "bucket name is invalid", K(uri), K(ret));
}
}
}
// parse the object name
if (OB_SUCC(ret)) {
if (bucket_end + 1 >= uri.length()) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "object name is NULL", K(uri), K(ret));
} else {
object_start = bucket_end + 1;
ObString::obstr_size_t object_length = uri.length() - object_start;
//must end with '\0'
if (OB_FAIL(databuff_printf(object_name_buff, OB_MAX_URI_LENGTH, "%.*s", object_length, uri.ptr() + object_start))) {
OB_LOG(WARN, "fail to deep copy object", K(uri), K(object_start), K(object_length), K(ret));
} else {
object_name_string_.assign_ptr(object_name_buff, strlen(object_name_buff) + 1);//must include '\0'
object_name_ = obstring_to_string_buffer(object_name_string_);
}
}
}
if (OB_SUCC(ret)) {
OB_LOG(DEBUG, "get bucket object name ", K(uri), K(bucket_name_string_), K(object_name_string_));
}
return ret;
}
qcloud_cos::ObCosWrapper::Handle *ObCosObject::create_cos_handle()
{
int ret = OB_SUCCESS;
qcloud_cos::ObCosWrapper::Handle *h = NULL;
qcloud_cos::OB_COS_customMem cos_mem = {ob_cos_malloc, ob_cos_free, &allocator_};
if (OB_ISNULL(cos_base_)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "cos base is null!", K(ret));
} else if (OB_FAIL(qcloud_cos::ObCosWrapper::create_cos_handle(cos_mem, *(cos_base_->account_), &h))) {
OB_LOG(WARN, "failed to create cos handle", K(ret));
} else if (OB_ISNULL(h)) {
ret = OB_COS_ERROR;
OB_LOG(WARN, "create handle succeed, but returned handle is null", K(ret));
}
return h;
}
void ObCosObject::destroy_cos_handle(qcloud_cos::ObCosWrapper::Handle *handle)
{
if (NULL != handle) {
qcloud_cos::ObCosWrapper::destroy_cos_handle(handle);
}
}
/**
* ------------------------------ObCosObject---------------------
*/
int ObCosObject::head_meta(const ObString &uri,
bool &is_exist,
qcloud_cos::CosObjectMeta &meta)
{
int ret = OB_SUCCESS;
if (OB_FAIL(build_bucket_and_object_name(uri))) {
OB_LOG(WARN, "failed to build object", K(ret), K(uri));
} else if (OB_FAIL(head_meta(is_exist, meta))) {
OB_LOG(WARN, "failed to head meta", K(ret), K(uri));
}
return ret;
}
int ObCosObject::del(const ObString &uri, int64_t &deleted_cnt)
{
int ret = OB_SUCCESS;
if (OB_FAIL(build_bucket_and_object_name(uri))) {
OB_LOG(WARN, "failed to build object", K(ret), K(uri));
} else if (OB_FAIL(del(deleted_cnt))) {
OB_LOG(WARN, "failed to del object", K(ret), K(uri), K(deleted_cnt));
}
return ret;
}
ObCosObject::ObCosObject(ObCosBase* cos_base) : cos_base_(cos_base)
{
}
ObCosObject::ObCosObject() : cos_base_(NULL)
{
}
int ObCosObject::head_meta(
bool &is_exist,
qcloud_cos::CosObjectMeta &meta)
{
int ret = OB_SUCCESS;
qcloud_cos::ObCosWrapper::Handle *h = NULL;
if (OB_ISNULL(cos_base_)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "cos base is not init", K(ret));
} else if (OB_ISNULL(h = create_cos_handle())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(WARN, "create cos handle failed", K(ret), K(bucket_name_string_), K(object_name_string_));
} else if (OB_FAIL(qcloud_cos::ObCosWrapper::head_object_meta(h, bucket_name_, object_name_, is_exist, meta))) {
OB_LOG(WARN, "failed to head object meta from cos", K(ret), K(bucket_name_string_), K(object_name_string_));
}
meta.type = ObCosObjectType::COS_OBJECT_NORMAL;
destroy_cos_handle(h);
return ret;
}
int ObCosObject::del(int64_t &deleted_cnt)
{
int ret = OB_SUCCESS;
deleted_cnt = 0;
qcloud_cos::ObCosWrapper::Handle *h = create_cos_handle();
if (OB_ISNULL(cos_base_)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "cos base is not init", K(ret));
}
if (NULL == h) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(WARN, "create cos handle failed", K(ret), K(bucket_name_string_), K(object_name_string_));
} else {
OB_LOG(DEBUG, "start to delete object", K(bucket_name_string_), K(object_name_string_));
if (OB_FAIL(qcloud_cos::ObCosWrapper::del(h, bucket_name_, object_name_))) {
OB_LOG(WARN, "failed to delete object from cos", K(ret), K(bucket_name_string_), K(object_name_string_));
} else {
deleted_cnt = 1;
OB_LOG(DEBUG, "succeed to delete object from cos", K(ret), K(bucket_name_string_), K(object_name_string_));
}
}
destroy_cos_handle(h);
return ret;
}
}
}

115
deps/oblib/src/lib/restore/cos/ob_storage_cos_obj.h vendored
View File

@ -1,115 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_COS_OBJ_H_
#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_COS_OBJ_H_
#include <sys/types.h>
#include <sys/stat.h>
#include "lib/string/ob_string.h"
#include "lib/allocator/page_arena.h"
#include "ob_cos_wrapper.h"
namespace oceanbase
{
namespace common
{
// Allocator for creating cos handle
class ObCosCtxAllocator
{
public:
ObCosCtxAllocator();
virtual ~ObCosCtxAllocator();
void *alloc(size_t size);
void free(void *addr);
void reuse();
private:
ObArenaAllocator allocator_;
};
// Object type, normal object or container.
enum ObCosObjectType
{
COS_OBJECT_NORMAL = 0,
COS_OBJECT_CONTAINER,
COS_OBJECT_TYPE_MAX,
};
// ObCosBase
class ObCosBase
{
public:
ObCosBase() : account_(NULL) {}
virtual ~ObCosBase() {}
int build_account(const ObString &storage_info);
// user account, include id & key, etc.
qcloud_cos::ObCosAccount *account_;
private:
// allocator for creating account
ObCosCtxAllocator cos_ctx_allocator_;
};
//obcosobject is the ctx of single file
class ObCosObject
{
public:
ObCosObject();
ObCosObject(ObCosBase* cos_base);
virtual ~ObCosObject() {};
// Returned the cos handle that user can use to access CosWrapper
// directly.
// Note you need build the account by calling build_account before
// create the handle.
qcloud_cos::ObCosWrapper::Handle *create_cos_handle();
// release cos handle, the created handle cannot use anymore.
void destroy_cos_handle(qcloud_cos::ObCosWrapper::Handle *handle);
// convert an ObString structure to qcloud_cos::CosStringBuffer
static inline qcloud_cos::CosStringBuffer obstring_to_string_buffer(const ObString &bucket)
{
return qcloud_cos::CosStringBuffer(bucket.ptr(), bucket.length());
}
const ObString& bucket_name_string() const { return bucket_name_string_; }
const ObString& object_name_string() const { return object_name_string_; }
const qcloud_cos::CosStringBuffer& bucket_name() const { return bucket_name_; }
const qcloud_cos::CosStringBuffer& object_name() const { return object_name_; }
int build_bucket_and_object_name(const ObString &uri);
int head_meta(const ObString &uri, bool &is_exist, qcloud_cos::CosObjectMeta &meta);
int del(const ObString &uri, int64_t &deleted_cnt);
virtual int head_meta(bool &is_exist, qcloud_cos::CosObjectMeta &meta);
virtual int del(int64_t &deleted_cnt);
ObCosBase* cos_base_;
protected:
// bucket and object name of object
ObString bucket_name_string_;
ObString object_name_string_;
// bucket and object name used for CosWrapper
qcloud_cos::CosStringBuffer bucket_name_;
qcloud_cos::CosStringBuffer object_name_;
private:
// allocator for object/bucket name/handle
ObCosCtxAllocator allocator_;
};
}
}
#endif

7
deps/oblib/src/lib/restore/ob_object_device.cpp vendored
View File

@ -19,7 +19,7 @@ namespace common
const char *OB_STORAGE_ACCESS_TYPES_STR[] = {"reader", "overwriter", "appender", "random_write"};
ObObjectDevice::ObObjectDevice() : oss_account_(), cos_base_(), base_info_(NULL), is_started_(false), lock_()
ObObjectDevice::ObObjectDevice() : oss_account_(), base_info_(NULL), is_started_(false), lock_()
{
}
@ -72,11 +72,6 @@ int ObObjectDevice::start(const ObIODOpts &opts)
OB_LOG(WARN, "fail to init oss base", K(storage_info) ,K(ret));
}
base_info_ = (void*)&oss_account_;
} else if (OB_STORAGE_COS == device_type_) {
if (OB_FAIL(cos_base_.build_account(storage_info))) {
OB_LOG(WARN, "fail to build cos account", K(storage_info), K(ret));
}
base_info_ = (void*)&cos_base_;
} else if (OB_STORAGE_FILE == device_type_) {
//do nothing
base_info_ = NULL;

1
deps/oblib/src/lib/restore/ob_object_device.h vendored
View File

@ -87,7 +87,6 @@ private:
//maybe fd mng can be device level
common::ObFdSimulator fd_mng_;
ObOssAccount oss_account_;
ObCosBase cos_base_; /*todo*/
ObStorageUtil util_;
/*obj ctx pool: use to create fd ctx(reader/writer)*/

35
deps/oblib/src/lib/restore/ob_storage.cpp vendored
View File

@ -56,8 +56,7 @@ int validate_uri_type(const common::ObString &uri)
{
int ret = OB_SUCCESS;
if (!uri.prefix_match(OB_OSS_PREFIX) &&
!uri.prefix_match(OB_FILE_PREFIX) &&
!uri.prefix_match(OB_COS_PREFIX) ) {
!uri.prefix_match(OB_FILE_PREFIX)) {
ret = OB_INVALID_BACKUP_DEST;
STORAGE_LOG(ERROR, "invlaid backup uri", K(ret), K(uri));
}
@ -73,8 +72,6 @@ int get_storage_type_from_path(const common::ObString &uri, ObStorageType &type)
type = OB_STORAGE_OSS;
} else if (uri.prefix_match(OB_FILE_PREFIX)) {
type = OB_STORAGE_FILE;
} else if (uri.prefix_match(OB_COS_PREFIX)) {
type = OB_STORAGE_COS;
} else {
ret = OB_INVALID_BACKUP_DEST;
STORAGE_LOG(ERROR, "invlaid backup uri", K(ret), K(uri));
@ -116,11 +113,17 @@ int get_storage_type_from_name(const char *type_str, ObStorageType &type)
* ------------------------------ObStorageGlobalIns---------------------
*/
ObStorageGlobalIns::ObStorageGlobalIns()
:ObSingleton<ObStorageGlobalIns>(), io_prohibited_(false)
:io_prohibited_(false)
{
}
ObStorageGlobalIns& ObStorageGlobalIns::get_instance()
{
static ObStorageGlobalIns instance;
return instance;
}
int ObStorageGlobalIns::init()
{
int ret = OB_SUCCESS;
@ -199,8 +202,6 @@ int ObStorageUtil::open(void* obj_base, int device_type)
STORAGE_LOG(WARN, "double init the storage util", K(ret), K(device_type));
} else if (OB_STORAGE_OSS == device_type) {
util_ = &oss_util_;
} else if (OB_STORAGE_COS == device_type) {
util_ = &cos_util_;
} else if (OB_STORAGE_FILE == device_type) {
util_ = &file_util_;
} else {
@ -539,7 +540,6 @@ ObStorageReader::ObStorageReader()
reader_(NULL),
file_reader_(),
oss_reader_(),
cos_reader_(),
start_ts_(0)
{
uri_[0] = '\0';
@ -577,8 +577,6 @@ int ObStorageReader::open(const common::ObString &uri, void* obj_base_info)
reader_ = &oss_reader_;
} else if (OB_STORAGE_FILE == type) {
reader_ = &file_reader_;
} else if (OB_STORAGE_COS == type) {
reader_ = &cos_reader_;
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri));
@ -663,7 +661,6 @@ ObStorageWriter::ObStorageWriter()
: writer_(NULL),
file_writer_(),
oss_writer_(),
cos_writer_(),
start_ts_(0)
{
uri_[0] = '\0';
@ -701,8 +698,6 @@ int ObStorageWriter::open(const common::ObString &uri, void* obj_base_info)
writer_ = &oss_writer_;
} else if (OB_STORAGE_FILE == type) {
writer_ = &file_writer_;
} else if (OB_STORAGE_COS == type) {
writer_ = &cos_writer_;
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri));
@ -790,7 +785,6 @@ ObStorageAppender::ObStorageAppender(StorageOpenMode mode)
: appender_(NULL),
file_appender_(mode),
oss_appender_(),
cos_appender_(),
start_ts_(0),
is_opened_(false),
storage_info_(NULL)
@ -839,19 +833,6 @@ int ObStorageAppender::open(
}
} else if (OB_STORAGE_FILE == type) {
appender_ = &file_appender_;
} else if (OB_STORAGE_COS == type) {
if (OB_APPEND_USE_SLICE_PUT == param.strategy_) {
ObCosContainer::Option option;
option.open_version = param.version_param_.open_object_version_;
option.version = param.version_param_.version_;
// get slice size from cluster conf
option.threshold = qcloud_cos::ObCosEnv::get_instance().get_slice_size();
cos_appender_.set_obj_type(ObCosObjectType::COS_OBJECT_CONTAINER);
cos_appender_.set_container_option(option);
} else {
cos_appender_.set_obj_type(ObCosObjectType::COS_OBJECT_NORMAL);
}
appender_ = &cos_appender_;
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "unkown storage type", K(ret), K(uri));

12
deps/oblib/src/lib/restore/ob_storage.h vendored
View File

@ -15,7 +15,6 @@
#include "ob_i_storage.h"
#include "ob_storage_file.h"
#include "ob_storage_oss_base.h"
#include "ob_storage_cos.h"
namespace oceanbase
{
@ -44,7 +43,7 @@ public:
static const int64_t FLYING_IO_WAIT_TIMEOUT = 120000000; // default 120s.
private:
static int64_t flying_io_cnt_; // cos/oss doing io counter.
static int64_t flying_io_cnt_; // oss doing io counter.
};
class ObExternalIOCounterGuard final
@ -59,11 +58,14 @@ private:
};
class ObStorageGlobalIns : public qcloud_cos::ObSingleton<ObStorageGlobalIns>
class ObStorageGlobalIns
{
public:
ObStorageGlobalIns();
// not thread safe
static ObStorageGlobalIns& get_instance();
int init();
void fin();
@ -129,7 +131,6 @@ private:
ObStorageFileUtil file_util_;
ObStorageOssUtil oss_util_;
ObCosUtil cos_util_;
ObIStorageUtil* util_;
void* obj_base_info_;
bool init_state;
@ -150,7 +151,6 @@ private:
ObIStorageReader *reader_;
ObStorageFileReader file_reader_;
ObStorageOssReader oss_reader_;
ObCosRandomAccessReader cos_reader_;
int64_t start_ts_;
char uri_[OB_MAX_URI_LENGTH];
DISALLOW_COPY_AND_ASSIGN(ObStorageReader);
@ -168,7 +168,6 @@ private:
ObIStorageWriter *writer_;
ObStorageFileWriter file_writer_;
ObStorageOssWriter oss_writer_;
ObCosOverWriteObject cos_writer_;
int64_t start_ts_;
char uri_[OB_MAX_URI_LENGTH];
DISALLOW_COPY_AND_ASSIGN(ObStorageWriter);
@ -199,7 +198,6 @@ private:
ObIStorageWriter *appender_;
ObStorageFileAppender file_appender_;
ObStorageOssAppendWriter oss_appender_;
ObCosAppender cos_appender_;
int64_t start_ts_;
bool is_opened_;
char uri_[OB_MAX_URI_LENGTH];

2048
deps/oblib/src/lib/restore/ob_storage_cos.cpp vendored
View File

File diff suppressed because it is too large Load Diff

512
deps/oblib/src/lib/restore/ob_storage_cos.h vendored
View File

@ -1,512 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_COS_H_
#define SRC_LIBRARY_SRC_LIB_RESTORE_OB_STORAGE_COS_H_
#include "lib/allocator/page_arena.h"
#include "lib/container/ob_array.h"
#include "ob_i_storage.h"
#include "cos/ob_storage_cos_obj.h"
namespace oceanbase
{
namespace common
{
// Before using cos, you need to initialize cos enviroment.
// Thread safe guaranteed by user.
int init_cos_env();
// You need to clean cos resource when not use cos any more.
// Thread safe guaranteed by user.
void fin_cos_env();
class ObCosUtil: public ObIStorageUtil
{
public:
ObCosUtil() : cos_base_(NULL) {}
virtual ~ObCosUtil() {}
int open(void* base_info);
void close();
int is_exist(const ObString &uri, bool &exist) override;
int get_file_length(const ObString &uri, int64_t &file_length) override;
int write_single_file(const ObString &uri, const char *buf, const int64_t size) override;
// cos no dir
int mkdir(const ObString &uri) override;
int del_file(const ObString &uri) override;
int list_files(const ObString &dir_path, ObBaseDirEntryOperator &op) override;
int del_dir(const ObString &uri) override;
int check_backup_dest_lifecycle(const common::ObString &dir_path, bool &is_set_lifecycle) override;
int get_file_meta(const ObString &uri, bool &exist, qcloud_cos::CosObjectMeta &meta);
int list_directories(const ObString &dir_path, ObBaseDirEntryOperator &op) override;
int is_tagging(const common::ObString &uri, bool &is_tagging) override;
ObCosBase* cos_base_;
};
// Random accesss object
class ObCosRandomAccessObject: public ObIStorageReader
{
public:
ObCosRandomAccessObject();
virtual ~ObCosRandomAccessObject() {}
int open(const ObString &uri, void *cos_base) override;
int pread(
char *buf,
const int64_t buf_size,
int64_t offset,
int64_t &read_size) override;
int close() override;
int64_t get_length() const override { return meta_.file_length; }
bool is_opened() const override { return is_opened_; }
private:
// basic cos object
ObCosObject obj_;
qcloud_cos::CosObjectMeta meta_;
bool is_opened_;
DISALLOW_COPY_AND_ASSIGN(ObCosRandomAccessObject);
};
// Over write object
class ObCosOverWriteObject : public ObIStorageWriter
{
public:
ObCosOverWriteObject(bool append_flag = false);
virtual ~ObCosOverWriteObject() {}
int open(const ObString &uri, void *cos_base) override;
// append write, make sure no concurrent writer.
int write(const char *buf, const int64_t size) override;
int pwrite(const char *buf, const int64_t size, const int64_t offset) override;
int close() override;
// Return the amount of written data after open.
// The returned value is undefined if have not opened before.
int64_t get_length() const override { return file_length_; }
bool is_opened() const override { return is_opened_; }
// The returned value is undefined if have not opened before.
int64_t get_object_size() const { return object_size_; }
private:
// get the whole object from cos.
int get_ (
char *buf,
const int64_t buf_size);
inline const ObString &bucket_name_string_() const
{
return obj_.bucket_name_string();
}
inline const ObString &object_name_string_() const
{
return obj_.object_name_string();
}
private:
// basic cos object
ObCosObject obj_;
// TODO: to rename to data_written_amount
// The current value represents the amount of data written between open and close.
int64_t file_length_;
// The total object size in cos.
int64_t object_size_;
bool is_opened_;
bool append_flag_;
DISALLOW_COPY_AND_ASSIGN(ObCosOverWriteObject);
};
// ObCosSlice
class ObCosSlice
{
public:
enum Mask
{
OB_COS_SLICE_INVALID_MASK = 0,
OB_COS_SLICE_ID_MASK = 0x01,
OB_COS_MULTI_VERSION_MASK = 0x02, // multi-version slice
};
struct Option
{
int64_t mask;
int64_t sliceid;
int64_t version;
inline bool is_multi_version_slice() const
{
return 0 != (mask & Mask::OB_COS_MULTI_VERSION_MASK);
}
TO_STRING_KV(K(mask), K(sliceid), K(version));
};
// slice id marker in slice name.
static constexpr const char *COS_SLICE_MARKER = "@part@_";
// version marker in slice name.
static constexpr const char *COS_VERSION_MAKER = "@version@_";
// Concatenation to connect the slice id part and version part to a slice name.
static constexpr const char *COS_SLICE_CONCATENATION = "-";
ObCosSlice(const Option& option);
~ObCosSlice() {}
// A valid slice name must include slice id, version is optional.
static bool is_a_valid_slice_name(const ObString& path, bool &only);
// Parse slice id and version from slice name.
static int parse_slice_name(
const ObString &path,
ObCosSlice::Option &option);
static int get_container_name(
const ObString& slice_name,
ObString &container_name,
common::ObIAllocator &allocator);
inline bool is_valid() const
{
return 0 != (option_.mask & Mask::OB_COS_SLICE_ID_MASK) && 0 < option_.sliceid;
}
inline bool is_multi_version_slice() const
{
return option_.is_multi_version_slice();
}
int build_slice_name(char *buffer, int64_t buff_size);
TO_STRING_KV(K_(option));
private:
Option option_;
DISALLOW_COPY_AND_ASSIGN(ObCosSlice);
};
// ObCosContainer
class ObCosContainer : public ObCosObject
{
public:
struct Option
{
int64_t version;
bool open_version = false;
// threshold used to switch to next slice.
int64_t threshold = 0; // default 0
TO_STRING_KV(K(version), K(open_version), K(threshold));
};
//normal scenario, appender/reader call open to init
ObCosContainer() : ObCosObject() {}
//just for object del/head meta scenario
ObCosContainer(ObCosBase* cos_base) :
ObCosObject(cos_base)
{}
~ObCosContainer() {}
static inline int64_t get_start_slice_id()
{
// slice id start from 1.
return 1LL;
}
static inline int64_t generate_next_slice_id(int64_t current_slice_id)
{
return current_slice_id + 1;
}
using ObCosObject::head_meta;
using ObCosObject::del;
// Query container meta, returned the slices and each slice length.
int head_meta(
bool &is_exist,
qcloud_cos::CosObjectMeta &meta,
common::ObIAllocator &allocator,
common::ObIArray <ObString> &slice_names,
common::ObIArray <int64_t> &slice_lengths);
int head_meta(bool &is_exist, qcloud_cos::CosObjectMeta &meta) override;
// delete all slices in container
int del(int64_t &deleted_cnt) override;
// list all slices in container
int list_slices(
bool &is_exist,
common::ObIAllocator &allocator,
common::ObIArray <ObString> &slice_names,
common::ObIArray <int64_t> &slice_lengths);
// return max slice id in container
int find_max_slice_option(
const int64_t version,
bool &is_exist,
int64_t &container_size,
ObCosSlice::Option &max_slice_option,
int64_t &last_slice_size);
// let slice number in container not too much, limit 800
static const int64_t MAX_SLICE_ID = 800;
private:
DISALLOW_COPY_AND_ASSIGN(ObCosContainer);
};
class ObCosWritableContainer: public ObIStorageWriter
{
public:
ObCosWritableContainer();
ObCosWritableContainer(const ObCosContainer::Option &option);
virtual ~ObCosWritableContainer() {}
void set_option(const ObCosContainer::Option &option)
{
option_ = option;
}
int open(const ObString &uri, void *cos_base) override;
// append write
int write(const char *buf, const int64_t size) override;
int pwrite(const char *buf, const int64_t size, const int64_t offset) override;
int close() override;
int64_t get_length() const override { return length_; }
bool is_opened() const override { return is_opened_; }
private:
int64_t get_slice_mask() const
{
int64_t mask = ObCosSlice::Mask::OB_COS_SLICE_INVALID_MASK;
mask |= ObCosSlice::Mask::OB_COS_SLICE_ID_MASK;
if (option_.open_version) {
mask |= ObCosSlice::Mask::OB_COS_MULTI_VERSION_MASK;
}
return mask;
}
int open_current_slice();
int build_current_slice_name(char *slice_name_buff, int32_t buff_size);
int switch_to_next_slice(const int64_t size);
bool need_switch_next_slice(const int64_t size) const;
const ObString& object_name_string() const
{
return container_.object_name_string();
}
const ObString& bucket_name_string() const
{
return container_.bucket_name_string();
}
private:
ObCosContainer::Option option_;
bool is_opened_;
int64_t length_;
int64_t container_size_;
int64_t last_slice_size_;
ObCosContainer container_;
ObCosOverWriteObject overwrite_obj_;
int64_t current_slice_id_;
//uri will be used in construct slice uri
ObString uri_;
ObArenaAllocator allocator_;
DISALLOW_COPY_AND_ASSIGN(ObCosWritableContainer);
};
// ObCosRandomAccessContainer
class ObCosRandomAccessContainer: public ObIStorageReader
{
public:
ObCosRandomAccessContainer();
virtual ~ObCosRandomAccessContainer() {}
int open(const ObString &uri, void *cos_base) override;
int pread(
char *buf,
const int64_t buf_size,
int64_t offset,
int64_t &read_size) override;
int close() override;
int64_t get_length() const override { return meta_.file_length; }
bool is_opened() const override
{
return is_opened_;
}
private:
int build_slice_name(
int64_t slice_idx,
char *slice_name_buff,
int32_t buff_size);
int pread_from_slice(
int64_t slice_idx,
char *buf,
const int64_t buf_size,
int64_t offset,
int64_t &read_size);
private:
ObCosContainer container_;
bool is_opened_;
qcloud_cos::CosObjectMeta meta_;
ObArenaAllocator allocator_;
ObArray<ObString> slice_names_array_;
ObArray<int64_t> slice_lengths_array_;
ObString uri_;
DISALLOW_COPY_AND_ASSIGN(ObCosRandomAccessContainer);
};
// For a uri is either an object or container, however we do not it.
// ObCosRandomAccessReader is actually acted as an adapter. It will
// check which type the uri is, and then decides the corresponding reader.
class ObCosRandomAccessReader: public ObIStorageReader {
public:
ObCosRandomAccessReader();
virtual ~ObCosRandomAccessReader() {}
int open(const ObString &uri, void *cos_base) override;
int pread(
char *buf,
const int64_t buf_size,
int64_t offset,
int64_t &read_size) override;
int close() override;
int64_t get_length() const override
{
int64_t length = -1;
if (NULL != reader_) {
length = reader_->get_length();
}
return length;
}
bool is_opened() const override
{
return NULL != reader_;
}
private:
ObCosRandomAccessContainer random_access_container_;
ObCosRandomAccessObject random_access_object_;
ObIStorageReader *reader_;
};
// Over write object
class ObCosAppender : public ObIStorageWriter
{
public:
ObCosAppender();
virtual ~ObCosAppender() {}
int open(const ObString &uri, void *cos_base) override;
// append write, make sure no concurrent writer.
int write(const char *buf, const int64_t size) override;
int pwrite(const char *buf, const int64_t size, const int64_t offset) override;
int close() override;
// Return the amount of written data after open.
// The returned value is undefined if have not opened before.
int64_t get_length() const override
{
int64_t length = -1;
if (NULL != writer_) {
length = writer_->get_length();
}
return length;
}
bool is_opened() const override
{
return NULL != writer_;
}
void set_obj_type(ObCosObjectType type)
{
obj_type_ = type;
}
void set_container_option(ObCosContainer::Option option)
{
writable_container_.set_option(option);
}
private:
ObCosWritableContainer writable_container_;
ObCosOverWriteObject overwrite_obj_;
ObIStorageWriter *writer_;
ObCosObjectType obj_type_;
DISALLOW_COPY_AND_ASSIGN(ObCosAppender);
};
}
}
#endif

1
deps/oblib/src/lib/timezone/ob_oracle_format_models.h vendored
View File

@ -400,6 +400,7 @@ struct ObDFMElem
ObDFMElem(): elem_flag_(ObDFMFlag::INVALID_FLAG),
offset_(OB_INVALID_INDEX_INT64),
len_(0),
is_single_dot_before_(false),
upper_case_mode_(UpperCaseMode::NON_CHARACTER)
{}

2
deps/oblib/src/rpc/obrpc/ob_net_keepalive.cpp vendored
View File

@ -312,7 +312,7 @@ void ObNetKeepAlive::do_server_loop()
int ret = OB_SUCCESS;
struct epoll_event events[512];
struct epoll_event ev;
ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
ev.events = EPOLLIN | EPOLLOUT;
ev.data.ptr = NULL;
int epfd = -1;
if (pipefd_ < 0) {

1
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -541,6 +541,7 @@ PCODE_DEF(OB_TX_2PC_CLEAR_RESP, 0x658)
PCODE_DEF(OB_TX_ROLLBACK_SAVEPOINT, 0x659)
PCODE_DEF(OB_TX_KEEPALIVE, 0x65A)
PCODE_DEF(OB_REGISTER_TX_DATA, 0x65B)
PCODE_DEF(OB_TX_KEEPALIVE_RESP, 0x65C)
// trans id
PCODE_DEF(OB_GET_GTI_REQUEST, 0x660)

2
deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp vendored
View File

@ -248,7 +248,7 @@ int ObRpcProxy::init_pkt(
pkt->set_src_cluster_id(ObRpcNetHandler::CLUSTER_ID);
pkt->set_unis_version(opts.unis_version_);
pkt->set_group_id((0 != get_group_id()) ? get_group_id() : this_worker().get_group_id());
if (pcode > OB_SQL_PCODE_START && pcode < OB_SQL_PCODE_END) {
if (need_increment_request_level(pcode)) {
if (this_worker().get_worker_level() == INT32_MAX) { // The inner sql request is not sent from the tenant thread, so the worker level is still the initial value, given
// inner sql a special nesting level
pkt->set_request_level(5);

9
deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h vendored
View File

@ -142,6 +142,15 @@ public:
void set_dst_cluster(int64_t dst_cluster_id) { dst_cluster_id_ = dst_cluster_id; }
void set_transport_impl(int transport_impl) { transport_impl_ = transport_impl; }
bool need_increment_request_level(int pcode) const {
return (pcode > OB_SQL_PCODE_START && pcode < OB_SQL_PCODE_END)
|| pcode == OB_OUT_TRANS_LOCK_TABLE || pcode == OB_OUT_TRANS_UNLOCK_TABLE || pcode == OB_TABLE_LOCK_TASK
|| pcode == OB_HIGH_PRIORITY_TABLE_LOCK_TASK || pcode == OB_REGISTER_TX_DATA
|| pcode == OB_REFRESH_SYNC_VALUE || pcode == OB_CLEAR_AUTOINC_CACHE || pcode == OB_CLEAN_SEQUENCE_CACHE || pcode == OB_FETCH_TABLET_AUTOINC_SEQ_CACHE
|| pcode == OB_BATCH_GET_TABLET_AUTOINC_SEQ || pcode == OB_BATCH_SET_TABLET_AUTOINC_SEQ
|| pcode == OB_CALC_COLUMN_CHECKSUM_REQUEST || pcode == OB_REMOTE_WRITE_DDL_REDO_LOG || pcode == OB_REMOTE_WRITE_DDL_COMMIT_LOG;
}
// when active is set as false, all RPC calls will simply return OB_INACTIVE_RPC_PROXY.
void active(const bool active) { active_ = active; }

1
deps/oblib/unittest/lib/CMakeLists.txt vendored
View File

@ -86,7 +86,6 @@ oblib_addtest(regex/test_regex.cpp)
oblib_addtest(resource/test_resource_mgr.cpp)
#oblib_addtest(restore/test_storage_file.cpp)
#oblib_addtest(restore/test_storage_oss.cpp)
#oblib_addtest(restore/test_storage_cos.cpp)
#oblib_addtest(restore/test_storage.cpp)
oblib_addtest(stat/test_di_cache.cpp)
oblib_addtest(stat/test_diagnose_info.cpp)

1017
deps/oblib/unittest/lib/restore/test_storage_cos.cpp vendored
View File

File diff suppressed because it is too large Load Diff

1
rpm/.compile/ld
View File

@ -1 +0,0 @@
../.dep_create/bin/ld.lld

7
rpm/oceanbase-ce-build.sh
View File

@ -12,10 +12,11 @@ echo "[BUILD] args: TOP_DIR=${TOP_DIR} PROJECT_NAME=${PROJECT_NAME} VERSION=${VE
cd ${TOP_DIR}
./build.sh clean
./build.sh \
rpm \
./build.sh \
rpm \
-DOB_RELEASEID=$RELEASE \
--init \
-DBUILD_NUMBER=$RELEASE \
--init \
--make rpm || exit 1
cd ${TOP_DIR}/build_rpm

8
rpm/oceanbase-ce-cdc-build.sh
View File

@ -12,10 +12,12 @@ echo "[BUILD] args: TOP_DIR=${TOP_DIR} PROJECT_NAME=${PROJECT_NAME} VERSION=${VE
cd ${TOP_DIR}
./build.sh clean
./build.sh \
rpm \
./build.sh \
rpm \
-DOB_BUILD_CDC=ON \
-DOB_RELEASEID=$RELEASE \
--init \
-DBUILD_NUMBER=$RELEASE \
--init \
--make rpm || exit 1
cd ${TOP_DIR}/build_rpm

2
src/CMakeLists.txt
View File

@ -10,7 +10,7 @@ target_compile_definitions(ob_base INTERFACE
PACKAGE_VERSION="${PROJECT_VERSION}"
PACKAGE_STRING="${PROJECT_NAME} ${PROJECT_VERSION}"
# change to real release ID.
RELEASEID="${AONE_BUILD_NUMBER}"
RELEASEID="${BUILD_NUMBER}"
)
target_compile_options(ob_base INTERFACE
$<$<COMPILE_LANGUAGE:CXX>:-Wno-invalid-offsetof>)

31
src/logservice/applyservice/ob_log_apply_service.cpp
View File

@ -872,20 +872,23 @@ void ObApplyStatus::statistics_cb_cost_(const LSN &lsn,
const int64_t cb_first_handle_time,
const int64_t cb_start_time)
{
const int64_t cb_finish_time = ObTimeUtility::fast_current_time();
int64_t total_cost_time = cb_finish_time - append_start_time;
int64_t append_cost_time = append_finish_time - append_start_time;
int64_t cb_wait_thread_time = cb_first_handle_time - append_finish_time;
int64_t cb_wait_commit_time = cb_start_time - cb_first_handle_time;
int64_t cb_cost_time = cb_finish_time - cb_start_time;
cb_append_stat_.stat(append_cost_time);
cb_wait_thread_stat_.stat(cb_wait_thread_time);
cb_wait_commit_stat_.stat(cb_wait_commit_time);
cb_execute_stat_.stat(cb_cost_time);
if (total_cost_time > 1000 * 1000) { //1s
CLOG_LOG(WARN, "cb cost too much time", K(lsn), K(scn), K(total_cost_time), K(append_cost_time),
K(cb_wait_thread_time), K(cb_wait_commit_time), K(cb_cost_time), K(append_start_time), K(append_finish_time),
K(cb_first_handle_time), K(cb_first_handle_time), K(cb_finish_time));
// no need to print debug log when config [default value is true] is false;
if (GCONF.enable_record_trace_log) {
const int64_t cb_finish_time = ObTimeUtility::fast_current_time();
int64_t total_cost_time = cb_finish_time - append_start_time;
int64_t append_cost_time = append_finish_time - append_start_time;
int64_t cb_wait_thread_time = cb_first_handle_time - append_finish_time;
int64_t cb_wait_commit_time = cb_start_time - cb_first_handle_time;
int64_t cb_cost_time = cb_finish_time - cb_start_time;
cb_append_stat_.stat(append_cost_time);
cb_wait_thread_stat_.stat(cb_wait_thread_time);
cb_wait_commit_stat_.stat(cb_wait_commit_time);
cb_execute_stat_.stat(cb_cost_time);
if (total_cost_time > 1000 * 1000) { //1s
CLOG_LOG(WARN, "cb cost too much time", K(lsn), K(scn), K(total_cost_time), K(append_cost_time),
K(cb_wait_thread_time), K(cb_wait_commit_time), K(cb_cost_time), K(append_start_time), K(append_finish_time),
K(cb_first_handle_time), K(cb_first_handle_time), K(cb_finish_time));
}
}
}

46
src/logservice/leader_coordinator/election_priority_impl/election_priority_impl.h
View File

@ -92,6 +92,48 @@ OB_SERIALIZE_MEMBER_TEMP(inline, PriorityV0, is_valid_, port_number_);
// 适用于[4_0_0, latest]
struct PriorityV1 : public AbstractPriority
{
/**********************this is for some strange compact reason***********************/
struct SCN_WRAPPER {
SCN_WRAPPER() {value_.set_min();}
SCN_WRAPPER(const palf::SCN &scn) : value_(scn) {}
SCN_WRAPPER &operator=(const palf::SCN &scn) {
value_ = scn;
return *this;
}
int serialize(char *buf, const int64_t buf_len, int64_t &pos) const {
int ret = OB_SUCCESS;
int64_t new_pos = pos;
if (NULL == buf && buf_len <= 0) {
ret = OB_INVALID_ARGUMENT;
} else if (OB_FAIL(value_.fixed_serialize(buf, buf_len, new_pos))) {
ret = OB_BUF_NOT_ENOUGH;
} else {
pos = new_pos;
}
return ret;
}
int deserialize(const char *buf, const int64_t data_len, int64_t &pos)
{
int ret = OB_SUCCESS;
int64_t new_pos = pos;
if (NULL == buf && data_len <= 0) {
ret = OB_INVALID_ARGUMENT;
} else if (OB_FAIL(value_.fixed_deserialize(buf, data_len, new_pos))) {
ret = OB_BUF_NOT_ENOUGH;
} else {
pos = new_pos;
}
return ret;
}
int64_t get_serialize_size() const
{
int64_t size = 0 ;
size += value_.get_fixed_serialize_size();
return size;
}
palf::SCN value_;
};
/*********************************************************************************/
static constexpr int64_t MAX_UNREPLAYED_LOG_TS_DIFF_THRESHOLD_US = 2 * 1000 * 1000L;
friend class unittest::TestElectionPriority;
OB_UNIS_VERSION(1);
@ -106,7 +148,7 @@ public:
// int assign(const PriorityV1 &rhs);
TO_STRING_KV(K_(is_valid), K_(is_observer_stopped), K_(is_server_stopped), K_(is_zone_stopped),
K_(fatal_failures), K_(is_primary_region), K_(serious_failures), K_(is_in_blacklist),
K_(in_blacklist_reason), K_(scn), K_(is_manual_leader), K_(zone_priority));
K_(in_blacklist_reason), K_(scn_.value), K_(is_manual_leader), K_(zone_priority));
protected:
// 刷新优先级的方法
virtual int refresh_(const share::ObLSID &ls_id) override;
@ -129,7 +171,7 @@ private:
common::ObSArray<FailureEvent> fatal_failures_;// negative infos
bool is_primary_region_;
common::ObSArray<FailureEvent> serious_failures_;// negative infos
palf::SCN scn_;
SCN_WRAPPER scn_;
bool is_in_blacklist_;
common::ObStringHolder in_blacklist_reason_;
bool is_manual_leader_;

27
src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp
View File

@ -81,7 +81,7 @@ int PriorityV1::compare(const AbstractPriority &rhs, int &result, ObStringHolder
int PriorityV1::get_scn_(const share::ObLSID &ls_id, palf::SCN &scn)
{
#define PRINT_WRAPPER KR(ret), K(MTL_ID()), K(*this)
#define PRINT_WRAPPER KR(ret), K(MTL_ID()), K(ls_id), K(*this)
int ret = OB_SUCCESS;
palf::PalfHandleGuard palf_handle_guard;
palf::AccessMode access_mode = palf::AccessMode::INVALID_ACCESS_MODE;
@ -99,19 +99,18 @@ int PriorityV1::get_scn_(const share::ObLSID &ls_id, palf::SCN &scn)
palf::SCN min_unreplay_scn;
if (OB_FAIL(palf_handle_guard.get_role(role, unused_pid))) {
COORDINATOR_LOG_(WARN, "get_role failed");
} else if (LEADER == role) {
// return max_ts_ns for leader
if (OB_FAIL(palf_handle_guard.get_max_scn(scn))) {
COORDINATOR_LOG_(WARN, "get_max_ts_ns failed");
}
} else {
// Generate scn from replay_service for follower
} else if (OB_FAIL(palf_handle_guard.get_max_scn(scn))) {
COORDINATOR_LOG_(WARN, "get_max_scn failed");
} else if (FOLLOWER == role) {
if (OB_FAIL(MTL(ObLogService*)->get_log_replay_service()->get_min_unreplayed_log_scn(ls_id, min_unreplay_scn))) {
COORDINATOR_LOG_(WARN, "failed to get_min_unreplayed_scn");
ret = OB_SUCCESS;
}
scn = min_unreplay_scn > SCN::base_scn() ? SCN::scn_dec(min_unreplay_scn) : SCN::min_scn();
}
} else if (SCN::minus(min_unreplay_scn, 1) < scn) {
// For restore case, min_unreplay_scn may be larger than max_ts.
scn = SCN::minus(min_unreplay_scn, 1) ;
} else {}
} else {}
COORDINATOR_LOG_(TRACE, "get_scn_ finished", K(role), K(min_unreplay_scn), K(scn));
}
return ret;
#undef PRINT_WRAPPER
@ -148,7 +147,7 @@ int PriorityV1::refresh_(const share::ObLSID &ls_id)
is_primary_region_ = election_reference_info.element<6>();
is_observer_stopped_ = (observer::ObServer::get_instance().is_stopped()
|| observer::ObServer::get_instance().is_prepare_stopped());
scn_ = scn;
scn_.value_ = scn;
}
return ret;
#undef PRINT_WRAPPERd
@ -312,9 +311,9 @@ int PriorityV1::compare_scn_(int &ret, const PriorityV1&rhs) const
{
int compare_result = 0;
if (OB_SUCC(ret)) {
if (std::max(scn_, rhs.scn_).convert_to_ts() - std::min(scn_, rhs.scn_).convert_to_ts() <= MAX_UNREPLAYED_LOG_TS_DIFF_THRESHOLD_US) {
if (std::max(scn_.value_, rhs.scn_.value_).convert_to_ts() - std::min(scn_.value_, rhs.scn_.value_).convert_to_ts() <= MAX_UNREPLAYED_LOG_TS_DIFF_THRESHOLD_US) {
compare_result = 0;
} else if (std::max(scn_, rhs.scn_) == scn_) {
} else if (std::max(scn_.value_, rhs.scn_.value_) == scn_.value_) {
compare_result = 1;
} else {
compare_result = -1;

51
src/logservice/leader_coordinator/table_accessor.cpp
View File

@ -86,42 +86,6 @@ int LsElectionReferenceInfoRow::end_(const bool true_to_commit)
return trans_.end(true_to_commit);
}
int LsElectionReferenceInfoRow::insert_or_replace_row(const ObArray<ObArray<ObStringHolder>> &zone_list_list,
const common::ObAddr &manual_leader_server,
const ObArray<ObTuple<ObAddr, ObStringHolder>> &blacklist)
{
LC_TIME_GUARD(1_s);
#define PRINT_WRAPPER K(*this), KR(ret), K(zone_list_list), K(manual_leader_server), K(blacklist)
int ret = OB_SUCCESS;
row_for_user_.element<0>() = tenant_id_;
row_for_user_.element<1>() = ls_id_.id();
if (CLICK_FAIL(row_for_user_.element<2>().assign(zone_list_list))) {
COORDINATOR_LOG_(WARN, "copy zone list failed");
} else {
row_for_user_.element<3>() = manual_leader_server;
if (CLICK_FAIL(row_for_user_.element<4>().assign(blacklist))) {
COORDINATOR_LOG_(WARN, "copy remove member info failed");
} else if (CLICK_FAIL(convert_user_info_to_table_info_())) {
COORDINATOR_LOG_(WARN, "convert user info to table info failed");
} else if (CLICK_FAIL(begin_())) {
COORDINATOR_LOG_(WARN, "start trans failed");
} else if (CLICK_FAIL(insert_or_update_row_to_table_())) {
COORDINATOR_LOG_(WARN, "insert or update failed");
} else if (CLICK_FAIL(end_(true))) {
COORDINATOR_LOG_(WARN, "commit failed");
}
}
if (trans_.is_started()) {
COORDINATOR_LOG_(WARN, "transaction execute failed");
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = end_(false))) {
COORDINATOR_LOG_(WARN, "fail to roll back transaction");
}
}
return ret;
#undef PRINT_WRAPPER
}
int LsElectionReferenceInfoRow::change_zone_priority(const ObArray<ObArray<ObStringHolder>> &zone_list_list)
{
LC_TIME_GUARD(1_s);
@ -257,7 +221,7 @@ int LsElectionReferenceInfoRow::start_and_read_()
int ret = OB_SUCCESS;
if (CLICK_FAIL(begin_())) {
COORDINATOR_LOG_(WARN, "start transaction failed");
} else if (CLICK_FAIL(get_row_from_table_()) && ret != OB_ENTRY_NOT_EXIST) {
} else if (CLICK_FAIL(get_row_from_table_())) {
COORDINATOR_LOG_(WARN, "read row from table failed");
} else if (CLICK_FAIL(convert_table_info_to_user_info_())) {
COORDINATOR_LOG_(WARN, "convert table info to user info failed");
@ -273,7 +237,7 @@ int LsElectionReferenceInfoRow::write_and_commit_()
int ret = OB_SUCCESS;
if (CLICK_FAIL(convert_user_info_to_table_info_())) {
COORDINATOR_LOG_(WARN, "convert user info to table info failed");
} else if (CLICK_FAIL(insert_or_update_row_to_table_())) {
} else if (CLICK_FAIL(update_row_to_table_())) {
COORDINATOR_LOG_(WARN, "update column failed");
} else if (CLICK_FAIL(end_(true))) {
COORDINATOR_LOG_(WARN, "commit change failed");
@ -398,7 +362,7 @@ int LsElectionReferenceInfoRow::convert_table_info_to_user_info_()
#undef PRINT_WRAPPER
}
int LsElectionReferenceInfoRow::insert_or_update_row_to_table_()
int LsElectionReferenceInfoRow::update_row_to_table_()
{
LC_TIME_GUARD(1_s);
#define PRINT_WRAPPER KR(ret), K(*this), K(sql), K(affected_rows)
@ -406,11 +370,10 @@ int LsElectionReferenceInfoRow::insert_or_update_row_to_table_()
ObSqlString sql;
int64_t affected_rows = 0;
if (CLICK_FAIL(sql.append_fmt(
"INSERT INTO %s (tenant_id, ls_id, zone_priority, manual_leader_server, blacklist)"
"VALUES (%ld, %ld, '%s', '%s', '%s')"
"ON DUPLICATE KEY UPDATE tenant_id=%ld, ls_id=%ld, zone_priority='%s', manual_leader_server='%s', blacklist='%s'", share::OB_ALL_LS_ELECTION_REFERENCE_INFO_TNAME,
row_for_table_.element<0>(), row_for_table_.element<1>(), to_cstring(row_for_table_.element<2>()), to_cstring(row_for_table_.element<3>()), to_cstring(row_for_table_.element<4>()),
row_for_table_.element<0>(), row_for_table_.element<1>(), to_cstring(row_for_table_.element<2>()), to_cstring(row_for_table_.element<3>()), to_cstring(row_for_table_.element<4>())))) {
"UPDATE %s SET zone_priority='%s', manual_leader_server='%s', blacklist='%s' WHERE tenant_id=%ld and ls_id=%ld",
share::OB_ALL_LS_ELECTION_REFERENCE_INFO_TNAME,
to_cstring(row_for_table_.element<2>()), to_cstring(row_for_table_.element<3>()), to_cstring(row_for_table_.element<4>()),
row_for_table_.element<0>(), row_for_table_.element<1>()))) {
COORDINATOR_LOG_(WARN, "format insert or update sql failed");
} else if (!trans_.is_started()) {
ret = OB_ERR_UNEXPECTED;

13
src/logservice/leader_coordinator/table_accessor.h
View File

@ -76,17 +76,6 @@ public:
*/
LsElectionReferenceInfoRow(const uint64_t tenant_id, const share::ObLSID &ls_id);
~LsElectionReferenceInfoRow();
/**
* @description: 创建日志流的时候同步调用该接口在__all_ls_election_reference_info表中创建出日志流对应的选举参考信息，当对应的行已经存在时，将行中的内容替换为指定的信息
* @param {const ObArray<ObArray<ObStringHolder>> &} zone_list_list 选举参考的zone优先级，表示为二维数组，形如{{z1,z2},{z3},{z4,z5}}：z1和z2具有最高优先级，z3次之，z4和z5最低
* @param {const common::ObAddr &} manual_leader_server 指定某个server成为该日志流的leader
* @param {const ObArray<ObTuple<ObAddr, ObStringHolder>> &} remove_member_info 指定需要删除的副本的列表及其原因，形如{{127.0.0.1:1080, 'migrate'}, {127.0.0.1:1080, 'remove member'}}
* @return {int} 错误码
* @Date: 2022-01-29 16:32:11
*/
int insert_or_replace_row(const ObArray<ObArray<ObStringHolder>> &zone_list_list = ObArray<ObArray<ObStringHolder>>(),
const common::ObAddr &manual_leader_server = ObAddr(ObAddr::VER::IPV4, "0.0.0.0", 0),
const ObArray<ObTuple<ObAddr, ObStringHolder>> &remove_member_info = ObArray<ObTuple<ObAddr, ObStringHolder>>());
/**
* @description: 当且仅当对应的行在__all_ls_election_reference_info表存在时，修改zone_priority列的内容
* @param {ObArray<ObArray<ObStringHolder>>} &zone_list_list 选举参考的zone优先级，表示为二维数组，形如{{z1,z2},{z3},{z4,z5}}：z1和z2具有最高优先级，z3次之，z4和z5最低
@ -122,7 +111,7 @@ private:
int end_(const bool true_to_commit);
int convert_table_info_to_user_info_();
int convert_user_info_to_table_info_();
int insert_or_update_row_to_table_();
int update_row_to_table_();
int get_row_from_table_();
int start_and_read_();
int write_and_commit_();

2
src/logservice/libobcdc/src/libobcdc.cpp
View File

@ -49,7 +49,7 @@ ObCDCFactory::~ObCDCFactory()
curl_global_cleanup();
}
IObCDCInstance *ObCDCFactory::construct_oblog()
IObCDCInstance *ObCDCFactory::construct_obcdc()
{
return ObLogInstance::get_instance();
}

2
src/logservice/libobcdc/src/libobcdc.h
View File

@ -141,7 +141,7 @@ public:
ObCDCFactory();
~ObCDCFactory();
public:
IObCDCInstance *construct_oblog();
IObCDCInstance *construct_obcdc();
void deconstruct(IObCDCInstance *log);
};
}

1
src/logservice/libobcdc/src/ob_log_config.h
View File

@ -208,6 +208,7 @@ public:
DEF_INT(log_clean_cycle_time_in_hours, OB_CLUSTER_PARAMETER, "24", "[0,]",
"clean log cycle time in hours, 0 means not to clean log");
DEF_INT(max_log_file_count, OB_CLUSTER_PARAMETER, "40", "[0,]", "max log file count, 0 means no limit");
T_DEF_BOOL(skip_dirty_data, OB_CLUSTER_PARAMETER, 0, "0:disabled, 1:enabled");

8
src/logservice/libobcdc/src/ob_log_instance.cpp
View File

@ -398,8 +398,10 @@ int ObLogInstance::init_logger_()
if (OB_FAIL(common::FileDirectoryUtils::create_full_path(log_dir))) {
LOG_ERROR("FileDirectoryUtils create_full_path fail", KR(ret), K(log_dir));
} else {
const int64_t max_log_file_count = TCONF.max_log_file_count;
easy_log_level = EASY_LOG_INFO;
OB_LOGGER.set_max_file_size(MAX_LOG_FILE_SIZE);
OB_LOGGER.set_max_file_index(max_log_file_count);
OB_LOGGER.set_file_name(log_file, disable_redirect_log_, false);
OB_LOGGER.set_log_level("INFO");
OB_LOGGER.disable_thread_log_level();
@ -1983,8 +1985,10 @@ void ObLogInstance::reload_config_()
if (OB_FAIL(config.load_from_file(default_config_fpath))) {
LOG_ERROR("load_from_file fail", KR(ret), K(default_config_fpath));
} else {
LOG_INFO("reset log level", "log_level", config.log_level.str());
const int64_t max_log_file_count = config.max_log_file_count;
LOG_INFO("reset log config", "log_level", config.log_level.str(), K(max_log_file_count));
OB_LOGGER.set_mod_log_levels(config.log_level.str());
OB_LOGGER.set_max_file_index(max_log_file_count);
ATOMIC_STORE(&log_clean_cycle_time_us_, config.log_clean_cycle_time_in_hours * _HOUR_);
@ -2411,7 +2415,7 @@ int ObLogInstance::get_task_count_(int64_t &ready_to_seq_task_count,
committer_pending_dml_trans_count,
committer_ddl_part_trans_task_count,
committer_dml_part_trans_task_count);
_LOG_INFO("[TASK_COUNT_STAT] [USER_QUEQUE] [PART_TRANS_TASK=%ld] [DDL_BR=%ld] [DML_BR=%ld]",
_LOG_INFO("[TASK_COUNT_STAT] [USER_QUEUE] [PART_TRANS_TASK=%ld] [DDL_BR=%ld] [DML_BR=%ld]",
br_queue_part_trans_task_count,
ddl_br_count_in_user_queue,
dml_br_count_in_user_queue);

18
src/logservice/libobcdc/src/ob_log_tenant_mgr.cpp
View File

@ -352,6 +352,10 @@ int ObLogTenantMgr::do_add_tenant_(const uint64_t tenant_id,
}
}
// del tenant_start_ddl_info if exists regardless of ret(in case of tenant already dropped etc.)
try_del_tenant_start_ddl_info_(tenant_id);
if (OB_SUCCESS == ret) {
ISTAT("[ADD_TENANT]", K(tenant_id), K(tenant_name), K(is_new_created_tenant), K(is_new_tenant_by_restore),
K(is_tenant_served), K(start_tstamp_ns), K(tenant_start_serve_ts_ns), K(sys_schema_version),
@ -1484,6 +1488,20 @@ int ObLogTenantMgr::get_min_add_tenant_start_ddl_commit_version_(int64_t &commit
return ret;
}
void ObLogTenantMgr::try_del_tenant_start_ddl_info_(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
TenantID tid(tenant_id);
if (OB_FAIL(add_tenant_start_ddl_info_map_.del(tid))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("try_del_tenant_start_ddl_info_ failed, ignore", KR(ret), K(tenant_id));
} else {
LOG_INFO("add_tenant_start_ddl_info is not found, ignore", KR(ret), K(tenant_id));
}
// no need return error code.
}
}
bool ObLogTenantMgr::SetDataStartSchemaVersionFunc::operator()(const TenantID &tid, ObLogTenant *tenant)
{
UNUSED(tid);

1
src/logservice/libobcdc/src/ob_log_tenant_mgr.h
View File

@ -397,6 +397,7 @@ private:
const uint64_t tenant_id,
int64_t &start_commit_version);
int get_min_add_tenant_start_ddl_commit_version_(int64_t &commit_version);
void try_del_tenant_start_ddl_info_(const uint64_t tenant_id);
private:
bool inited_;

8
src/logservice/libobcdc/tests/CMakeLists.txt
View File

@ -15,3 +15,11 @@ add_executable(obcdc_tailf_static EXCLUDE_FROM_ALL)
ob_add_new_object_target(obcdc_tailf_objects_static obcdc_tailf_object_list)
target_link_libraries(obcdc_tailf_objects_static PRIVATE obcdc_static)
target_link_libraries(obcdc_tailf_static PRIVATE obcdc_tailf_objects_static)
if (OB_BUILD_CDC)
# execute build_cdc_demo to check dlopen and compile
add_custom_command(TARGET obcdc_tailf POST_BUILD
COMMAND /usr/bin/sh build_cdc_demo.sh ${DEVTOOLS_DIR}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/demo/
)
endif()

10
src/logservice/libobcdc/tests/demo/build_cdc_demo.sh Normal file
View File

@ -0,0 +1,10 @@
#!/bin/bash -x
CLANG_PATH=$1/bin
mkdir -p build_dir &&
cd build_dir
../../copy_obcdc.sh &&
$CLANG_PATH/clang++ ../obcdc_dlopen.cpp -o cdc_dl -ldl -std=c++11 -fpic &&
./cdc_dl &&
cd -

24
src/logservice/libobcdc/tests/demo/obcdc_dlopen.cpp Normal file
View File

@ -0,0 +1,24 @@
#include <iostream>
#include <dlfcn.h>
using namespace std;
#define LOG(msg) \
do { \
std::cout << msg << std::endl; \
} while (0)
int main(int argc, char **argv)
{
const char* lib_path = "lib/libobcdc.so";
void *libobcdc = dlopen(lib_path, RTLD_LOCAL | RTLD_LAZY);
char *errstr;
errstr = dlerror();
if (errstr != NULL) {
LOG("[ERROR][DL]A dynamic linking error occurred");
LOG(errstr);
return 1;
}
dlclose(libobcdc);
return 0;
}

4
src/logservice/libobcdc/tests/obcdc_main.cpp
View File

@ -314,8 +314,8 @@ int ObLogMain::start()
} else if (stop_flag_) {
stop_flag_ = false;
if (NULL == (obcdc_instance_ = obcdc_factory_.construct_oblog())) {
LOG_ERROR("construct oblog fail");
if (NULL == (obcdc_instance_ = obcdc_factory_.construct_obcdc())) {
LOG_ERROR("construct obcdc fail");
ret = OB_INIT_FAIL;
} else {
ObLogInstance *instance = (ObLogInstance *)obcdc_instance_;

Some files were not shown because too many files have changed in this diff Show More