diff --git a/src/logservice/ob_server_log_block_mgr.cpp b/src/logservice/ob_server_log_block_mgr.cpp index c357e93f9..d8f570a84 100644 --- a/src/logservice/ob_server_log_block_mgr.cpp +++ b/src/logservice/ob_server_log_block_mgr.cpp @@ -592,10 +592,9 @@ int ObServerLogBlockMgr::do_load_(const char *log_disk_path) CLOG_LOG(WARN, "try_continous_do_resize_ failed", K(ret), KPC(this), K(log_disk_path), K(has_allocated_block_cnt)); } else if (FALSE_IT(time_guard.click("try_continous_to_resize_")) - || (false + || false == check_log_pool_whehter_is_integrity_(has_allocated_block_cnt - * BLOCK_SIZE) && - OB_FAIL(recover_(has_allocated_block_cnt * BLOCK_SIZE)))) { + * BLOCK_SIZE)) { ret = OB_ERR_UNEXPECTED; CLOG_LOG(ERROR, "check_log_pool_whehter_is_integrity_ failed, unexpected error", K(ret), KPC(this), K(log_disk_path), K(has_allocated_block_cnt)); @@ -606,32 +605,6 @@ int ObServerLogBlockMgr::do_load_(const char *log_disk_path) return ret; } -int ObServerLogBlockMgr::recover_(const int64_t has_allocated_block_byte_size) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - CLOG_LOG(WARN, "ObServerLogBlockMGR is not inited", K(ret), KPC(this)); - } else { - const int64_t meta_curr_total_size_byte = log_pool_meta_.curr_total_size_; - const int64_t free_size_byte = get_free_size_guarded_by_lock_(); - if (meta_curr_total_size_byte == free_size_byte + has_allocated_block_byte_size) { - // do nothing - } else if (meta_curr_total_size_byte > free_size_byte + has_allocated_block_byte_size) { - LogPoolMeta new_log_pool_meta = log_pool_meta_; - new_log_pool_meta.curr_total_size_ = free_size_byte + has_allocated_block_byte_size; - new_log_pool_meta.status_ = EXPANDING_STATUS; - return do_resize_(log_pool_meta_, - calc_block_cnt_by_size_(meta_curr_total_size_byte) - - calc_block_cnt_by_size_(free_size_byte + has_allocated_block_byte_size), - new_log_pool_meta); - } else { - ret = OB_ERR_UNEXPECTED; - } - } - return ret; -} - int ObServerLogBlockMgr::scan_log_disk_dir_(const char *log_disk_path, int64_t &has_allocated_block_cnt) { diff --git a/src/logservice/ob_server_log_block_mgr.h b/src/logservice/ob_server_log_block_mgr.h index fc9ff638c..9c76f3054 100644 --- a/src/logservice/ob_server_log_block_mgr.h +++ b/src/logservice/ob_server_log_block_mgr.h @@ -224,7 +224,6 @@ private: int prepare_dir_and_create_meta_(const char *log_pool_path, const char *log_pool_tmp_path); int do_load_(const char *log_disk_path); - int recover_(const int64_t has_allocated_block_byte_size); int scan_log_disk_dir_(const char *log_disk_path, int64_t &has_allocated_block_cnt); int scan_log_pool_dir_and_do_trim_(); int trim_log_pool_dir_and_init_block_id_range_(const BlockIdArray &block_id_array, diff --git a/tools/docker/standalone/Dockerfile b/tools/docker/standalone/Dockerfile index 60c6a3e86..f75113482 100644 --- a/tools/docker/standalone/Dockerfile +++ b/tools/docker/standalone/Dockerfile @@ -1,10 +1,7 @@ FROM openanolis/anolisos -# docker build --build-arg --build-arg LOCAL_RPM="oceanbase-ce-4.3.0.0-1.alios7.aarch64.rpm" --build-arg LOCAL_LIB_RPM="oceanbase-ce-libs-4.3.0.0-1.alios7.aarch64.rpm" -t observer . - -ARG VERSION="4.3.0.0" -ARG LOCAL_RPM -ARG LOCAL_LIB_RPM +# docker build --build-arg VERSION={VERSION} . +ARG VERSION RUN yum install -y yum-utils && \ yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo && \ @@ -15,15 +12,10 @@ RUN yum install -y yum-utils && \ RUN mkdir -p /root/pkg && \ cd /root/pkg && \ - if [ "${LOCAL_RPM}" != "" ]; then \ - yum install -y --downloadonly --downloaddir=. obagent; \ - else \ - yum install -y --downloadonly --downloaddir=. oceanbase-ce-${VERSION}.el7 oceanbase-ce-libs-${VERSION}.el7 obagent; \ - fi && \ + yum install -y --downloadonly --downloaddir=. oceanbase-ce-${VERSION}.el7 oceanbase-ce-libs-${VERSION}.el7 obagent && \ + rm -rf /usr/obd/mirror/remote/* && \ yum clean all -COPY ${LOCAL_RPM} /root/pkg -COPY ${LOCAL_LIB_RPM} /root/pkg COPY boot /root/boot/ ENV PATH /root/boot:$PATH ENV LD_LIBRARY_PATH /root/ob/lib:$LD_LIBRARY_PATH diff --git a/tools/docker/standalone/README-CN.md b/tools/docker/standalone/README-CN.md index cf4865962..1183dfc87 100644 --- a/tools/docker/standalone/README-CN.md +++ b/tools/docker/standalone/README-CN.md @@ -117,9 +117,8 @@ docker -v 参数的详细说明可以参考 [docker volumn](https://docs.docker. - 必须:将`MODE`配置项修改为`STANDALONE` - 可选:修改其余配置项 -修改完毕后,执行镜像构建脚本,目前支持使用远程RPM包构建以及本地编译生成的RPM包两种方式: +修改完毕后,执行镜像构建脚本: -- 本地:`./fast_boot_docker_build.sh -L ` -- 远程:`./fast_boot_docker_build.sh -R ` +- `./fast_boot_docker_build.sh ` 例如:`./fast_boot_docker_build.sh 4.2.1.0-100000102023092807` 等待构建完毕后,可使用前述相同的方式启动、测试实例。 \ No newline at end of file diff --git a/tools/docker/standalone/README.md b/tools/docker/standalone/README.md index f19b69c6e..4313f3a31 100644 --- a/tools/docker/standalone/README.md +++ b/tools/docker/standalone/README.md @@ -114,9 +114,8 @@ The `fast_boot_docker_build.sh` script is provided in the `tools/docker/standalo - Required: Modify the `MODE` configuration item to `STANDALONE` - Optional: Modify the remaining configuration items -After the modification is completed, execute the image build script. Currently, two methods are supported: building with remote RPM packages and building with RPM packages generated by local compilation: +After the modification is completed, execute the image build script: -- Local: `./fast_boot_docker_build.sh -L ` -- Remote: `./fast_boot_docker_build.sh -R ` +- `./fast_boot_docker_build.sh `. For example `./fast_boot_docker_build.sh 4.2.1.0-100000102023092807` After waiting for the build to be completed, you can start and test the instance in the same way as mentioned above. \ No newline at end of file diff --git a/tools/docker/standalone/boot/_boot b/tools/docker/standalone/boot/_boot index 2a84c91a9..2228fbc40 100755 --- a/tools/docker/standalone/boot/_boot +++ b/tools/docker/standalone/boot/_boot @@ -10,12 +10,12 @@ PHASE_START_TIME=0 function timediff() { start_time=$1 end_time=$2 - + start_s=${start_time%.*} start_nanos=${start_time#*.} end_s=${end_time%.*} end_nanos=${end_time#*.} - + if [ "$end_nanos" -lt "$start_nanos" ];then end_s=$(( 10#$end_s - 1 )) end_nanos=$(( 10#$end_nanos + 10**9 )) @@ -151,7 +151,7 @@ else # nothing here, bootstrap sed -i "s|@OB_DATAFILE_SIZE@|${OB_DATAFILE_SIZE}|g" $TMPFILE sed -i "s|@OB_LOG_DISK_SIZE@|${OB_LOG_DISK_SIZE}|g" $TMPFILE sed -i "s|@OB_ROOT_PASSWORD@|${OB_ROOT_PASSWORD}|g" $TMPFILE - + [ "${OB_DATA_DIR}" ] && echo " data_dir: ${OB_DATA_DIR}" >> $TMPFILE [ "${OB_REDO_DIR}" ] && echo " redo_dir: ${OB_REDO_DIR}" >> $TMPFILE print_end_phase @@ -171,9 +171,15 @@ else # nothing here, bootstrap deploy_failed fi print_end_phase - + print_start_phase "Ob-deploy restore store dir" rm -rf ${OB_HOME_PATH}/store && tar -Sxzvf /root/boot/store.tar.gz -C ${OB_HOME_PATH} + BLOCK_CNT=`cat /root/boot/block_cnt` + for i in $(seq 0 $[ $BLOCK_CNT - 1 ]) + do + fallocate -o 0 -l 67108864 ${OB_HOME_PATH}/store/clog/log_pool/$i + echo "restore block id $i" + done print_end_phase print_start_phase "Ob-deploy import etc" diff --git a/tools/docker/standalone/fast_boot_docker_build.sh b/tools/docker/standalone/fast_boot_docker_build.sh index 0988fa682..253ec4a25 100644 --- a/tools/docker/standalone/fast_boot_docker_build.sh +++ b/tools/docker/standalone/fast_boot_docker_build.sh @@ -1,10 +1,5 @@ #!/bin/bash -MODE_FLAG=$1 # -L | --local | -R | --remote -REMOTE_VERSION_OR_LOCAL_RPM_PATH=$2 -LOCAL_LIB_RPM_PATH=$3 - -REMOTE_VERSION_OR_LOCAL_RPM_NAME="" -LOCAL_LIB_RPM_NAME="" +RPM_VERSION=$1 TMP_INIT_STORE_PY_SCRIPT="init_store_for_fast_start.tmp.py" ACTUAL_INIT_STORE_PY_SCRIPT="./fast_boot_docker_build_prepare/init_store_for_fast_start.py" @@ -12,26 +7,9 @@ ACTUAL_INIT_STORE_PY_SCRIPT="./fast_boot_docker_build_prepare/init_store_for_fas CWD=$(cd `dirname $0`;pwd) cd "${CWD}" -function local_rpm_build() { - if [ ! -e ${REMOTE_VERSION_OR_LOCAL_RPM_PATH} ]; then - echo "local rpm is not exist" - exit -1 - fi - - if [ ! -e ${LOCAL_LIB_RPM_PATH} ]; then - echo "local lib rpm is not exist" - exit -1 - fi - - cp ${REMOTE_VERSION_OR_LOCAL_RPM_PATH} ./fast_boot_docker_build_prepare - cp ${REMOTE_VERSION_OR_LOCAL_RPM_PATH} . - cp ${LOCAL_LIB_RPM_PATH} ./fast_boot_docker_build_prepare - cp ${LOCAL_LIB_RPM_PATH} . - REMOTE_VERSION_OR_LOCAL_RPM_NAME=$(basename ${REMOTE_VERSION_OR_LOCAL_RPM_PATH}) - LOCAL_LIB_RPM_NAME=$(basename ${LOCAL_LIB_RPM_PATH}) - +function fast_boot_docker_build() { cd fast_boot_docker_build_prepare && \ - docker build --build-arg LOCAL_RPM="${REMOTE_VERSION_OR_LOCAL_RPM_NAME}" --build-arg LOCAL_LIB_RPM="${LOCAL_LIB_RPM_NAME}" -t raw_observer . + docker build --build-arg VERSION="${RPM_VERSION}" -t raw_observer . if [ $? == 0 ]; then echo "================== build prepare docker ok ===============" else @@ -51,38 +29,7 @@ function local_rpm_build() { fi cd "${CWD}" - docker build --build-arg LOCAL_RPM="${REMOTE_VERSION_OR_LOCAL_RPM_NAME}" --build-arg LOCAL_LIB_RPM="${LOCAL_LIB_RPM_NAME}" -t observer . - if [ $? == 0 ]; then - echo "================== fast boot docker build ok ===============" - else - echo "================== fast boot docker build failed ===============" - exit -1 - fi -} - -function remote_rpm_build() { - cd fast_boot_docker_build_prepare && \ - docker build --build-arg VERSION="${REMOTE_VERSION_OR_LOCAL_RPM_NAME}" -t raw_observer . - if [ $? == 0 ]; then - echo "================== build prepare docker ok ===============" - else - echo "================== build prepare docker failed ===============" - exit -1 - fi - - cd "${CWD}" && mkdir -p ${CWD}/boot/etc - docker run -it -v ${CWD}/boot:/root/dest raw_observer - if [ $? == 0 ]; then - echo "================== prepare docker run ok ===============" - else - echo "================== prepare docker run failed ===============" - rm -rf ${CWD}/boot/etc - rm -rf ${CWD}/boot/store.tar.gz - exit -1 - fi - - cd "${CWD}" - docker build --build-arg VERSION="${REMOTE_VERSION_OR_LOCAL_RPM_NAME}" -t observer . + docker build --build-arg VERSION="${RPM_VERSION}" -t oceanbase-ce . if [ $? == 0 ]; then echo "================== fast boot docker build ok ===============" else @@ -108,19 +55,8 @@ else sed -i'' -e "s/@OB_TENANT_NAME@/${OB_TENANT_NAME}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} fi -case $MODE_FLAG in - -L | --local) - local_rpm_build - if [ $? != 0 ]; then - echo "use local rpm build docker failed" - exit -1 - fi - ;; - -R | --remote) - remote_rpm_build - if [ $? != 0 ]; then - echo "use remote rpm build docker failed" - exit -1 - fi - ;; -esac \ No newline at end of file +fast_boot_docker_build +if [ $? != 0 ]; then + echo "use local rpm build docker failed" + exit -1 +fi \ No newline at end of file diff --git a/tools/docker/standalone/fast_boot_docker_build_prepare/Dockerfile b/tools/docker/standalone/fast_boot_docker_build_prepare/Dockerfile index 5a7e4da94..d7c550a6b 100644 --- a/tools/docker/standalone/fast_boot_docker_build_prepare/Dockerfile +++ b/tools/docker/standalone/fast_boot_docker_build_prepare/Dockerfile @@ -1,11 +1,7 @@ FROM openanolis/anolisos -# docker build --build-arg LOCAL_RPM="oceanbase-ce-4.3.0.0-1.alios7.aarch64.rpm" --build-arg LOCAL_LIB_RPM="oceanbase-ce-libs-4.3.0.0-1.alios7.aarch64.rpm" -t raw_observer . - -# Maybe another default version? -ARG VERSION="4.3.0.0" -ARG LOCAL_RPM -ARG LOCAL_LIB_RPM +# docker build --build-arg VERSION={VERSION} . +ARG VERSION RUN yum install -y yum-utils && \ yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo && \ @@ -14,18 +10,15 @@ RUN yum install -y yum-utils && \ yum clean all RUN mkdir -p /root/pkg && mkdir -p /root/store && mkdir -p /root/dest && \ - if [ "${LOCAL_RPM}" == "" ]; then \ - yum install -y --downloadonly --downloaddir=. oceanbase-ce-${VERSION}.el7 oceanbase-ce-libs-${VERSION}.el7; \ - fi && \ + cd /root/pkg && \ + yum install -y --downloadonly --downloaddir=. oceanbase-ce-${VERSION}.el7 oceanbase-ce-libs-${VERSION}.el7 && \ + rm -rf /usr/obd/mirror/remote/* && \ yum clean all -COPY ${LOCAL_RPM} /root/pkg -COPY ${LOCAL_LIB_RPM} /root/pkg COPY boot /root/boot/ +COPY init_store_for_fast_start.py /root/boot/ ENV PATH /root/boot:$PATH ENV LD_LIBRARY_PATH /home/admin/oceanbase/lib:/root/ob/lib:$LD_LIBRARY_PATH -ENV BOOT_LOCAL_RPM ${LOCAL_RPM} -ENV BOOT_LOCAL_LIB_RPM ${LOCAL_LIB_RPM} WORKDIR /root -CMD ["sh", "-c", "_boot ${BOOT_LOCAL_RPM} ${BOOT_LOCAL_LIB_RPM}"] \ No newline at end of file +CMD _boot \ No newline at end of file diff --git a/tools/docker/standalone/fast_boot_docker_build_prepare/boot/_boot b/tools/docker/standalone/fast_boot_docker_build_prepare/boot/_boot index 62b21a7bf..f018d1925 100644 --- a/tools/docker/standalone/fast_boot_docker_build_prepare/boot/_boot +++ b/tools/docker/standalone/fast_boot_docker_build_prepare/boot/_boot @@ -1,13 +1,9 @@ #!/bin/bash -LOCAL_RPM=$1 -LOCAL_LIB_RPM=$2 - CWD=$(cd `dirname $0`;pwd) cd "${CWD}" echo "================== prepare python env ... ==================" -yum install -y wget python3 && wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ -python3 ./get-pip.py && pip install pymysql +yum install -y wget python3 && pip3 install pymysql if [ $? == 0 ]; then echo "================== prepare python env ok ==================" else @@ -16,7 +12,7 @@ else fi echo "================== install oceanbase rpm ... ==================" -cd /root/pkg && yum install -y ${LOCAL_RPM} && yum install -y ${LOCAL_LIB_RPM} +cd /root/pkg && yum install -y *.rpm if [ $? == 0 ]; then echo "================== install oceanbase rpm ok ==================" else diff --git a/tools/docker/standalone/init_store_for_fast_start.tmp.py b/tools/docker/standalone/init_store_for_fast_start.tmp.py index 08fd4f554..c6b7d103f 100644 --- a/tools/docker/standalone/init_store_for_fast_start.tmp.py +++ b/tools/docker/standalone/init_store_for_fast_start.tmp.py @@ -58,7 +58,7 @@ if __name__ == "__main__": parser.add_argument("--memory_size", type=int, default=3221225472) parser.add_argument("--log_disk_size", type=int, default=3221225472) args = parser.parse_args() - + bin_abs_path = os.path.abspath(args.observer_bin_path) home_abs_path = os.path.abspath(args.observer_home_path) data_abs_path = os.path.abspath(args.data_path) @@ -129,10 +129,18 @@ if __name__ == "__main__": logging.warn("deploy observer failed") kill_server() exit(-1) - + # stop observer kill_server() + # record block cnt + record_block_cnt_cmd = "cd %s/clog/log_pool && echo $[ `ls | wc -l` - 1 ] > %s/block_cnt" % (data_abs_path, store_tar_file_path) + record_res = subprocess.call(record_block_cnt_cmd, shell=True) + if record_res != 0: + logging.warn("record block cnt failed") + exit(-1) + logging.info("record block cnt ok") + # build store tar file build_store_tar_cmd = "cd %s/clog/log_pool && ls | grep '[0-9]' | xargs rm && cd %s/.. && \ tar -Sczvf %s/store.tar.gz ./store" % (data_abs_path, data_abs_path, store_tar_file_path)