database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](docker)add be init script option (#16909)

Browse Source
This commit is contained in:
FreeOnePlus
2023-04-18 20:03:18 +08:00
committed by GitHub
parent 380bc16595
commit c323bc44ff
2 changed files with 185 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
docker/runtime/be/resource/entry_point.sh
View File

@ -59,22 +59,111 @@ docker_setup_env() {
fi
}
# Check the variables required for startup
docker_required_variables_env() {
declare -g RUN_TYPE
if [ -n "$BUILD_TYPE" ]; then
RUN_TYPE="K8S"
if [[ $BUILD_TYPE =~ ^([kK]8[sS])$ ]]; then
doris_warn "BUILD_TYPE" $BUILD_TYPE
else
doris_error "BUILD_TYPE rule error！example: [k8s], Default Value: docker"
fi
export RUN_TYPE=${RUN_TYPE}
return
fi
if [[ -n "$FE_SERVERS" && -n "$BE_ADDR" ]]; then
RUN_TYPE="ELECTION"
if [[ $FE_SERVERS =~ ^.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}(,.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4})*$ || $FE_SERVERS =~ ^.+:([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
doris_warn "FE_SERVERS" $FE_SERVERS
else
doris_error "FE_SERVERS rule error！example: \$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT[,\$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT]..."
fi
if [[ $BE_ADDR =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}$ || $BE_ADDR =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:):[1-6]{0,1}[0-9]{1,4}$ ]]; then
doris_warn "BE_ADDR" $BE_ADDR
else
doris_error "BE_ADDR rule error！example: \$BE_IP:\$HEARTBEAT_SERVICE_PORT"
fi
export RUN_TYPE=${RUN_TYPE}
return
fi
if [[ -n "$FE_MASTER_IP" && -n "$BE_IP" && -n "$BE_PORT" ]]; then
RUN_TYPE="ASSIGN"
if [[ $FE_MASTER_IP =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}$ || $FE_MASTER_IP =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
doris_warn "FE_MASTER_IP" $FE_MASTER_IP
else
doris_error "FE_MASTER_IP rule error！example: \$FE_MASTER_IP"
fi
if [[ $BE_IP =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}$ || $BE_IP =~ ^([0-9a-fA-F]{1,4}:){7,7}([0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,6}(:[0-9a-fA-F]{1,4}|:)|([0-9a-fA-F]{1,4}:){1,5}((:[0-9a-fA-F]{1,4}){1,2}|:)|([0-9a-fA-F]{1,4}:){1,4}((:[0-9a-fA-F]{1,4}){1,3}|:)|([0-9a-fA-F]{1,4}:){1,3}((:[0-9a-fA-F]{1,4}){1,4}|:)|([0-9a-fA-F]{1,4}:){1,2}((:[0-9a-fA-F]{1,4}){1,5}|:)|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6}|:)|:((:[0-9a-fA-F]{1,4}){1,7}|:)$ ]]; then
doris_warn "BE_IP" $BE_IP
else
doris_error "BE_IP rule error！example: \$BE_IP"
fi
if [[ $BE_PORT =~ ^[1-6]{0,1}[0-9]{1,4}$ ]]; then
doris_warn "BE_PORT" $BE_PORT
else
doris_error "BE_PORT rule error！example: \$BE_PORT."
fi
export RUN_TYPE=${RUN_TYPE}
return
fi
doris_error EOF "
Note that you did not configure the required parameters!
plan 1:
BUILD_TYPE
plan 2:
FE_SERVERS & BE_ADDR
plan 3:
FE_MASTER_IP & FE_MASTER_PORT & BE_IP & BE_PORT"
EOF
}
get_doris_args() {
local feServerArray=($(echo "${FE_SERVERS}" | awk '{gsub (/,/," "); print $0}'))
for i in "${feServerArray[@]}"; do
declare -g MASTER_FE_IP CURRENT_BE_IP CURRENT_BE_PORT PRIORITY_NETWORKS
if [ $RUN_TYPE == "ELECTION" ]; then
local feServerArray=($(echo "${FE_SERVERS}" | awk '{gsub (/,/," "); print $0}'))
for i in "${feServerArray[@]}"; do
val=${i}
val=${val// /}
tmpFeId=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, ""); print$1}')
tmpFeName=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, ""); print$1}')
tmpFeIp=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$2}')
feIpArray[$tmpFeId]=${tmpFeIp}
done
tmpFeEditLogPort=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$3}')
check_arg "TMP_FE_NAME" $tmpFeName
feIpArray[$tmpFeName]=${tmpFeIp}
done
declare -g MASTER_FE_IP BE_HOST_IP BE_HEARTBEAT_PORT
MASTER_FE_IP=${feIpArray[1]}
doris_note "masterFe = ${MASTER_FE_IP}"
BE_HOST_IP=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}')
BE_HEARTBEAT_PORT=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$2}')
doris_note "be_addr = ${BE_HOST_IP}:${BE_HEARTBEAT_PORT}"
FE_MASTER_IP=${feIpArray[1]}
check_arg "FE_MASTER_IP" $FE_MASTER_IP
BE_IP=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}')
check_arg "BE_IP" $BE_IP
BE_PORT=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$2}')
check_arg "BE_PORT" $BE_PORT
elif [ $RUN_TYPE == "ASSIGN" ]; then
check_arg "FE_MASTER_IP" $FE_MASTER_IP
check_arg "BE_IP" $BE_IP
check_arg "BE_PORT" $BE_PORT
fi
PRIORITY_NETWORKS=$(echo "${BE_IP}" | awk -F '.' '{print$1"."$2"."$3".0/24"}')
check_arg "PRIORITY_NETWORKS" $PRIORITY_NETWORKS
# export be args
export MASTER_FE_IP=${FE_MASTER_IP}
export CURRENT_BE_IP=${BE_IP}
export CURRENT_BE_PORT=${BE_PORT}
export PRIORITY_NETWORKS=${PRIORITY_NETWORKS}
doris_note "MASTER_FE_IP ${MASTER_FE_IP}"
doris_note "CURRENT_BE_IP ${CURRENT_BE_IP}"
doris_note "CURRENT_BE_PORT ${CURRENT_BE_PORT}"
doris_note "PRIORITY_NETWORKS ${PRIORITY_NETWORKS}"
check_be_status true
}
# Execute sql script, passed via stdin
@ -83,32 +172,36 @@ get_doris_args() {
# ie: docker_process_sql --database=mydb <my-file.sql
docker_process_sql() {
set +e
mysql -uroot -P9030 -h${MASTER_FE_IP} --comments "$@" 2>/dev/null
mysql -uroot -P9030 -h${MASTER_FE_IP} --comments "$@" 2>&1
}
check_be_status() {
set +e
local is_fe_start=false
for i in {1..300}; do
if [[ $(($i % 20)) == 1 ]]; then
doris_warn "start check be status~"
fi
docker_process_sql <<<"show backends;" | grep "[[:space:]]${BE_HOST_IP}[[:space:]]" | grep "[[:space:]]${BE_HEARTBEAT_PORT}[[:space:]]" | grep "[[:space:]]true[[:space:]]"
be_join_status=$?
if [[ "${be_join_status}" == 0 ]]; then
doris_note "Verify that BE is registered to FE successfully"
is_fe_start=true
break
else
if [[ $(($i % 20)) == 1 ]]; then
doris_note "register is failed, wait next~"
fi
fi
sleep 1
done
if ! [[ $is_fe_start ]]; then
doris_error "Failed to register BE to FE！Tried 30 times！Maybe FE Start Failed！"
set +e
for i in {1..300}; do
if [[ $1 == true ]]; then
docker_process_sql <<<"show frontends" | grep "[[:space:]]${MASTER_FE_IP}[[:space:]]"
else
docker_process_sql <<<"show backends" | grep "[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep "[[:space:]]${CURRENT_BE_PORT}[[:space:]]" | grep "[[:space:]]true[[:space:]]"
fi
be_join_status=$?
if [[ "${be_join_status}" == 0 ]]; then
if [[ $1 == true ]]; then
doris_note "MASTER FE is started!"
else
doris_note "EntryPoint Check - Verify that BE is registered to FE successfully"
BE_ALREADY_EXISTS=true
fi
break
fi
if [[ $(( $i % 20 )) == 1 ]]; then
if [[ $1 == true ]]; then
doris_note "MASTER FE is not started. retry."
else
doris_note "BE is not register. retry."
fi
fi
sleep 1
done
}
# usage: docker_process_init_files [file [file [...]]]
@ -158,10 +251,19 @@ docker_process_init_files() {
done
}
# Check whether the passed parameters are empty to avoid subsequent task execution failures. At the same time,
# enumeration checks can be added, such as checking whether a certain parameter appears repeatedly, etc.
check_arg() {
if [ -z $2 ]; then
doris_error "$1 is null!"
fi
}
_main() {
docker_setup_env
docker_required_variables_env
# get init args
get_doris_args
docker_setup_env
# Start Doris BE
{
set +e
@ -171,6 +273,7 @@ _main() {
check_be_status
if [ -z ${DATABASE_ALREADY_EXISTS} ]; then
# run script
sleep 15
docker_process_init_files /docker-entrypoint-initdb.d/*
fi

144
docker/runtime/be/resource/init_be.sh
View File

@ -55,65 +55,27 @@ _is_sourced() {
}
docker_setup_env() {
sysctl -w vm.max_map_count=2000000
declare -g DATABASE_ALREADY_EXISTS
if [ -d "${DORIS_HOME}/be/storage/data" ]; then
DATABASE_ALREADY_EXISTS='true'
fi
}
# Check the variables required for startup
docker_required_variables_env() {
if [[ $FE_SERVERS =~ ^.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}(,.+:[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4})*$ ]]; then
doris_warn "FE_SERVERS" $FE_SERVERS
else
doris_error "FE_SERVERS rule error！example: \$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT[,\$FE_NAME:\$FE_HOST_IP:\$FE_EDIT_LOG_PORT]..."
fi
if [[ $BE_ADDR =~ ^[1-2]{0,1}[0-9]{0,1}[0-9]{1}(\.[1-2]{0,1}[0-9]{0,1}[0-9]{1}){3}:[1-6]{0,1}[0-9]{1,4}$ ]]; then
doris_warn "BE_ADDR" $BE_ADDR
else
doris_error "BE_ADDR rule error！example: \$BE_HOST_IP:\$HEARTBEAT_SERVICE_PORT"
fi
}
get_doris_be_args() {
local feServerArray=($(echo "${FE_SERVERS}" | awk '{gsub (/,/," "); print $0}'))
for i in "${feServerArray[@]}"; do
val=${i}
val=${val// /}
tmpFeId=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, ""); print$1}')
tmpFeIp=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$2}')
tmpFeEditLogPort=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$3}')
check_arg "tmpFeIp" $tmpFeIp
feIpArray[$tmpFeId]=${tmpFeIp}
check_arg "tmpFeEditLogPort" $tmpFeEditLogPort
feEditLogPortArray[$tmpFeId]=${tmpFeEditLogPort}
done
declare -g MASTER_FE_IP BE_HOST_IP BE_HEARTBEAT_PORT PRIORITY_NETWORKS
MASTER_FE_IP=${feIpArray[1]}
check_arg "MASTER_FE_IP" $MASTER_FE_IP
BE_HOST_IP=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}')
check_arg "BE_HOST_IP" $BE_HOST_IP
BE_HEARTBEAT_PORT=$(echo "${BE_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$2}')
check_arg "BE_HEARTBEAT_PORT" $BE_HEARTBEAT_PORT
PRIORITY_NETWORKS=$(echo "${BE_HOST_IP}" | awk -F '.' '{print$1"."$2"."$3".0/24"}')
check_arg "priority_networks" $PRIORITY_NETWORKS
doris_note "feIpArray = ${feIpArray[*]}"
doris_note "feEditLogPortArray = ${feEditLogPortArray[*]}"
doris_note "masterFe = ${feIpArray[1]}:${feEditLogPortArray[1]}"
doris_note "be_addr = ${BE_HOST_IP}:${BE_HEARTBEAT_PORT}"
doris_note "priority_networks = ${PRIORITY_NETWORKS}"
# wait fe start
check_be_status true
}
add_priority_networks() {
doris_note "add priority_networks ${1} to ${DORIS_HOME}/be/conf/be.conf"
echo "priority_networks = ${1}" >>${DORIS_HOME}/be/conf/be.conf
}
show_be_args(){
doris_note "============= init args ================"
doris_note "MASTER_FE_IP " ${MASTER_FE_IP}
doris_note "CURRENT_BE_IP " ${CURRENT_BE_IP}
doris_note "CURRENT_BE_PORT " ${CURRENT_BE_PORT}
doris_note "RUN_TYPE " ${RUN_TYPE}
doris_note "PRIORITY_NETWORKS " ${PRIORITY_NETWORKS}
}
# Execute sql script, passed via stdin
# usage: docker_process_sql sql_script
docker_process_sql() {
@ -135,7 +97,7 @@ register_be_to_fe() {
# check fe status
local is_fe_start=false
for i in {1..300}; do
docker_process_sql <<<"alter system add backend '${BE_HOST_IP}:${BE_HEARTBEAT_PORT}'"
docker_process_sql <<<"alter system add backend '${CURRENT_BE_IP}:${CURRENT_BE_PORT}'"
register_be_status=$?
if [[ $register_be_status == 0 ]]; then
doris_note "BE successfully registered to FE！"
@ -162,58 +124,48 @@ register_be_to_fe() {
fi
}
# Check whether the passed parameters are empty to avoid subsequent task execution failures. At the same time,
# enumeration checks can be added, such as checking whether a certain parameter appears repeatedly, etc.
check_arg() {
if [ -z $2 ]; then
doris_error "$1 is null!"
fi
check_be_status() {
set +e
local is_fe_start=false
for i in {1..300}; do
if [[ $(($i % 20)) == 1 ]]; then
doris_warn "start check be register status~"
fi
docker_process_sql <<<"show backends;" | grep "[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep "[[:space:]]${CURRENT_BE_PORT}[[:space:]]"
be_join_status=$?
if [[ "${be_join_status}" == 0 ]]; then
doris_note "Verify that BE is registered to FE successfully"
is_fe_start=true
return
else
if [[ $(($i % 20)) == 1 ]]; then
doris_note "register is failed, wait next~"
fi
fi
sleep 1
done
if [[ ! $is_fe_start ]]; then
doris_error "Failed to register BE to FE！Tried 30 times！Maybe FE Start Failed！"
fi
}
check_be_status() {
set +e
for i in {1..300}; do
if [[ $1 == true ]]; then
docker_process_sql <<<"show frontends" | grep "[[:space:]]${MASTER_FE_IP}[[:space:]]"
else
docker_process_sql <<<"show backends" | grep "[[:space:]]${BE_HOST_IP}[[:space:]]" | grep "[[:space:]]${BE_HEARTBEAT_PORT}[[:space:]]"
fi
be_join_status=$?
if [[ "${be_join_status}" == 0 ]]; then
if [[ $1 == true ]]; then
doris_note "MASTER FE is started!"
else
doris_note "Init Check - Verify that BE is registered to FE successfully"
BE_ALREADY_EXISTS=true
fi
break
fi
if [[ $(( $i % 20 )) == 1 ]]; then
if [[ $1 == true ]]; then
doris_note "MASTER FE is not started. retry."
else
doris_note "BE is not register. retry."
fi
fi
sleep 1
done
}
_main() {
docker_setup_env
docker_required_variables_env
get_doris_be_args
if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
add_priority_networks $PRIORITY_NETWORKS
node_role_conf
if [[ $RUN_TYPE == "K8S" ]]; then
start_be.sh
else
docker_setup_env
if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
add_priority_networks $PRIORITY_NETWORKS
node_role_conf
fi
show_be_args
register_be_to_fe
check_be_status
doris_note "Ready to start BE！"
start_be.sh
exec "$@"
fi
register_be_to_fe
check_be_status
doris_note "Ready to start BE！"
start_be.sh
exec "$@"
}
if ! _is_sourced; then