30df268c1f
We set LIBHDFS3_CONF env in start_be.sh, so libhdfs3 will try to read this hdfs-site.xml, if file does not exist, it will throw error. But Doris does not handle this error, cause BE crash. This CL mainly changes: Modify start_be.sh to only set LIBHDFS3_CONF if hdfs-site.xml exist. Refactor the HDFSCommonBuilder so that it can return error correctly. Add BE IP info in status, so that we can get ip from error msg like: ERROR 1105 (HY000): errCode = 2, detailMessage = [INTERNAL_ERROR]failed to init reader for file 000.snappy.orc, err: [INTERNAL_ERROR][172.21.0.101]failed to init HDFSCommonBuilder, please check check be/conf/hdfs-site.xml The logic of prefer compute node is wrong, which causing the external table query can only assign up to 3 backends. This CL refactor this logic and also change some FE config: prefer_compute_node_for_external_table If set to true, query on external table will prefer to assign to compute node. And the max number of compute node is controlled by min_backend_num_for_external_table. If set to false, query on external table will assign to any node. min_backend_num_for_external_table Only take effect when prefer_compute_node_for_external_table is true. If the compute node number is less than this value, query on external table will try to get some mix node to assign, to let the total number of node reach this value. If the compute node number is larger than this value, query on external table will assign to compute node only.
247 lines
7.0 KiB
Bash
Executable File
247 lines
7.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
set -eo pipefail
|
|
|
|
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
|
|
|
if [[ "$(uname -s)" == 'Darwin' ]] && command -v brew &>/dev/null; then
|
|
PATH="$(brew --prefix)/opt/gnu-getopt/bin:${PATH}"
|
|
export PATH
|
|
fi
|
|
|
|
OPTS="$(getopt \
|
|
-n "$0" \
|
|
-o '' \
|
|
-l 'daemon' \
|
|
-- "$@")"
|
|
|
|
eval set -- "${OPTS}"
|
|
|
|
RUN_DAEMON=0
|
|
RUN_IN_AWS=0
|
|
while true; do
|
|
case "$1" in
|
|
--daemon)
|
|
RUN_DAEMON=1
|
|
shift
|
|
;;
|
|
--aws)
|
|
RUN_IN_AWS=1
|
|
shift
|
|
;;
|
|
--)
|
|
shift
|
|
break
|
|
;;
|
|
*)
|
|
echo "Internal error"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
DORIS_HOME="$(
|
|
cd "${curdir}/.."
|
|
pwd
|
|
)"
|
|
export DORIS_HOME
|
|
|
|
if [[ "$(uname -s)" != 'Darwin' ]]; then
|
|
MAX_MAP_COUNT="$(cat /proc/sys/vm/max_map_count)"
|
|
if [[ "${MAX_MAP_COUNT}" -lt 2000000 ]]; then
|
|
echo "Please set vm.max_map_count to be 2000000 under root using 'sysctl -w vm.max_map_count=2000000'."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# add libs to CLASSPATH
|
|
for f in "${DORIS_HOME}/lib"/*.jar; do
|
|
if [[ -z "${DORIS_JNI_CLASSPATH_PARAMETER}" ]]; then
|
|
export DORIS_JNI_CLASSPATH_PARAMETER="${f}"
|
|
else
|
|
export DORIS_JNI_CLASSPATH_PARAMETER="${f}:${DORIS_JNI_CLASSPATH_PARAMETER}"
|
|
fi
|
|
done
|
|
# DORIS_JNI_CLASSPATH_PARAMETER is used to configure additional jar path to jvm. e.g. -Djava.class.path=$DORIS_HOME/lib/java-udf.jar
|
|
export DORIS_JNI_CLASSPATH_PARAMETER="-Djava.class.path=${DORIS_JNI_CLASSPATH_PARAMETER}"
|
|
|
|
jdk_version() {
|
|
local java_cmd="${1}"
|
|
local result
|
|
local IFS=$'\n'
|
|
|
|
if [[ -z "${java_cmd}" ]]; then
|
|
result=no_java
|
|
return 1
|
|
else
|
|
local version
|
|
# remove \r for Cygwin
|
|
version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')"
|
|
version="${version//\"/}"
|
|
if [[ "${version}" =~ ^1\. ]]; then
|
|
result="$(echo "${version}" | awk -F '.' '{print $2}')"
|
|
else
|
|
result="$(echo "${version}" | awk -F '.' '{print $1}')"
|
|
fi
|
|
fi
|
|
echo "${result}"
|
|
return 0
|
|
}
|
|
|
|
# export env variables from be.conf
|
|
#
|
|
# UDF_RUNTIME_DIR
|
|
# LOG_DIR
|
|
# PID_DIR
|
|
export UDF_RUNTIME_DIR="${DORIS_HOME}/lib/udf-runtime"
|
|
export LOG_DIR="${DORIS_HOME}/log"
|
|
PID_DIR="$(
|
|
cd "${curdir}"
|
|
pwd
|
|
)"
|
|
export PID_DIR
|
|
|
|
# set odbc conf path
|
|
export ODBCSYSINI="${DORIS_HOME}/conf"
|
|
|
|
# support utf8 for oracle database
|
|
export NLS_LANG='AMERICAN_AMERICA.AL32UTF8'
|
|
|
|
#filter known leak for lsan.
|
|
export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf"
|
|
|
|
while read -r line; do
|
|
envline="$(echo "${line}" |
|
|
sed 's/[[:blank:]]*=[[:blank:]]*/=/g' |
|
|
sed 's/^[[:blank:]]*//g' |
|
|
grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" ||
|
|
true)"
|
|
envline="$(eval "echo ${envline}")"
|
|
if [[ "${envline}" == *"="* ]]; then
|
|
eval 'export "${envline}"'
|
|
fi
|
|
done <"${DORIS_HOME}/conf/be.conf"
|
|
|
|
if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then
|
|
# shellcheck disable=1091
|
|
source "${DORIS_HOME}/bin/palo_env.sh"
|
|
fi
|
|
|
|
if [[ -z "${JAVA_HOME}" ]]; then
|
|
echo "The JAVA_HOME environment variable is not defined correctly"
|
|
echo "This environment variable is needed to run this program"
|
|
echo "NB: JAVA_HOME should point to a JDK not a JRE"
|
|
echo "You can set it in be.conf"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -d "${LOG_DIR}" ]]; then
|
|
mkdir -p "${LOG_DIR}"
|
|
fi
|
|
|
|
if [[ ! -d "${UDF_RUNTIME_DIR}" ]]; then
|
|
mkdir -p "${UDF_RUNTIME_DIR}"
|
|
fi
|
|
|
|
rm -f "${UDF_RUNTIME_DIR}"/*
|
|
|
|
pidfile="${PID_DIR}/be.pid"
|
|
|
|
if [[ -f "${pidfile}" ]]; then
|
|
if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then
|
|
echo "Backend running as process $(cat "${pidfile}"). Stop it first."
|
|
exit 1
|
|
else
|
|
rm "${pidfile}"
|
|
fi
|
|
fi
|
|
|
|
chmod 755 "${DORIS_HOME}/lib/doris_be"
|
|
echo "start time: $(date)" >>"${LOG_DIR}/be.out"
|
|
|
|
if [[ ! -f '/bin/limit3' ]]; then
|
|
LIMIT=''
|
|
else
|
|
LIMIT="/bin/limit3 -c 0 -n 65536"
|
|
fi
|
|
|
|
## If you are not running in aws cloud, disable this env since https://github.com/aws/aws-sdk-cpp/issues/1410.
|
|
if [[ "${RUN_IN_AWS}" -eq 0 ]]; then
|
|
export AWS_EC2_METADATA_DISABLED=true
|
|
fi
|
|
|
|
## set asan and ubsan env to generate core file
|
|
export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
|
|
export UBSAN_OPTIONS=print_stacktrace=1
|
|
|
|
## set TCMALLOC_HEAP_LIMIT_MB to limit memory used by tcmalloc
|
|
set_tcmalloc_heap_limit() {
|
|
local total_mem_mb
|
|
local mem_limit_str
|
|
|
|
if [[ "$(uname -s)" != 'Darwin' ]]; then
|
|
total_mem_mb="$(free -m | grep Mem | awk '{print $2}')"
|
|
else
|
|
total_mem_mb="$(($(sysctl -a hw.memsize | awk '{print $NF}') / 1024))"
|
|
fi
|
|
mem_limit_str=$(grep ^mem_limit "${DORIS_HOME}"/conf/be.conf)
|
|
local digits_unit=${mem_limit_str##*=}
|
|
digits_unit="${digits_unit#"${digits_unit%%[![:space:]]*}"}"
|
|
digits_unit="${digits_unit%"${digits_unit##*[![:space:]]}"}"
|
|
local digits=${digits_unit%%[^[:digit:]]*}
|
|
local unit=${digits_unit##*[[:digit:] ]}
|
|
|
|
mem_limit_mb=0
|
|
case ${unit} in
|
|
t | T) mem_limit_mb=$((digits * 1024 * 1024)) ;;
|
|
g | G) mem_limit_mb=$((digits * 1024)) ;;
|
|
m | M) mem_limit_mb=$((digits)) ;;
|
|
k | K) mem_limit_mb=$((digits / 1024)) ;;
|
|
%) mem_limit_mb=$((total_mem_mb * digits / 100)) ;;
|
|
*) mem_limit_mb=$((digits / 1024 / 1024 / 1024)) ;;
|
|
esac
|
|
|
|
if [[ "${mem_limit_mb}" -eq 0 ]]; then
|
|
mem_limit_mb=$((total_mem_mb * 90 / 100))
|
|
fi
|
|
|
|
if [[ "${mem_limit_mb}" -gt "${total_mem_mb}" ]]; then
|
|
echo "mem_limit is larger than whole memory of the server. ${mem_limit_mb} > ${total_mem_mb}."
|
|
return 1
|
|
fi
|
|
export TCMALLOC_HEAP_LIMIT_MB=${mem_limit_mb}
|
|
}
|
|
|
|
# set_tcmalloc_heap_limit || exit 1
|
|
|
|
## set hdfs conf
|
|
if [[ -f "${DORIS_HOME}/conf/hdfs-site.xml" ]]; then
|
|
export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml"
|
|
fi
|
|
|
|
# see https://github.com/jemalloc/jemalloc/issues/2366
|
|
export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:30000,dirty_decay_ms:30000,oversize_threshold:0,lg_tcache_max:16,prof:true,prof_prefix:jeprof.out"
|
|
|
|
if [[ "${RUN_DAEMON}" -eq 1 ]]; then
|
|
nohup ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 </dev/null &
|
|
else
|
|
export DORIS_LOG_TO_STDERR=1
|
|
${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" 2>&1 </dev/null
|
|
fi
|