01d012bab7
Remove page cache regular clear Now the page cache is turned off by default. If the user manually opens the page cache, it can be considered that the user can accept the memory usage of the page cache, and then can consider adding a manual clear command to the cache. fix memory gc cancel top memory query jemalloc prof is not enabled by default
247 lines
7.1 KiB
Bash
Executable File
247 lines
7.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
set -eo pipefail
|
|
|
|
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
|
|
|
if [[ "$(uname -s)" == 'Darwin' ]] && command -v brew &>/dev/null; then
|
|
PATH="$(brew --prefix)/opt/gnu-getopt/bin:${PATH}"
|
|
export PATH
|
|
fi
|
|
|
|
OPTS="$(getopt \
|
|
-n "$0" \
|
|
-o '' \
|
|
-l 'daemon' \
|
|
-- "$@")"
|
|
|
|
eval set -- "${OPTS}"
|
|
|
|
RUN_DAEMON=0
|
|
RUN_IN_AWS=0
|
|
while true; do
|
|
case "$1" in
|
|
--daemon)
|
|
RUN_DAEMON=1
|
|
shift
|
|
;;
|
|
--aws)
|
|
RUN_IN_AWS=1
|
|
shift
|
|
;;
|
|
--)
|
|
shift
|
|
break
|
|
;;
|
|
*)
|
|
echo "Internal error"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
DORIS_HOME="$(
|
|
cd "${curdir}/.."
|
|
pwd
|
|
)"
|
|
export DORIS_HOME
|
|
|
|
if [[ "$(uname -s)" != 'Darwin' ]]; then
|
|
MAX_MAP_COUNT="$(cat /proc/sys/vm/max_map_count)"
|
|
if [[ "${MAX_MAP_COUNT}" -lt 2000000 ]]; then
|
|
echo "Please set vm.max_map_count to be 2000000 under root using 'sysctl -w vm.max_map_count=2000000'."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# add libs to CLASSPATH
|
|
for f in "${DORIS_HOME}/lib"/*.jar; do
|
|
if [[ -z "${DORIS_JNI_CLASSPATH_PARAMETER}" ]]; then
|
|
export DORIS_JNI_CLASSPATH_PARAMETER="${f}"
|
|
else
|
|
export DORIS_JNI_CLASSPATH_PARAMETER="${f}:${DORIS_JNI_CLASSPATH_PARAMETER}"
|
|
fi
|
|
done
|
|
# DORIS_JNI_CLASSPATH_PARAMETER is used to configure additional jar path to jvm. e.g. -Djava.class.path=$DORIS_HOME/lib/java-udf.jar
|
|
export DORIS_JNI_CLASSPATH_PARAMETER="-Djava.class.path=${DORIS_JNI_CLASSPATH_PARAMETER}"
|
|
|
|
jdk_version() {
|
|
local java_cmd="${1}"
|
|
local result
|
|
local IFS=$'\n'
|
|
|
|
if [[ -z "${java_cmd}" ]]; then
|
|
result=no_java
|
|
return 1
|
|
else
|
|
local version
|
|
# remove \r for Cygwin
|
|
version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')"
|
|
version="${version//\"/}"
|
|
if [[ "${version}" =~ ^1\. ]]; then
|
|
result="$(echo "${version}" | awk -F '.' '{print $2}')"
|
|
else
|
|
result="$(echo "${version}" | awk -F '.' '{print $1}')"
|
|
fi
|
|
fi
|
|
echo "${result}"
|
|
return 0
|
|
}
|
|
|
|
# export env variables from be.conf
|
|
#
|
|
# UDF_RUNTIME_DIR
|
|
# LOG_DIR
|
|
# PID_DIR
|
|
export UDF_RUNTIME_DIR="${DORIS_HOME}/lib/udf-runtime"
|
|
export LOG_DIR="${DORIS_HOME}/log"
|
|
PID_DIR="$(
|
|
cd "${curdir}"
|
|
pwd
|
|
)"
|
|
export PID_DIR
|
|
|
|
# set odbc conf path
|
|
export ODBCSYSINI="${DORIS_HOME}/conf"
|
|
|
|
# support utf8 for oracle database
|
|
export NLS_LANG='AMERICAN_AMERICA.AL32UTF8'
|
|
|
|
#filter known leak for lsan.
|
|
export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf"
|
|
|
|
while read -r line; do
|
|
envline="$(echo "${line}" |
|
|
sed 's/[[:blank:]]*=[[:blank:]]*/=/g' |
|
|
sed 's/^[[:blank:]]*//g' |
|
|
grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" ||
|
|
true)"
|
|
envline="$(eval "echo ${envline}")"
|
|
if [[ "${envline}" == *"="* ]]; then
|
|
eval 'export "${envline}"'
|
|
fi
|
|
done <"${DORIS_HOME}/conf/be.conf"
|
|
|
|
if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then
|
|
# shellcheck disable=1091
|
|
source "${DORIS_HOME}/bin/palo_env.sh"
|
|
fi
|
|
|
|
if [[ -z "${JAVA_HOME}" ]]; then
|
|
echo "The JAVA_HOME environment variable is not defined correctly"
|
|
echo "This environment variable is needed to run this program"
|
|
echo "NB: JAVA_HOME should point to a JDK not a JRE"
|
|
echo "You can set it in be.conf"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -d "${LOG_DIR}" ]]; then
|
|
mkdir -p "${LOG_DIR}"
|
|
fi
|
|
|
|
if [[ ! -d "${UDF_RUNTIME_DIR}" ]]; then
|
|
mkdir -p "${UDF_RUNTIME_DIR}"
|
|
fi
|
|
|
|
rm -f "${UDF_RUNTIME_DIR}"/*
|
|
|
|
pidfile="${PID_DIR}/be.pid"
|
|
|
|
if [[ -f "${pidfile}" ]]; then
|
|
if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then
|
|
echo "Backend running as process $(cat "${pidfile}"). Stop it first."
|
|
exit 1
|
|
else
|
|
rm "${pidfile}"
|
|
fi
|
|
fi
|
|
|
|
chmod 755 "${DORIS_HOME}/lib/doris_be"
|
|
echo "start time: $(date)" >>"${LOG_DIR}/be.out"
|
|
|
|
if [[ ! -f '/bin/limit3' ]]; then
|
|
LIMIT=''
|
|
else
|
|
LIMIT="/bin/limit3 -c 0 -n 65536"
|
|
fi
|
|
|
|
## If you are not running in aws cloud, disable this env since https://github.com/aws/aws-sdk-cpp/issues/1410.
|
|
if [[ "${RUN_IN_AWS}" -eq 0 ]]; then
|
|
export AWS_EC2_METADATA_DISABLED=true
|
|
fi
|
|
|
|
## set asan and ubsan env to generate core file
|
|
export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=0
|
|
export UBSAN_OPTIONS=print_stacktrace=1
|
|
|
|
## set TCMALLOC_HEAP_LIMIT_MB to limit memory used by tcmalloc
|
|
set_tcmalloc_heap_limit() {
|
|
local total_mem_mb
|
|
local mem_limit_str
|
|
|
|
if [[ "$(uname -s)" != 'Darwin' ]]; then
|
|
total_mem_mb="$(free -m | grep Mem | awk '{print $2}')"
|
|
else
|
|
total_mem_mb="$(($(sysctl -a hw.memsize | awk '{print $NF}') / 1024))"
|
|
fi
|
|
mem_limit_str=$(grep ^mem_limit "${DORIS_HOME}"/conf/be.conf)
|
|
local digits_unit=${mem_limit_str##*=}
|
|
digits_unit="${digits_unit#"${digits_unit%%[![:space:]]*}"}"
|
|
digits_unit="${digits_unit%"${digits_unit##*[![:space:]]}"}"
|
|
local digits=${digits_unit%%[^[:digit:]]*}
|
|
local unit=${digits_unit##*[[:digit:] ]}
|
|
|
|
mem_limit_mb=0
|
|
case ${unit} in
|
|
t | T) mem_limit_mb=$((digits * 1024 * 1024)) ;;
|
|
g | G) mem_limit_mb=$((digits * 1024)) ;;
|
|
m | M) mem_limit_mb=$((digits)) ;;
|
|
k | K) mem_limit_mb=$((digits / 1024)) ;;
|
|
%) mem_limit_mb=$((total_mem_mb * digits / 100)) ;;
|
|
*) mem_limit_mb=$((digits / 1024 / 1024 / 1024)) ;;
|
|
esac
|
|
|
|
if [[ "${mem_limit_mb}" -eq 0 ]]; then
|
|
mem_limit_mb=$((total_mem_mb * 90 / 100))
|
|
fi
|
|
|
|
if [[ "${mem_limit_mb}" -gt "${total_mem_mb}" ]]; then
|
|
echo "mem_limit is larger than whole memory of the server. ${mem_limit_mb} > ${total_mem_mb}."
|
|
return 1
|
|
fi
|
|
export TCMALLOC_HEAP_LIMIT_MB=${mem_limit_mb}
|
|
}
|
|
|
|
# set_tcmalloc_heap_limit || exit 1
|
|
|
|
## set hdfs conf
|
|
if [[ -f "${DORIS_HOME}/conf/hdfs-site.xml" ]]; then
|
|
export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml"
|
|
fi
|
|
|
|
# see https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
|
|
export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:30000,dirty_decay_ms:30000,oversize_threshold:0,lg_tcache_max:16,prof_prefix:jeprof.out"
|
|
|
|
if [[ "${RUN_DAEMON}" -eq 1 ]]; then
|
|
nohup ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 </dev/null &
|
|
else
|
|
export DORIS_LOG_TO_STDERR=1
|
|
${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" 2>&1 </dev/null
|
|
fi
|