From bc4ceeca4428f0d726aefaf4f973afaed4fcd807 Mon Sep 17 00:00:00 2001 From: Zhengguo Yang Date: Thu, 30 Dec 2021 10:16:56 +0800 Subject: [PATCH] [improvement] optimize java cmd find (#7428) * optimize java cmd find, if java_home not set use java in PATH --- bin/start_be.sh | 50 +++++--- bin/start_fe.sh | 120 ++++++++++-------- env.sh | 35 +++-- .../apache_hdfs_broker/bin/start_broker.sh | 71 +++++++---- gensrc/script/gen_build_version.sh | 57 +++++---- 5 files changed, 202 insertions(+), 131 deletions(-) diff --git a/bin/start_be.sh b/bin/start_be.sh index 0d9287d4de..c29da577ba 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -16,27 +16,42 @@ # specific language governing permissions and limitations # under the License. -curdir=`dirname "$0"` -curdir=`cd "$curdir"; pwd` +curdir=$(dirname "$0") +curdir=$( + cd "$curdir" + pwd +) OPTS=$(getopt \ - -n $0 \ - -o '' \ - -l 'daemon' \ - -- "$@") + -n $0 \ + -o '' \ + -l 'daemon' \ + -- "$@") eval set -- "$OPTS" RUN_DAEMON=0 while true; do case "$1" in - --daemon) RUN_DAEMON=1 ; shift ;; - --) shift ; break ;; - *) echo "Internal error" ; exit 1 ;; + --daemon) + RUN_DAEMON=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; esac done -export DORIS_HOME=`cd "$curdir/.."; pwd` +export DORIS_HOME=$( + cd "$curdir/.." + pwd +) # export env variables from be.conf # @@ -45,7 +60,10 @@ export DORIS_HOME=`cd "$curdir/.."; pwd` # PID_DIR export UDF_RUNTIME_DIR=${DORIS_HOME}/lib/udf-runtime export LOG_DIR=${DORIS_HOME}/log -export PID_DIR=`cd "$curdir"; pwd` +export PID_DIR=$( + cd "$curdir" + pwd +) # set odbc conf path export ODBCSYSINI=$DORIS_HOME/conf @@ -54,8 +72,8 @@ export ODBCSYSINI=$DORIS_HOME/conf export NLS_LANG=AMERICAN_AMERICA.AL32UTF8 while read line; do - envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="` - envline=`eval "echo $envline"` + envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") + envline=$(eval "echo $envline") if [[ $envline == *"="* ]]; then eval 'export "$envline"' fi @@ -79,7 +97,7 @@ pidfile=$PID_DIR/be.pid if [ -f $pidfile ]; then if kill -0 $(cat $pidfile) > /dev/null 2>&1; then - echo "Backend running as process `cat $pidfile`. Stop it first." + echo "Backend running as process $(cat $pidfile). Stop it first." exit 1 else rm $pidfile @@ -96,7 +114,7 @@ else fi if [ ${RUN_DAEMON} -eq 1 ]; then - nohup $LIMIT ${DORIS_HOME}/lib/palo_be "$@" >> $LOG_DIR/be.out 2>&1 > $LOG_DIR/be.out 2>&1 < /dev/null & else - $LIMIT ${DORIS_HOME}/lib/palo_be "$@" >> $LOG_DIR/be.out 2>&1 > $LOG_DIR/be.out 2>&1 < /dev/null fi diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 200d9bda6a..dd39d25ff3 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -16,15 +16,18 @@ # specific language governing permissions and limitations # under the License. -curdir=`dirname "$0"` -curdir=`cd "$curdir"; pwd` +curdir=$(dirname "$0") +curdir=$( + cd "$curdir" + pwd +) OPTS=$(getopt \ - -n $0 \ - -o '' \ - -l 'daemon' \ - -l 'helper:' \ - -- "$@") + -n $0 \ + -o '' \ + -l 'daemon' \ + -l 'helper:' \ + -- "$@") eval set -- "$OPTS" @@ -32,14 +35,29 @@ RUN_DAEMON=0 HELPER= while true; do case "$1" in - --daemon) RUN_DAEMON=1 ; shift ;; - --helper) HELPER=$2 ; shift 2 ;; - --) shift ; break ;; - *) ehco "Internal error" ; exit 1 ;; + --daemon) + RUN_DAEMON=1 + shift + ;; + --helper) + HELPER=$2 + shift 2 + ;; + --) + shift + break + ;; + *) + ehco "Internal error" + exit 1 + ;; esac done -export DORIS_HOME=`cd "$curdir/.."; pwd` +export DORIS_HOME=$( + cd "$curdir/.." + pwd +) # export env variables from fe.conf # @@ -48,11 +66,14 @@ export DORIS_HOME=`cd "$curdir/.."; pwd` # PID_DIR export JAVA_OPTS="-Xmx1024m" export LOG_DIR="$DORIS_HOME/log" -export PID_DIR=`cd "$curdir"; pwd` +export PID_DIR=$( + cd "$curdir" + pwd +) while read line; do - envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="` - envline=`eval "echo $envline"` + envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") + envline=$(eval "echo $envline") if [[ $envline == *"="* ]]; then eval 'export "$envline"' fi @@ -62,39 +83,37 @@ if [ -e $DORIS_HOME/bin/palo_env.sh ]; then source $DORIS_HOME/bin/palo_env.sh fi -# java -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 +if [ -z "$JAVA_HOME" ]; then + JAVA=$(which java) +else + JAVA="$JAVA_HOME/bin/java" +fi + +if [ ! -x "$JAVA" ]; then + echo "The JAVA_HOME environment variable is not defined correctly" + echo "This environment variable is needed to run this program" + echo "NB: JAVA_HOME should point to a JDK not a JRE" + exit 1 fi -JAVA=$JAVA_HOME/bin/java # get jdk version, return version as an Integer. # 1.8 => 8, 13.0 => 13 jdk_version() { local result - local java_cmd=$JAVA_HOME/bin/java local IFS=$'\n' # remove \r for Cygwin - local lines=$("$java_cmd" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n') - if [[ -z $java_cmd ]] - then - result=no_java - else - for line in $lines; do - if [[ (-z $result) && ($line = *"version \""*) ]] - then - local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q') - # on macOS, sed doesn't support '?' - if [[ $ver = "1."* ]] - then - result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q') - else - result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q') - fi + local lines=$("$JAVA" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n') + for line in $lines; do + if [[ (-z $result) && ($line = *"version \""*) ]]; then + local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q') + # on macOS, sed doesn't support '?' + if [[ $ver = "1."* ]]; then + result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q') + else + result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q') fi - done - fi + fi + done echo "$result" } @@ -102,15 +121,15 @@ jdk_version() { if [ ! -d $LOG_DIR ]; then mkdir -p $LOG_DIR fi - + # check java version and choose correct JAVA_OPTS java_version=$(jdk_version) final_java_opt=$JAVA_OPTS if [ $java_version -gt 8 ]; then if [ -z "$JAVA_OPTS_FOR_JDK_9" ]; then echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >> $LOG_DIR/fe.out - exit -1 - fi + exit 1 + fi final_java_opt=$JAVA_OPTS_FOR_JDK_9 fi echo "using java version $java_version" >> $LOG_DIR/fe.out @@ -118,18 +137,17 @@ echo $final_java_opt >> $LOG_DIR/fe.out # add libs to CLASSPATH for f in $DORIS_HOME/lib/*.jar; do - CLASSPATH=$f:${CLASSPATH}; + CLASSPATH=$f:${CLASSPATH} done export CLASSPATH=${CLASSPATH}:${DORIS_HOME}/lib - pidfile=$PID_DIR/fe.pid if [ -f $pidfile ]; then - if kill -0 `cat $pidfile` > /dev/null 2>&1; then - echo Frontend running as process `cat $pidfile`. Stop it first. - exit 1 - fi + if kill -0 $(cat $pidfile) > /dev/null 2>&1; then + echo Frontend running as process $(cat $pidfile). Stop it first. + exit 1 + fi fi if [ ! -f /bin/limit ]; then @@ -138,7 +156,7 @@ else LIMIT=/bin/limit fi -echo `date` >> $LOG_DIR/fe.out +echo $(date) >> $LOG_DIR/fe.out if [ x"$HELPER" != x"" ]; then # change it to '-helper' to be compatible with code in Frontend @@ -146,9 +164,9 @@ if [ x"$HELPER" != x"" ]; then fi if [ ${RUN_DAEMON} -eq 1 ]; then - nohup $LIMIT $JAVA $final_java_opt org.apache.doris.PaloFe ${HELPER} "$@" >> $LOG_DIR/fe.out 2>&1 > $LOG_DIR/fe.out 2>&1 < /dev/null & else - $LIMIT $JAVA $final_java_opt org.apache.doris.PaloFe ${HELPER} "$@" >> $LOG_DIR/fe.out 2>&1 > $LOG_DIR/fe.out 2>&1 < /dev/null fi echo $! > $pidfile diff --git a/env.sh b/env.sh index efc4b105bd..98a96f4a5b 100755 --- a/env.sh +++ b/env.sh @@ -53,12 +53,12 @@ fi # set GCC HOME if [[ -z ${DORIS_GCC_HOME} ]]; then - export DORIS_GCC_HOME=$(dirname `which gcc`)/.. + export DORIS_GCC_HOME=$(dirname $(which gcc))/.. fi -gcc_ver=`${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion` +gcc_ver=$(${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion) required_ver="7.3.0" -if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then +if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then echo "Error: GCC version (${gcc_ver}) must be greater than or equal to ${required_ver}" exit 1 fi @@ -66,33 +66,40 @@ fi # register keyword is forbidden to use in C++17 # the C++ code generated by flex that remove register keyword after version 2.6.0 # so we need check flex version here to avoid compilation failed -flex_ver=`flex --version | awk '{print $2}'` +flex_ver=$(flex --version | awk '{print $2}') required_ver="2.6.0" -if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then +if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then echo "Error: flex version (${flex_ver}) must be greater than or equal to ${required_ver}" exit 1 fi # find binutils -if test -x ${DORIS_GCC_HOME}/bin/ld ;then +if test -x ${DORIS_GCC_HOME}/bin/ld; then export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/ else export DORIS_BIN_UTILS=/usr/bin/ fi # export CLANG COMPATIBLE FLAGS -export CLANG_COMPATIBLE_FLAGS=`echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 \ - | grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' '` +export CLANG_COMPATIBLE_FLAGS=$(echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 | + grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ') # check java home -if [[ -z ${JAVA_HOME} ]]; then - echo "Error: JAVA_HOME is not set" +if [ -z "$JAVA_HOME" ]; then + export JAVACMD=$(which java) + JAVAP=$(which javap) +else + export JAVA="${JAVA_HOME}/bin/java" + JAVAP="${JAVA_HOME}/bin/javap" +fi + +if [ ! -x "$JAVA" ]; then + echo "The JAVA_HOME environment variable is not defined correctly" + echo "This environment variable is needed to run this program" + echo "NB: JAVA_HOME should point to a JDK not a JRE" exit 1 fi -# check java version -export JAVA=${JAVA_HOME}/bin/java -JAVAP=${JAVA_HOME}/bin/javap JAVA_VER=$(${JAVAP} -verbose java.lang.String | grep "major version" | cut -d " " -f5) if [[ $JAVA_VER -lt 52 ]]; then echo "Error: require JAVA with JDK version at least 1.8" @@ -122,7 +129,7 @@ export CMAKE_CMD GENERATOR="Unix Makefiles" BUILD_SYSTEM="make" -if ninja --version 2>/dev/null; then +if ninja --version 2> /dev/null; then GENERATOR="Ninja" BUILD_SYSTEM="ninja" fi diff --git a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh index d0263d25f9..6fc6fb97d7 100755 --- a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh +++ b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh @@ -16,49 +16,72 @@ # specific language governing permissions and limitations # under the License. -curdir=`dirname "$0"` -curdir=`cd "$curdir"; pwd` +curdir=$(dirname "$0") +curdir=$( + cd "$curdir" + pwd +) OPTS=$(getopt \ - -n $0 \ - -o '' \ - -l 'daemon' \ - -- "$@") + -n $0 \ + -o '' \ + -l 'daemon' \ + -- "$@") eval set -- "$OPTS" RUN_DAEMON=0 while true; do case "$1" in - --daemon) RUN_DAEMON=1 ; shift ;; - --) shift ; break ;; - *) ehco "Internal error" ; exit 1 ;; + --daemon) + RUN_DAEMON=1 + shift + ;; + --) + shift + break + ;; + *) + ehco "Internal error" + exit 1 + ;; esac done -export BROKER_HOME=`cd "$curdir/.."; pwd` -export PID_DIR=`cd "$curdir"; pwd` +export BROKER_HOME=$( + cd "$curdir/.." + pwd +) +export PID_DIR=$( + cd "$curdir" + pwd +) export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8" export BROKER_LOG_DIR="$BROKER_HOME/log" -# export JAVA_HOME="/usr/java/jdk1.8.0_131" # java -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 +if [ -z "$JAVA_HOME" ]; then + JAVA=$(which java) +else + JAVA="$JAVA_HOME/bin/java" fi -JAVA=$JAVA_HOME/bin/java +if [ ! -x "$JAVA" ]; then + echo "The JAVA_HOME environment variable is not defined correctly" + echo "This environment variable is needed to run this program" + echo "NB: JAVA_HOME should point to a JDK not a JRE" + exit 1 +fi # add libs to CLASSPATH for f in $BROKER_HOME/lib/*.jar; do - CLASSPATH=$f:${CLASSPATH}; + CLASSPATH=$f:${CLASSPATH} done export CLASSPATH=${CLASSPATH}:${BROKER_HOME}/lib:$BROKER_HOME/conf while read line; do - envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="` - envline=`eval "echo $envline"` + envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") + envline=$(eval "echo $envline") if [[ $envline == *"="* ]]; then eval 'export "$envline"' fi @@ -67,8 +90,8 @@ done < $BROKER_HOME/conf/apache_hdfs_broker.conf pidfile=$PID_DIR/apache_hdfs_broker.pid if [ -f $pidfile ]; then - if kill -0 `cat $pidfile` > /dev/null 2>&1; then - echo "Broker running as process `cat $pidfile`. Stop it first." + if kill -0 $(cat $pidfile) > /dev/null 2>&1; then + echo "Broker running as process $(cat $pidfile). Stop it first." exit 1 fi fi @@ -77,12 +100,12 @@ if [ ! -d $BROKER_LOG_DIR ]; then mkdir -p $BROKER_LOG_DIR fi -echo `date` >> $BROKER_LOG_DIR/apache_hdfs_broker.out +echo $(date) >> $BROKER_LOG_DIR/apache_hdfs_broker.out if [ ${RUN_DAEMON} -eq 1 ]; then - nohup $LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 > $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 < /dev/null & else - $LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 > $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 < /dev/null fi echo $! > $pidfile diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index 0b39001636..2937cdb9aa 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -20,8 +20,8 @@ # This script is used to build Doris used in Baidu ############################################################## -# the script generates src/be/src/common/version.h and -# fe/src/main/java/org/apache/doris/common/Version.java which +# the script generates src/be/src/common/version.h and +# fe/src/main/java/org/apache/doris/common/Version.java which # contains the build version based on the git hash or svn revision. ############################################################## @@ -30,39 +30,40 @@ build_version="trunk" unset LANG unset LC_CTYPE -user=`whoami` -date=`date +"%a, %d %b %Y %H:%M:%S %Z"` -hostname=`hostname` +user=$(whoami) +date=$(date +"%a, %d %b %Y %H:%M:%S %Z") +hostname=$(hostname) -cwd=`pwd` +cwd=$(pwd) -if [ -z ${DORIS_HOME+x} ] -then - ROOT=`dirname "$0"` - ROOT=`cd "$ROOT"; pwd` +if [ -z ${DORIS_HOME+x} ]; then + ROOT=$(dirname "$0") + ROOT=$( + cd "$ROOT" + pwd + ) DORIS_HOME=${ROOT}/../.. echo "DORIS_HOME: ${DORIS_HOME}" fi if [[ -z ${DORIS_TEST_BINARY_DIR} ]]; then if [ -e ${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common/Version.java \ - -a -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then + -a -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then exit fi fi cd ${DORIS_HOME} if [ -d .svn ]; then - revision=`svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'` + revision=$(svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p') short_revision="${revision}" - url=`svn info | sed -n -e 's/^URL: \(.*\)/\1/p'` - if echo ${url} | grep '\/tags\/' > /dev/null - then - build_version="`echo ${url} | sed 's/.*_\([0-9-]\+\)_PD_BL.*/\1/g' | sed 's/-/\./g'`" + url=$(svn info | sed -n -e 's/^URL: \(.*\)/\1/p') + if echo ${url} | grep '\/tags\/' > /dev/null; then + build_version="$(echo ${url} | sed 's/.*_\([0-9-]\+\)_PD_BL.*/\1/g' | sed 's/-/\./g')" fi elif [ -d .git ]; then - revision=`git log -1 --pretty=format:"%H"` - short_revision=`git log -1 --pretty=format:"%h"` + revision=$(git log -1 --pretty=format:"%H") + short_revision=$(git log -1 --pretty=format:"%h") url="git://${hostname}${DORIS_HOME}" else revision="Unknown" @@ -77,15 +78,19 @@ build_short_hash="${short_revision}" build_time="${date}" build_info="${user}@${hostname}" -java_cmd= -if [[ (-n "$JAVA_HOME") && (-x "$JAVA_HOME/bin/java") ]]; then - java_cmd="$JAVA_HOME/bin/java" +if [ -z "$JAVA_HOME" ]; then + java_cmd=$(which java) else - echo "JAVA_HOME is not set, or java bin is not found" - exit -1 + java_cmd="$JAVA_HOME/bin/java" fi -java_version_str=`$java_cmd -fullversion 2>&1` +if [ ! -x "$java_cmd" ]; then + echo "The JAVA_HOME environment variable is not defined correctly" + echo "This environment variable is needed to run this program" + echo "NB: JAVA_HOME should point to a JDK not a JRE" + exit 1 +fi +java_version_str=$($java_cmd -fullversion 2>&1) java_version_str=$(echo $java_version_str | sed -e 's/"/\\"/g') echo "get java cmd: $java_cmd" @@ -93,7 +98,7 @@ echo "get java version: $java_version_str" VERSION_PACKAGE="${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common" mkdir -p ${VERSION_PACKAGE} -cat >"${VERSION_PACKAGE}/Version.java" < "${VERSION_PACKAGE}/Version.java" << EOF // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -140,7 +145,7 @@ EOF GEN_CPP_DIR=${DORIS_HOME}/gensrc/build/gen_cpp/ mkdir -p ${GEN_CPP_DIR} -cat >"${GEN_CPP_DIR}/version.h" < "${GEN_CPP_DIR}/version.h" << EOF // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information