diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index c500983541..38925140d4 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -35,4 +35,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: sh_checker_comment: true - sh_checker_exclude: .git .github ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/ssb-tools ^tools/tpch-tools ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output + sh_checker_exclude: .git .github ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output diff --git a/tools/ssb-tools/bin/build-ssb-dbgen.sh b/tools/ssb-tools/bin/build-ssb-dbgen.sh index 56fd2056e4..b7141f87fb 100755 --- a/tools/ssb-tools/bin/build-ssb-dbgen.sh +++ b/tools/ssb-tools/bin/build-ssb-dbgen.sh @@ -27,34 +27,36 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" + cd "${ROOT}" pwd ) -CURDIR=${ROOT} -SSB_DBGEN_DIR=$CURDIR/ssb-dbgen/ +CURDIR="${ROOT}" +SSB_DBGEN_DIR="${CURDIR}/ssb-dbgen/" # download ssb-dbgen first -if [[ -d $SSB_DBGEN_DIR ]]; then - echo "Dir $CURDIR/ssb-dbgen/ already exists. No need to download." +if [[ -d ${SSB_DBGEN_DIR} ]]; then + echo "Dir ${CURDIR}/ssb-dbgen/ already exists. No need to download." echo "If you want to download ssb-dbgen again, please delete this dir first." exit 1 else - cd "$CURDIR" - wget https://palo-cloud-repo-bd.bd.bcebos.com/baidu-doris-release/ssb-dbgen-linux.tar.gz && tar -xzvf ssb-dbgen-linux.tar.gz -C $CURDIR/ + cd "${CURDIR}" + wget https://palo-cloud-repo-bd.bd.bcebos.com/baidu-doris-release/ssb-dbgen-linux.tar.gz + tar -xzvf ssb-dbgen-linux.tar.gz -C "${CURDIR}"/ fi # compile ssb-dbgen -cd "$SSB_DBGEN_DIR/" && make +cd "${SSB_DBGEN_DIR}/" +make cd - # check -if [[ -f $CURDIR/ssb-dbgen/dbgen ]]; then +if [[ -f ${CURDIR}/ssb-dbgen/dbgen ]]; then echo -e " ################ Build succeed! ################ -Run $CURDIR/ssb-dbgen/dbgen -h" +Run ${CURDIR}/ssb-dbgen/dbgen -h" exit 0 else echo "Build failed!" diff --git a/tools/ssb-tools/bin/create-ssb-tables.sh b/tools/ssb-tools/bin/create-ssb-tables.sh index 41177843e2..4980c52caf 100755 --- a/tools/ssb-tools/bin/create-ssb-tables.sh +++ b/tools/ssb-tools/bin/create-ssb-tables.sh @@ -24,83 +24,82 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" - pwd + cd "${ROOT}" + pwd ) -CURDIR=${ROOT} +CURDIR="${ROOT}" SSB_DDL="${CURDIR}/../ddl/create-ssb-tables.sql" SSB_FLAT_DDL="${CURDIR}/../ddl/create-ssb-flat-table.sql" usage() { - echo " + echo " This script is used to create SSB tables, will use mysql client to connect Doris server which is specified in conf/doris-cluster.conf file. Usage: $0 " - exit 1 + exit 1 } OPTS=$(getopt \ - -n "$0" \ - -o '' \ - -o 'h' \ - -- "$@") + -n "$0" \ + -o '' \ + -o 'h' \ + -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" HELP=0 -if [ $# == 0 ]; then - usage +if [[ $# == 0 ]]; then + usage fi while true; do - case "$1" in - -h) - HELP=1 - shift - ;; - --) - shift - break - ;; - *) - echo "Internal error" - exit 1 - ;; - esac + case "$1" in + -h) + HELP=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; + esac done if [[ ${HELP} -eq 1 ]]; then - usage - exit + usage + exit fi check_prerequest() { - local CMD=$1 - local NAME=$2 - if ! $CMD; then - echo "$NAME is missing. This script depends on mysql to create tables in Doris." - exit 1 - fi + local CMD=$1 + local NAME=$2 + if ! ${CMD}; then + echo "${NAME} is missing. This script depends on mysql to create tables in Doris." + exit 1 + fi } check_prerequest "mysql --version" "mysql" -# shellcheck source=/dev/null -source "$CURDIR/../conf/doris-cluster.conf" -export MYSQL_PWD=$PASSWORD +source "${CURDIR}/../conf/doris-cluster.conf" +export MYSQL_PWD="${PASSWORD}" -echo "FE_HOST: $FE_HOST" -echo "FE_QUERY_PORT: $FE_QUERY_PORT" -echo "USER: $USER" -echo "PASSWORD: $PASSWORD" -echo "DB: $DB" +echo "FE_HOST: ${FE_HOST}" +echo "FE_QUERY_PORT: ${FE_QUERY_PORT}" +echo "USER: ${USER}" +echo "PASSWORD: ${PASSWORD}" +echo "DB: ${DB}" -mysql -h"$FE_HOST" -u"$USER" -P"$FE_QUERY_PORT" -e "CREATE DATABASE IF NOT EXISTS $DB" +mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -e "CREATE DATABASE IF NOT EXISTS ${DB}" -echo "Run DDL from $SSB_DDL" -mysql -h"$FE_HOST" -u"$USER" -P"$FE_QUERY_PORT" -D"$DB" <"$SSB_DDL" +echo "Run DDL from ${SSB_DDL}" +mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" <"${SSB_DDL}" -echo "Run DDL from $SSB_FLAT_DDL" -mysql -h"$FE_HOST" -u"$USER" -P"$FE_QUERY_PORT" -D"$DB" <"$SSB_FLAT_DDL" +echo "Run DDL from ${SSB_FLAT_DDL}" +mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" <"${SSB_FLAT_DDL}" diff --git a/tools/ssb-tools/bin/gen-ssb-data.sh b/tools/ssb-tools/bin/gen-ssb-data.sh index afc9e8bf68..4f06f9ead1 100755 --- a/tools/ssb-tools/bin/gen-ssb-data.sh +++ b/tools/ssb-tools/bin/gen-ssb-data.sh @@ -24,16 +24,16 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" - pwd + cd "${ROOT}" + pwd ) CURDIR=${ROOT} -SSB_DBGEN_DIR=$CURDIR/ssb-dbgen/ -SSB_DATA_DIR=$CURDIR/ssb-data/ +SSB_DBGEN_DIR=${CURDIR}/ssb-dbgen/ +SSB_DATA_DIR=${CURDIR}/ssb-data/ usage() { - echo " + echo " Usage: $0 Optional options: -s scale factor, default is 100 @@ -44,87 +44,87 @@ Usage: $0 $0 -s 10 generate data with scale factor 10. $0 -s 10 -c 5 generate data with scale factor 10. And using 5 threads to generate data concurrently. " - exit 1 + exit 1 } OPTS=$(getopt \ - -n "$0" \ - -o '' \ - -o 'hs:c:' \ - -- "$@") + -n "$0" \ + -o '' \ + -o 'hs:c:' \ + -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" SCALE_FACTOR=100 PARALLEL=10 HELP=0 -if [ $# == 0 ]; then - usage +if [[ $# == 0 ]]; then + usage fi while true; do - case "$1" in - -h) - HELP=1 - shift - ;; - -s) - SCALE_FACTOR=$2 - shift 2 - ;; - -c) - PARALLEL=$2 - shift 2 - ;; - --) - shift - break - ;; - *) - echo "Internal error" - exit 1 - ;; - esac + case "$1" in + -h) + HELP=1 + shift + ;; + -s) + SCALE_FACTOR=$2 + shift 2 + ;; + -c) + PARALLEL=$2 + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; + esac done if [[ ${HELP} -eq 1 ]]; then - usage - exit + usage + exit fi -echo "Scale Factor: $SCALE_FACTOR" -echo "Parallelism: $PARALLEL" +echo "Scale Factor: ${SCALE_FACTOR}" +echo "Parallelism: ${PARALLEL}" # check if dbgen exists -if [[ ! -f $SSB_DBGEN_DIR/dbgen ]]; then - echo "$SSB_DBGEN_DIR/dbgen does not exist. Run build-ssb-dbgen.sh first to build it first." - exit 1 +if [[ ! -f ${SSB_DBGEN_DIR}/dbgen ]]; then + echo "${SSB_DBGEN_DIR}/dbgen does not exist. Run build-ssb-dbgen.sh first to build it first." + exit 1 fi -if [[ -d $SSB_DATA_DIR/ ]]; then - echo "$SSB_DATA_DIR exists. Remove it before generating data" - exit 1 +if [[ -d ${SSB_DATA_DIR}/ ]]; then + echo "${SSB_DATA_DIR} exists. Remove it before generating data" + exit 1 fi -mkdir "$SSB_DATA_DIR/" +mkdir "${SSB_DATA_DIR}/" # gen data -cd "$SSB_DBGEN_DIR" +cd "${SSB_DBGEN_DIR}" echo "Begin to generate data for table: customer" -"$SSB_DBGEN_DIR/dbgen" -f -s "$SCALE_FACTOR" -T c +"${SSB_DBGEN_DIR}/dbgen" -f -s "${SCALE_FACTOR}" -T c echo "Begin to generate data for table: part" -"$SSB_DBGEN_DIR/dbgen" -f -s "$SCALE_FACTOR" -T p +"${SSB_DBGEN_DIR}/dbgen" -f -s "${SCALE_FACTOR}" -T p echo "Begin to generate data for table: supplier" -"$SSB_DBGEN_DIR/dbgen" -f -s "$SCALE_FACTOR" -T s +"${SSB_DBGEN_DIR}/dbgen" -f -s "${SCALE_FACTOR}" -T s echo "Begin to generate data for table: date" -"$SSB_DBGEN_DIR/dbgen" -f -s "$SCALE_FACTOR" -T d +"${SSB_DBGEN_DIR}/dbgen" -f -s "${SCALE_FACTOR}" -T d echo "Begin to generate data for table: lineorder" -"$SSB_DBGEN_DIR/dbgen" -f -s "$SCALE_FACTOR" -T l -C "$PARALLEL" +"${SSB_DBGEN_DIR}/dbgen" -f -s "${SCALE_FACTOR}" -T l -C "${PARALLEL}" cd - # move data to $SSB_DATA_DIR -mv "$SSB_DBGEN_DIR"/*.tbl* "$SSB_DATA_DIR/" +mv "${SSB_DBGEN_DIR}"/*.tbl* "${SSB_DATA_DIR}/" # check data -du -sh "$SSB_DATA_DIR"/*.tbl* +du -sh "${SSB_DATA_DIR}"/*.tbl* diff --git a/tools/ssb-tools/bin/load-ssb-data.sh b/tools/ssb-tools/bin/load-ssb-data.sh index f6be6bbe72..bda0d758a2 100755 --- a/tools/ssb-tools/bin/load-ssb-data.sh +++ b/tools/ssb-tools/bin/load-ssb-data.sh @@ -25,12 +25,12 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" + cd "${ROOT}" pwd ) -CURDIR=${ROOT} -SSB_DATA_DIR="$CURDIR/ssb-data/" +CURDIR="${ROOT}" +SSB_DATA_DIR="${CURDIR}/ssb-data/" usage() { echo " @@ -51,12 +51,12 @@ OPTS=$(getopt \ -o 'hc:' \ -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" PARALLEL=5 HELP=0 -if [ $# == 0 ]; then +if [[ $# == 0 ]]; then usage fi @@ -86,33 +86,33 @@ if [[ ${HELP} -eq 1 ]]; then exit fi -echo "Parallelism: $PARALLEL" +echo "Parallelism: ${PARALLEL}" # check if ssb-data exists -if [[ ! -d $SSB_DATA_DIR/ ]]; then - echo "$SSB_DATA_DIR does not exist. Run sh gen-ssb-data.sh first." +if [[ ! -d ${SSB_DATA_DIR}/ ]]; then + echo "${SSB_DATA_DIR} does not exist. Run sh gen-ssb-data.sh first." exit 1 fi check_prerequest() { local CMD=$1 local NAME=$2 - if ! $CMD; then - echo "$NAME is missing. This script depends on cURL to load data to Doris." + if ! ${CMD}; then + echo "${NAME} is missing. This script depends on cURL to load data to Doris." exit 1 fi } run_sql() { sql="$*" - echo "$sql" - mysql -h"$FE_HOST" -u"$USER" -P"$FE_QUERY_PORT" -D"$DB" -e "$@" + echo "${sql}" + mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" -e "$@" } load_lineitem_flat() { # Loading data in batches by year. for con in 'lo_orderdate<19930101' 'lo_orderdate>=19930101 and lo_orderdate<19940101' 'lo_orderdate>=19940101 and lo_orderdate<19950101' 'lo_orderdate>=19950101 and lo_orderdate<19960101' 'lo_orderdate>=19960101 and lo_orderdate<19970101' 'lo_orderdate>=19970101 and lo_orderdate<19980101' 'lo_orderdate>=19980101'; do - echo -e "\n$con" + echo -e "\n${con}" run_sql " INSERT INTO lineorder_flat SELECT @@ -189,53 +189,52 @@ ON (p.p_partkey = l.lo_partkey); check_prerequest "curl --version" "curl" # load lineorder -# shellcheck source=/dev/null -source "$CURDIR/../conf/doris-cluster.conf" -export MYSQL_PWD=$PASSWORD +source "${CURDIR}/../conf/doris-cluster.conf" +export MYSQL_PWD=${PASSWORD} -echo "FE_HOST: $FE_HOST" -echo "FE_HTTP_PORT: $FE_HTTP_PORT" -echo "USER: $USER" -echo "PASSWORD: $PASSWORD" -echo "DB: $DB" +echo "FE_HOST: ${FE_HOST}" +echo "FE_HTTP_PORT: ${FE_HTTP_PORT}" +echo "USER: ${USER}" +echo "PASSWORD: ${PASSWORD}" +echo "DB: ${DB}" date echo "==========Start to load data into ssb tables==========" echo 'Loading data for table: part' -curl --location-trusted -u "$USER":"$PASSWORD" \ +curl --location-trusted -u "${USER}":"${PASSWORD}" \ -H "column_separator:|" \ -H "columns:p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy" \ - -T "$SSB_DATA_DIR"/part.tbl http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/part/_stream_load + -T "${SSB_DATA_DIR}"/part.tbl http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/part/_stream_load echo 'Loading data for table: date' -curl --location-trusted -u "$USER":"$PASSWORD" \ +curl --location-trusted -u "${USER}":"${PASSWORD}" \ -H "column_separator:|" \ -H "columns:d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth,d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear,d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy" \ - -T "$SSB_DATA_DIR"/date.tbl http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/dates/_stream_load + -T "${SSB_DATA_DIR}"/date.tbl http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/dates/_stream_load echo 'Loading data for table: supplier' -curl --location-trusted -u "$USER":"$PASSWORD" \ +curl --location-trusted -u "${USER}":"${PASSWORD}" \ -H "column_separator:|" \ -H "columns:s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy" \ - -T "$SSB_DATA_DIR"/supplier.tbl http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/supplier/_stream_load + -T "${SSB_DATA_DIR}"/supplier.tbl http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/supplier/_stream_load echo 'Loading data for table: customer' -curl --location-trusted -u "$USER":"$PASSWORD" \ +curl --location-trusted -u "${USER}":"${PASSWORD}" \ -H "column_separator:|" \ -H "columns:c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use" \ - -T "$SSB_DATA_DIR"/customer.tbl http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/customer/_stream_load + -T "${SSB_DATA_DIR}"/customer.tbl http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/customer/_stream_load -echo "Loading data for table: lineorder, with $PARALLEL parallel" +echo "Loading data for table: lineorder, with ${PARALLEL} parallel" function load() { echo "$@" - curl --location-trusted -u "$USER":"$PASSWORD" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" \ -H "column_separator:|" \ -H "columns:lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority,lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount,lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy" \ - -T "$@" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/lineorder/_stream_load + -T "$@" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/lineorder/_stream_load } # set parallelism -[ -e /tmp/fd1 ] || mkfifo /tmp/fd1 +[[ -e /tmp/fd1 ]] || mkfifo /tmp/fd1 exec 3<>/tmp/fd1 rm -rf /tmp/fd1 @@ -244,10 +243,10 @@ for ((i = 1; i <= PARALLEL; i++)); do done date -for file in "$SSB_DATA_DIR"/lineorder.tbl.*; do +for file in "${SSB_DATA_DIR}"/lineorder.tbl.*; do read -r -u3 { - load "$file" + load "${file}" echo >&3 } & done @@ -258,8 +257,14 @@ date echo "==========Start to insert data into ssb flat table==========" echo "change some session variables before load, and then restore after load." -origin_query_timeout=$(run_sql 'select @@query_timeout;' | sed -n '3p') -origin_parallel=$(run_sql 'select @@parallel_fragment_exec_instance_num;' | sed -n '3p') +origin_query_timeout=$( + set -e + run_sql 'select @@query_timeout;' | sed -n '3p' +) +origin_parallel=$( + set -e + run_sql 'select @@parallel_fragment_exec_instance_num;' | sed -n '3p' +) # set parallel_fragment_exec_instance_num=1, loading maybe slow but stable. run_sql "set global query_timeout=7200;" run_sql "set global parallel_fragment_exec_instance_num=1;" diff --git a/tools/ssb-tools/bin/run-ssb-flat-queries.sh b/tools/ssb-tools/bin/run-ssb-flat-queries.sh index 6db5ba3303..74d4b71691 100755 --- a/tools/ssb-tools/bin/run-ssb-flat-queries.sh +++ b/tools/ssb-tools/bin/run-ssb-flat-queries.sh @@ -24,12 +24,12 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" + cd "${ROOT}" pwd ) CURDIR="${ROOT}" -QUERIES_DIR="$CURDIR/../ssb-flat-queries" +QUERIES_DIR="${CURDIR}/../ssb-flat-queries" usage() { echo " @@ -46,10 +46,10 @@ OPTS=$(getopt \ -o 'h' \ -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" HELP=0 -if [ $# == 0 ]; then +if [[ $# == 0 ]]; then usage fi @@ -78,27 +78,26 @@ fi check_prerequest() { local CMD=$1 local NAME=$2 - if ! $CMD; then - echo "$NAME is missing. This script depends on mysql to create tables in Doris." + if ! ${CMD}; then + echo "${NAME} is missing. This script depends on mysql to create tables in Doris." exit 1 fi } check_prerequest "mysqlslap --version" "mysqlslap" -# shellcheck source=/dev/null -source "$CURDIR/../conf/doris-cluster.conf" -export MYSQL_PWD=$PASSWORD +source "${CURDIR}/../conf/doris-cluster.conf" +export MYSQL_PWD=${PASSWORD} -echo "FE_HOST: $FE_HOST" -echo "FE_QUERY_PORT: $FE_QUERY_PORT" -echo "USER: $USER" -echo "PASSWORD: $PASSWORD" -echo "DB: $DB" +echo "FE_HOST: ${FE_HOST}" +echo "FE_QUERY_PORT: ${FE_QUERY_PORT}" +echo "USER: ${USER}" +echo "PASSWORD: ${PASSWORD}" +echo "DB: ${DB}" pre_set() { echo "$@" - mysql -h"$FE_HOST" -P"$FE_QUERY_PORT" -u"$USER" -D"$DB" -e "$@" + mysql -h"${FE_HOST}" -P"${FE_QUERY_PORT}" -u"${USER}" -D"${DB}" -e "$@" } pre_set "set global enable_vectorized_engine=1;" @@ -113,8 +112,8 @@ echo '============================================' for i in '1.1' '1.2' '1.3' '2.1' '2.2' '2.3' '3.1' '3.2' '3.3' '3.4' '4.1' '4.2' '4.3'; do # First run to prevent the affect of cold start - mysql -h"$FE_HOST" -P"$FE_QUERY_PORT" -u"$USER" -D "$DB" <"$QUERIES_DIR"/q${i}.sql >/dev/null 2>&1 + mysql -h"${FE_HOST}" -P"${FE_QUERY_PORT}" -u"${USER}" -D "${DB}" <"${QUERIES_DIR}/q${i}.sql" >/dev/null 2>&1 # Then run 3 times and takes the average time - res=$(mysqlslap -h"$FE_HOST" -P"$FE_QUERY_PORT" -u"$USER" --create-schema="$DB" --query="$QUERIES_DIR"/q${i}.sql -F '\r' -i 3 | sed -n '2p' | cut -d ' ' -f 9,10) - echo "q$i: $res" + res=$(mysqlslap -h"${FE_HOST}" -P"${FE_QUERY_PORT}" -u"${USER}" --create-schema="${DB}" --query="${QUERIES_DIR}/q${i}.sql" -F '\r' -i 3 | sed -n '2p' | cut -d ' ' -f 9,10) + echo "q${i}: ${res}" done diff --git a/tools/ssb-tools/bin/run-ssb-queries.sh b/tools/ssb-tools/bin/run-ssb-queries.sh index 5a1db9a57e..035a4dab11 100755 --- a/tools/ssb-tools/bin/run-ssb-queries.sh +++ b/tools/ssb-tools/bin/run-ssb-queries.sh @@ -24,12 +24,12 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" + cd "${ROOT}" pwd ) CURDIR=${ROOT} -QUERIES_DIR=$CURDIR/../ssb-queries +QUERIES_DIR=${CURDIR}/../ssb-queries usage() { echo " @@ -46,10 +46,10 @@ OPTS=$(getopt \ -o 'h' \ -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" HELP=0 -if [ $# == 0 ]; then +if [[ $# == 0 ]]; then usage fi @@ -78,27 +78,26 @@ fi check_prerequest() { local CMD=$1 local NAME=$2 - if ! $CMD; then - echo "$NAME is missing. This script depends on mysql to create tables in Doris." + if ! ${CMD}; then + echo "${NAME} is missing. This script depends on mysql to create tables in Doris." exit 1 fi } check_prerequest "mysqlslap --version" "mysql slap" -# shellcheck source=/dev/null -source "$CURDIR/../conf/doris-cluster.conf" -export MYSQL_PWD=$PASSWORD +source "${CURDIR}/../conf/doris-cluster.conf" +export MYSQL_PWD=${PASSWORD} -echo "FE_HOST: $FE_HOST" -echo "FE_QUERY_PORT: $FE_QUERY_PORT" -echo "USER: $USER" -echo "PASSWORD: $PASSWORD" -echo "DB: $DB" +echo "FE_HOST: ${FE_HOST}" +echo "FE_QUERY_PORT: ${FE_QUERY_PORT}" +echo "USER: ${USER}" +echo "PASSWORD: ${PASSWORD}" +echo "DB: ${DB}" pre_set() { echo "$@" - mysql -h"$FE_HOST" -P"$FE_QUERY_PORT" -u"$USER" -D"$DB" -e "$@" + mysql -h"${FE_HOST}" -P"${FE_QUERY_PORT}" -u"${USER}" -D"${DB}" -e "$@" } pre_set "set global enable_vectorized_engine=1;" @@ -116,6 +115,6 @@ echo '============================================' for i in '1.1' '1.2' '1.3' '2.1' '2.2' '2.3' '3.1' '3.2' '3.3' '3.4' '4.1' '4.2' '4.3'; do # Each query is executed 3 times and takes the average time - res=$(mysqlslap -h"$FE_HOST" -P"$FE_QUERY_PORT" -u"$USER" --create-schema="$DB" --query="$QUERIES_DIR"/q${i}.sql -F '\r' -i 3 | sed -n '2p' | cut -d ' ' -f 9,10) - echo "q$i: $res" + res=$(mysqlslap -h"${FE_HOST}" -P"${FE_QUERY_PORT}" -u"${USER}" --create-schema="${DB}" --query="${QUERIES_DIR}/q${i}.sql" -F '\r' -i 3 | sed -n '2p' | cut -d ' ' -f 9,10) + echo "q${i}: ${res}" done diff --git a/tools/tpch-tools/bin/build-tpch-dbgen.sh b/tools/tpch-tools/bin/build-tpch-dbgen.sh index b0cce4b1da..c65ce1ca41 100755 --- a/tools/tpch-tools/bin/build-tpch-dbgen.sh +++ b/tools/tpch-tools/bin/build-tpch-dbgen.sh @@ -27,18 +27,18 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" + cd "${ROOT}" pwd ) -CURDIR=${ROOT} -TPCH_DBGEN_DIR=$CURDIR/TPC-H_Tools_v3.0.0/dbgen +CURDIR="${ROOT}" +TPCH_DBGEN_DIR="${CURDIR}/TPC-H_Tools_v3.0.0/dbgen" check_prerequest() { local CMD=$1 local NAME=$2 - if ! $CMD; then - echo "$NAME is missing. This script depends on unzip to extract files from TPC-H_Tools_v3.0.0.zip" + if ! ${CMD}; then + echo "${NAME} is missing. This script depends on unzip to extract files from TPC-H_Tools_v3.0.0.zip" exit 1 fi } @@ -46,16 +46,16 @@ check_prerequest() { check_prerequest "unzip -h" "unzip" # download tpch tools pacage first -if [[ -d $TPCH_DBGEN_DIR ]]; then - echo "Dir $TPCH_DBGEN_DIR already exists. No need to download." +if [[ -d ${TPCH_DBGEN_DIR} ]]; then + echo "Dir ${TPCH_DBGEN_DIR} already exists. No need to download." echo "If you want to download TPC-H_Tools_v3.0.0 again, please delete this dir first." else wget "https://tools-chengdu.oss-cn-chengdu.aliyuncs.com/TPC-H_Tools_v3.0.0.zip" - unzip TPC-H_Tools_v3.0.0.zip -d "$CURDIR"/ + unzip TPC-H_Tools_v3.0.0.zip -d "${CURDIR}/" fi # modify tpcd.h -cd "$TPCH_DBGEN_DIR"/ +cd "${TPCH_DBGEN_DIR}/" printf '%s' ' #ifdef MYSQL #define GEN_QUERY_PLAN "" @@ -79,12 +79,12 @@ make >/dev/null cd - # check -if [[ -f $TPCH_DBGEN_DIR/dbgen ]]; then +if [[ -f ${TPCH_DBGEN_DIR}/dbgen ]]; then echo " ################ Build succeed! ################ -Run $TPCH_DBGEN_DIR/dbgen -h" +Run ${TPCH_DBGEN_DIR}/dbgen -h" exit 0 else echo "Build failed!" diff --git a/tools/tpch-tools/bin/create-tpch-tables.sh b/tools/tpch-tools/bin/create-tpch-tables.sh index 01ca7a1345..59cd2efbd4 100755 --- a/tools/tpch-tools/bin/create-tpch-tables.sh +++ b/tools/tpch-tools/bin/create-tpch-tables.sh @@ -24,77 +24,76 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" - pwd + cd "${ROOT}" + pwd ) CURDIR=${ROOT} usage() { - echo " + echo " This script is used to create TPC-H tables, will use mysql client to connect Doris server which is specified in doris-cluster.conf file. Usage: $0 " - exit 1 + exit 1 } OPTS=$(getopt \ - -n "$0" \ - -o '' \ - -- "$@") + -n "$0" \ + -o '' \ + -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" HELP=0 -if [ $# == 0 ]; then - usage +if [[ $# == 0 ]]; then + usage fi while true; do - case "$1" in - -h) - HELP=1 - shift - ;; - --) - shift - break - ;; - *) - echo "Internal error" - exit 1 - ;; - esac + case "$1" in + -h) + HELP=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; + esac done if [[ ${HELP} -eq 1 ]]; then - usage - exit + usage + exit fi check_prerequest() { - local CMD=$1 - local NAME=$2 - if ! $CMD; then - echo "$NAME is missing. This script depends on mysql to create tables in Doris." - exit 1 - fi + local CMD=$1 + local NAME=$2 + if ! ${CMD}; then + echo "${NAME} is missing. This script depends on mysql to create tables in Doris." + exit 1 + fi } check_prerequest "mysql --version" "mysql" -# shellcheck source=/dev/null -source "$CURDIR/../conf/doris-cluster.conf" -export MYSQL_PWD=$PASSWORD +source "${CURDIR}/../conf/doris-cluster.conf" +export MYSQL_PWD=${PASSWORD} -echo "FE_HOST: $FE_HOST" -echo "FE_QUERY_PORT: $FE_QUERY_PORT" -echo "USER: $USER" -echo "PASSWORD: $PASSWORD" -echo "DB: $DB" +echo "FE_HOST: ${FE_HOST}" +echo "FE_QUERY_PORT: ${FE_QUERY_PORT}" +echo "USER: ${USER}" +echo "PASSWORD: ${PASSWORD}" +echo "DB: ${DB}" -mysql -h"$FE_HOST" -u"$USER" -P"$FE_QUERY_PORT" -e "CREATE DATABASE IF NOT EXISTS $DB" +mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -e "CREATE DATABASE IF NOT EXISTS ${DB}" -echo "Run SQLs from $CURDIR/create-tpch-tables.sql" -mysql -h"$FE_HOST" -u"$USER" -P"$FE_QUERY_PORT" -D"$DB" <"$CURDIR"/../ddl/create-tpch-tables.sql +echo "Run SQLs from ${CURDIR}/create-tpch-tables.sql" +mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" <"${CURDIR}"/../ddl/create-tpch-tables.sql diff --git a/tools/tpch-tools/bin/gen-tpch-data.sh b/tools/tpch-tools/bin/gen-tpch-data.sh index 4202e3c58d..bcac551e88 100755 --- a/tools/tpch-tools/bin/gen-tpch-data.sh +++ b/tools/tpch-tools/bin/gen-tpch-data.sh @@ -24,16 +24,16 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" - pwd + cd "${ROOT}" + pwd ) -CURDIR=${ROOT} -TPCH_DBGEN_DIR=$CURDIR/TPC-H_Tools_v3.0.0/dbgen/ -TPCH_DATA_DIR=$CURDIR/tpch-data/ +CURDIR="${ROOT}" +TPCH_DBGEN_DIR="${CURDIR}/TPC-H_Tools_v3.0.0/dbgen/" +TPCH_DATA_DIR="${CURDIR}/tpch-data/" usage() { - echo " + echo " Usage: $0 Optional options: -s scale factor, default is 100 @@ -44,111 +44,111 @@ Usage: $0 $0 -s 10 generate data with scale factor 10. $0 -s 10 -c 5 generate data with scale factor 10. And using 5 threads to generate data concurrently. " - exit 1 + exit 1 } OPTS=$(getopt \ - -n "$0" \ - -o '' \ - -o 'hs:c:' \ - -- "$@") + -n "$0" \ + -o '' \ + -o 'hs:c:' \ + -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" SCALE_FACTOR=100 PARALLEL=10 HELP=0 -if [ $# == 0 ]; then - usage +if [[ $# == 0 ]]; then + usage fi while true; do - case "$1" in - -h) - HELP=1 - shift - ;; - -s) - SCALE_FACTOR=$2 - shift 2 - ;; - -c) - PARALLEL=$2 - shift 2 - ;; - --) - shift - break - ;; - *) - echo "Internal error" - exit 1 - ;; - esac + case "$1" in + -h) + HELP=1 + shift + ;; + -s) + SCALE_FACTOR=$2 + shift 2 + ;; + -c) + PARALLEL=$2 + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; + esac done if [[ ${HELP} -eq 1 ]]; then - usage - exit + usage + exit fi -echo "Scale Factor: $SCALE_FACTOR" -echo "Parallelism: $PARALLEL" +echo "Scale Factor: ${SCALE_FACTOR}" +echo "Parallelism: ${PARALLEL}" # check if dbgen exists -if [[ ! -f $TPCH_DBGEN_DIR/dbgen ]]; then - echo "$TPCH_DBGEN_DIR/dbgen does not exist. Run build-tpch-dbgen.sh first to build it first." - exit 1 +if [[ ! -f ${TPCH_DBGEN_DIR}/dbgen ]]; then + echo "${TPCH_DBGEN_DIR}/dbgen does not exist. Run build-tpch-dbgen.sh first to build it first." + exit 1 fi -if [[ -d $TPCH_DATA_DIR/ ]]; then - echo "$TPCH_DATA_DIR exists. Remove it before generating data" - exit 1 +if [[ -d ${TPCH_DATA_DIR}/ ]]; then + echo "${TPCH_DATA_DIR} exists. Remove it before generating data" + exit 1 fi -mkdir "$TPCH_DATA_DIR"/ +mkdir "${TPCH_DATA_DIR}"/ # gen data -cd "$TPCH_DBGEN_DIR" +cd "${TPCH_DBGEN_DIR}" echo "Begin to generate data for table: region" -"$TPCH_DBGEN_DIR"/dbgen -f -s "$SCALE_FACTOR" -T r +"${TPCH_DBGEN_DIR}"/dbgen -f -s "${SCALE_FACTOR}" -T r echo "Begin to generate data for table: nation" -"$TPCH_DBGEN_DIR"/dbgen -f -s "$SCALE_FACTOR" -T n +"${TPCH_DBGEN_DIR}"/dbgen -f -s "${SCALE_FACTOR}" -T n echo "Begin to generate data for table: supplier" -"$TPCH_DBGEN_DIR"/dbgen -f -s "$SCALE_FACTOR" -T s +"${TPCH_DBGEN_DIR}"/dbgen -f -s "${SCALE_FACTOR}" -T s echo "Begin to generate data for table: part" -"$TPCH_DBGEN_DIR"/dbgen -f -s "$SCALE_FACTOR" -T P +"${TPCH_DBGEN_DIR}"/dbgen -f -s "${SCALE_FACTOR}" -T P echo "Begin to generate data for table: customer" -"$TPCH_DBGEN_DIR"/dbgen -f -s "$SCALE_FACTOR" -T c +"${TPCH_DBGEN_DIR}"/dbgen -f -s "${SCALE_FACTOR}" -T c echo "Begin to generate data for table: partsupp" -for i in $(seq 1 "$PARALLEL"); do - { - "$TPCH_DBGEN_DIR"/dbgen -f -s "$SCALE_FACTOR" -T S -C "$PARALLEL" -S "$i" - } & +for i in $(seq 1 "${PARALLEL}"); do + { + "${TPCH_DBGEN_DIR}"/dbgen -f -s "${SCALE_FACTOR}" -T S -C "${PARALLEL}" -S "${i}" + } & done wait echo "Begin to generate data for table: orders" -for i in $(seq 1 "$PARALLEL"); do - { - "$TPCH_DBGEN_DIR"/dbgen -f -s "$SCALE_FACTOR" -T O -C "$PARALLEL" -S "$i" - } & +for i in $(seq 1 "${PARALLEL}"); do + { + "${TPCH_DBGEN_DIR}"/dbgen -f -s "${SCALE_FACTOR}" -T O -C "${PARALLEL}" -S "${i}" + } & done wait echo "Begin to generate data for table: lineitem" -for i in $(seq 1 "$PARALLEL"); do - { - "$TPCH_DBGEN_DIR"/dbgen -f -s "$SCALE_FACTOR" -T L -C "$PARALLEL" -S "$i" - } & +for i in $(seq 1 "${PARALLEL}"); do + { + "${TPCH_DBGEN_DIR}"/dbgen -f -s "${SCALE_FACTOR}" -T L -C "${PARALLEL}" -S "${i}" + } & done wait cd - -# move data to $TPCH_DATA_DIR -mv "$TPCH_DBGEN_DIR"/*.tbl* "$TPCH_DATA_DIR"/ +# move data to ${TPCH_DATA_DIR} +mv "${TPCH_DBGEN_DIR}"/*.tbl* "${TPCH_DATA_DIR}"/ # check data -du -sh "$TPCH_DATA_DIR"/*.tbl* +du -sh "${TPCH_DATA_DIR}"/*.tbl* diff --git a/tools/tpch-tools/bin/load-tpch-data.sh b/tools/tpch-tools/bin/load-tpch-data.sh index 7a250ef2f4..2a92ab4d5c 100755 --- a/tools/tpch-tools/bin/load-tpch-data.sh +++ b/tools/tpch-tools/bin/load-tpch-data.sh @@ -25,12 +25,12 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" + cd "${ROOT}" pwd ) -CURDIR=${ROOT} -TPCH_DATA_DIR=$CURDIR/tpch-data +CURDIR="${ROOT}" +TPCH_DATA_DIR="${CURDIR}/tpch-data" usage() { echo " @@ -51,12 +51,12 @@ OPTS=$(getopt \ -o 'hc:' \ -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" PARALLEL=5 HELP=0 -if [ $# == 0 ]; then +if [[ $# == 0 ]]; then usage fi @@ -86,19 +86,19 @@ if [[ ${HELP} -eq 1 ]]; then exit fi -echo "Parallelism: $PARALLEL" +echo "Parallelism: ${PARALLEL}" # check if tpch-data exists -if [[ ! -d "$TPCH_DATA_DIR"/ ]]; then - echo "$TPCH_DATA_DIR does not exist. Run sh gen-tpch-data.sh first." +if [[ ! -d "${TPCH_DATA_DIR}"/ ]]; then + echo "${TPCH_DATA_DIR} does not exist. Run sh gen-tpch-data.sh first." exit 1 fi check_prerequest() { local CMD=$1 local NAME=$2 - if ! $CMD; then - echo "$NAME is missing. This script depends on cURL to load data to Doris." + if ! ${CMD}; then + echo "${NAME} is missing. This script depends on cURL to load data to Doris." exit 1 fi } @@ -106,83 +106,82 @@ check_prerequest() { check_prerequest "curl --version" "curl" # load tables -# shellcheck source=/dev/null -source "$CURDIR/../conf/doris-cluster.conf" -export MYSQL_PWD=$PASSWORD +source "${CURDIR}/../conf/doris-cluster.conf" +export MYSQL_PWD=${PASSWORD} -echo "FE_HOST: $FE_HOST" -echo "FE_HTTP_PORT: $FE_HTTP_PORT" -echo "USER: $USER" -echo "PASSWORD: $PASSWORD" -echo "DB: $DB" +echo "FE_HOST: ${FE_HOST}" +echo "FE_HTTP_PORT: ${FE_HTTP_PORT}" +echo "USER: ${USER}" +echo "PASSWORD: ${PASSWORD}" +echo "DB: ${DB}" function load_region() { echo "$*" - curl --location-trusted -u "$USER":"$PASSWORD" -H "column_separator:|" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" -H "column_separator:|" \ -H "columns: r_regionkey, r_name, r_comment, temp" \ - -T "$*" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/region/_stream_load + -T "$*" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/region/_stream_load } function load_nation() { echo "$*" - curl --location-trusted -u "$USER":"$PASSWORD" -H "column_separator:|" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" -H "column_separator:|" \ -H "columns: n_nationkey, n_name, n_regionkey, n_comment, temp" \ - -T "$*" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/nation/_stream_load + -T "$*" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/nation/_stream_load } function load_supplier() { echo "$*" - curl --location-trusted -u "$USER":"$PASSWORD" -H "column_separator:|" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" -H "column_separator:|" \ -H "columns: s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment, temp" \ - -T "$*" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/supplier/_stream_load + -T "$*" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/supplier/_stream_load } function load_customer() { echo "$*" - curl --location-trusted -u "$USER":"$PASSWORD" -H "column_separator:|" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" -H "column_separator:|" \ -H "columns: c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_mktsegment, c_comment, temp" \ - -T "$*" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/customer/_stream_load + -T "$*" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/customer/_stream_load } function load_part() { echo "$*" - curl --location-trusted -u "$USER":"$PASSWORD" -H "column_separator:|" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" -H "column_separator:|" \ -H "columns: p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, temp" \ - -T "$*" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/part/_stream_load + -T "$*" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/part/_stream_load } function load_partsupp() { echo "$*" - curl --location-trusted -u "$USER":"$PASSWORD" -H "column_separator:|" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" -H "column_separator:|" \ -H "columns: ps_partkey, ps_suppkey, ps_availqty, ps_supplycost, ps_comment, temp" \ - -T "$*" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/partsupp/_stream_load + -T "$*" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/partsupp/_stream_load } function load_orders() { echo "$*" - curl --location-trusted -u "$USER":"$PASSWORD" -H "column_separator:|" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" -H "column_separator:|" \ -H "columns: o_orderkey, o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, temp" \ - -T "$*" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/orders/_stream_load + -T "$*" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/orders/_stream_load } function load_lineitem() { echo "$*" - curl --location-trusted -u "$USER":"$PASSWORD" -H "column_separator:|" \ + curl --location-trusted -u "${USER}":"${PASSWORD}" -H "column_separator:|" \ -H "columns: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag,l_linestatus, l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment,temp" \ - -T "$*" http://"$FE_HOST":"$FE_HTTP_PORT"/api/"$DB"/lineitem/_stream_load + -T "$*" http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/lineitem/_stream_load } # start load date -load_region "$TPCH_DATA_DIR"/region.tbl -load_nation "$TPCH_DATA_DIR"/nation.tbl -load_supplier "$TPCH_DATA_DIR"/supplier.tbl -load_customer "$TPCH_DATA_DIR"/customer.tbl -load_part "$TPCH_DATA_DIR"/part.tbl +load_region "${TPCH_DATA_DIR}"/region.tbl +load_nation "${TPCH_DATA_DIR}"/nation.tbl +load_supplier "${TPCH_DATA_DIR}"/supplier.tbl +load_customer "${TPCH_DATA_DIR}"/customer.tbl +load_part "${TPCH_DATA_DIR}"/part.tbl date # set parallelism # 以PID为名, 防止创建命名管道时与已有文件重名,从而失败 fifo="/tmp/$$.fifo" # 创建命名管道 -mkfifo ${fifo} +mkfifo "${fifo}" # 以读写方式打开命名管道,文件标识符fd为3,fd可取除0,1,2,5外0-9中的任意数字 -exec 3<>${fifo} +exec 3<>"${fifo}" # 删除文件, 也可不删除, 不影响后面操作 -rm -rf ${fifo} +rm -rf "${fifo}" # 在fd3中放置$PARALLEL个空行作为令牌 for ((i = 1; i <= PARALLEL; i++)); do @@ -190,7 +189,7 @@ for ((i = 1; i <= PARALLEL; i++)); do done date -for file in "$TPCH_DATA_DIR"/lineitem.tbl*; do +for file in "${TPCH_DATA_DIR}"/lineitem.tbl*; do # 领取令牌, 即从fd3中读取行, 每次一行 # 对管道,读一行便少一行,每次只能读取一行 # 所有行读取完毕, 执行挂起, 直到管道再次有可读行 @@ -199,8 +198,8 @@ for file in "$TPCH_DATA_DIR"/lineitem.tbl*; do # 要批量执行的命令放在大括号内, 后台运行 { - load_lineitem "$file" - echo "----loaded $file" + load_lineitem "${file}" + echo "----loaded ${file}" sleep 2 # 归还令牌, 即进程结束后,再写入一行,使挂起的循环继续执行 echo >&3 @@ -208,22 +207,22 @@ for file in "$TPCH_DATA_DIR"/lineitem.tbl*; do done date -for file in "$TPCH_DATA_DIR"/orders.tbl*; do +for file in "${TPCH_DATA_DIR}"/orders.tbl*; do read -r -u3 { - load_orders "$file" - echo "----loaded $file" + load_orders "${file}" + echo "----loaded ${file}" sleep 2 echo >&3 } & done date -for file in "$TPCH_DATA_DIR"/partsupp.tbl*; do +for file in "${TPCH_DATA_DIR}"/partsupp.tbl*; do read -r -u3 { - load_partsupp "$file" - echo "----loaded $file" + load_partsupp "${file}" + echo "----loaded ${file}" sleep 2 echo >&3 } & @@ -235,4 +234,4 @@ wait exec 3>&- date -echo "DONE." \ No newline at end of file +echo "DONE." diff --git a/tools/tpch-tools/bin/run-tpch-queries.sh b/tools/tpch-tools/bin/run-tpch-queries.sh index 14d00e17bb..6a05a47010 100755 --- a/tools/tpch-tools/bin/run-tpch-queries.sh +++ b/tools/tpch-tools/bin/run-tpch-queries.sh @@ -24,12 +24,12 @@ set -eo pipefail ROOT=$(dirname "$0") ROOT=$( - cd "$ROOT" + cd "${ROOT}" pwd ) -CURDIR=${ROOT} -QUERIES_DIR=$CURDIR/../queries +CURDIR="${ROOT}" +QUERIES_DIR="${CURDIR}/../queries" usage() { echo " @@ -45,10 +45,10 @@ OPTS=$(getopt \ -o '' \ -- "$@") -eval set -- "$OPTS" +eval set -- "${OPTS}" HELP=0 -if [ $# == 0 ]; then +if [[ $# == 0 ]]; then usage fi @@ -77,28 +77,27 @@ fi check_prerequest() { local CMD=$1 local NAME=$2 - if ! $CMD; then - echo "$NAME is missing. This script depends on mysql to create tables in Doris." + if ! ${CMD}; then + echo "${NAME} is missing. This script depends on mysql to create tables in Doris." exit 1 fi } check_prerequest "mysql --version" "mysql" -# shellcheck source=/dev/null -source "$CURDIR/../conf/doris-cluster.conf" -export MYSQL_PWD=$PASSWORD +source "${CURDIR}/../conf/doris-cluster.conf" +export MYSQL_PWD=${PASSWORD} -echo "FE_HOST: $FE_HOST" -echo "FE_QUERY_PORT: $FE_QUERY_PORT" -echo "USER: $USER" -echo "PASSWORD: $PASSWORD" -echo "DB: $DB" +echo "FE_HOST: ${FE_HOST}" +echo "FE_QUERY_PORT: ${FE_QUERY_PORT}" +echo "USER: ${USER}" +echo "PASSWORD: ${PASSWORD}" +echo "DB: ${DB}" echo "Time Unit: ms" pre_set() { echo "$*" - mysql -h"$FE_HOST" -u"$USER" -P"$FE_QUERY_PORT" -D"$DB" -e "$*" + mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" -e "$*" } echo '============================================' @@ -112,14 +111,14 @@ for i in $(seq 1 22); do total=0 run=3 # Each query is executed ${run} times and takes the average time - for j in $(seq 1 ${run}); do + for ((j = 0; j < run; j++)); do start=$(date +%s%3N) - mysql -h"$FE_HOST" -u "$USER" -P"$FE_QUERY_PORT" -D"$DB" --comments <"$QUERIES_DIR"/q"$i".sql >/dev/null + mysql -h"${FE_HOST}" -u "${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" --comments <"${QUERIES_DIR}/q${i}.sql" >/dev/null end=$(date +%s%3N) total=$((total + end - start)) done cost=$((total / run)) - echo "q$i: ${cost}" + echo "q${i}: ${cost}" sum=$((sum + cost)) done -echo "Total cost: $sum" +echo "Total cost: ${sum}"