[ci](perf) add required (#30155)
Co-authored-by: stephen <hello-stephen@qq.com>
This commit is contained in:
Dongyang Li
@ -57,6 +57,7 @@ github:
|
||||
- External Regression (Doris External Regression)
|
||||
- FE UT (Doris FE UT)
|
||||
- BE UT (Doris BE UT)
|
||||
- performance (Performance New)
|
||||
- Build Broker
|
||||
- Build Documents
|
||||
- ShellCheck
|
||||
@ -92,6 +93,7 @@ github:
|
||||
- External Regression (Doris External Regression)
|
||||
- FE UT (Doris FE UT)
|
||||
- BE UT (Doris BE UT)
|
||||
- performance (Performance New)
|
||||
- Build Broker
|
||||
- ShellCheck
|
||||
- Build Documents
|
||||
|
||||
@ -45,7 +45,6 @@ jobs:
|
||||
"${COMMENT_BODY}" == *'run p1'* ||
|
||||
"${COMMENT_BODY}" == *'run external'* ||
|
||||
"${COMMENT_BODY}" == *'run pipelinex_p0'* ||
|
||||
"${COMMENT_BODY}" == *'run clickbench'* ||
|
||||
"${COMMENT_BODY}" == *'run arm'* ||
|
||||
"${COMMENT_BODY}" == *'run performance'* ]]; then
|
||||
echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
|
||||
@ -63,7 +62,7 @@ jobs:
|
||||
echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
|
||||
echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT"
|
||||
|
||||
reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|performance)( [1-9]*[0-9]+)*"
|
||||
reg="run (buildall|compile|p0|p1|feut|beut|external|pipelinex_p0|arm|performance)( [1-9]*[0-9]+)*"
|
||||
COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
|
||||
COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
|
||||
echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT"
|
||||
@ -90,11 +89,6 @@ jobs:
|
||||
else
|
||||
echo "changed_be_ut=false" | tee -a "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if file_changed_ckb; then
|
||||
echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "changed_ckb=false" | tee -a "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if file_changed_regression_p0; then
|
||||
echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
|
||||
echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
|
||||
@ -120,7 +114,6 @@ jobs:
|
||||
echo "INFO: failed to _get_pr_changed_files, default trigger all"
|
||||
echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT"
|
||||
echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT"
|
||||
echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT"
|
||||
echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
|
||||
echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
|
||||
echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT"
|
||||
@ -243,18 +236,6 @@ jobs:
|
||||
"arm" \
|
||||
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
|
||||
|
||||
- name: "Trigger or Skip clickbench"
|
||||
if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
|
||||
run: |
|
||||
source ./regression-test/pipeline/common/teamcity-utils.sh
|
||||
set -x
|
||||
trigger_or_skip_build \
|
||||
"${{ steps.changes.outputs.changed_ckb }}" \
|
||||
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
|
||||
"${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
|
||||
"clickbench" \
|
||||
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
|
||||
|
||||
- name: "Trigger or Skip performance"
|
||||
if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["performance", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
|
||||
run: |
|
||||
|
||||
@ -394,18 +394,18 @@ archive_doris_logs() {
|
||||
|
||||
print_doris_fe_log() {
|
||||
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
|
||||
echo "WARNING: --------------------tail -n 100 ${DORIS_HOME}/fe/log/fe.out--------------------"
|
||||
echo -e "\n\n\n\nWARNING: --------------------tail -n 100 ${DORIS_HOME}/fe/log/fe.out--------------------"
|
||||
tail -n 100 "${DORIS_HOME}"/fe/log/fe.out
|
||||
echo "WARNING: --------------------tail -n 100 ${DORIS_HOME}/fe/log/fe.log--------------------"
|
||||
echo -e "\n\n\n\nWARNING: --------------------tail -n 100 ${DORIS_HOME}/fe/log/fe.log--------------------"
|
||||
tail -n 100 "${DORIS_HOME}"/fe/log/fe.log
|
||||
echo "WARNING: ----------------------------------------"
|
||||
echo -e "WARNING: ----------------------------------------\n\n\n\n"
|
||||
}
|
||||
|
||||
print_doris_be_log() {
|
||||
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
|
||||
echo "WARNING: --------------------tail -n 100 ${DORIS_HOME}/be/log/be.out--------------------"
|
||||
echo -e "\n\n\n\nWARNING: --------------------tail -n 100 ${DORIS_HOME}/be/log/be.out--------------------"
|
||||
tail -n 100 "${DORIS_HOME}"/be/log/be.out
|
||||
echo "WARNING: --------------------tail -n 100 ${DORIS_HOME}/be/log/be.INFO--------------------"
|
||||
echo -e "\n\n\n\nWARNING: --------------------tail -n 100 ${DORIS_HOME}/be/log/be.INFO--------------------"
|
||||
tail -n 100 "${DORIS_HOME}"/be/log/be.INFO
|
||||
echo "WARNING: ----------------------------------------"
|
||||
echo -e "WARNING: ----------------------------------------\n\n\n\n"
|
||||
}
|
||||
|
||||
@ -304,31 +304,6 @@ file_changed_regression_p1() {
|
||||
file_changed_regression_p0
|
||||
}
|
||||
|
||||
file_changed_ckb() {
|
||||
local all_files
|
||||
all_files=$(cat all_files)
|
||||
if _only_modified_regression_conf; then echo "return no need" && return 1; fi
|
||||
if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
|
||||
for af in ${all_files}; do
|
||||
if [[ "${af}" == 'be'* ]] ||
|
||||
[[ "${af}" == 'bin'* ]] ||
|
||||
[[ "${af}" == 'conf'* ]] ||
|
||||
[[ "${af}" == 'fe'* ]] ||
|
||||
[[ "${af}" == 'gensrc'* ]] ||
|
||||
[[ "${af}" == 'thirdparty'* ]] ||
|
||||
[[ "${af}" == 'build.sh' ]] ||
|
||||
[[ "${af}" == 'env.sh' ]] ||
|
||||
[[ "${af}" == 'regression-test/pipeline/common/github-utils.sh' ]] ||
|
||||
[[ "${af}" == 'regression-test/pipeline/common/doris-utils.sh' ]] ||
|
||||
[[ "${af}" == 'regression-test/pipeline/common/oss-utils.sh' ]] ||
|
||||
[[ "${af}" == 'tools/tpch-tools/bin/run-tpch-queries.sh' ]] ||
|
||||
[[ "${af}" == 'regression-test/pipeline/tpch/tpch-sf100/'* ]]; then
|
||||
echo "clickbench performance related file changed, return need" && return 0
|
||||
fi
|
||||
done
|
||||
echo "return no need" && return 1
|
||||
}
|
||||
|
||||
file_changed_performance() {
|
||||
local all_files
|
||||
all_files=$(cat all_files)
|
||||
|
||||
@ -125,6 +125,7 @@ exit_flag=0
|
||||
echo "#### 3. run tpcds-sf${SF} query"
|
||||
set_session_variable runtime_filter_mode global
|
||||
bash "${teamcity_build_checkoutDir}"/tools/tpcds-tools/bin/run-tpcds-queries.sh -s "${SF}" | tee "${teamcity_build_checkoutDir}"/run-tpcds-queries.log
|
||||
echo
|
||||
cold_run_time_threshold=${cold_run_time_threshold_master:-315000} # ms
|
||||
hot_run_time_threshold=${hot_run_time_threshold_master:-190000} # ms
|
||||
if [[ "${target_branch}" == "branch-2.0" ]]; then
|
||||
|
||||
@ -117,6 +117,7 @@ exit_flag=0
|
||||
echo "#### 3. run tpch-sf${SF} query"
|
||||
set_session_variable runtime_filter_mode global
|
||||
bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh -s "${SF}" | tee "${teamcity_build_checkoutDir}"/run-tpch-queries.log
|
||||
echo
|
||||
cold_run_time_threshold=${cold_run_time_threshold_master:-120000} # ms
|
||||
hot_run_time_threshold=${hot_run_time_threshold_master:-42000} # ms
|
||||
if [[ "${target_branch}" == "branch-2.0" ]]; then
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Build Step: Command Line
|
||||
: <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
# Execute step even if some of the previous steps failed
|
||||
teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
|
||||
if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/tpch/tpch-sf100/clean.sh ]]; then
|
||||
cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/
|
||||
bash -x clean.sh
|
||||
else
|
||||
echo "Build Step file missing: regression-test/pipeline/tpch/tpch-sf100/clean.sh" && exit 1
|
||||
fi
|
||||
EOF
|
||||
|
||||
## clean.sh content ##
|
||||
|
||||
# stop_doris
|
||||
source ../../common/doris-utils.sh
|
||||
|
||||
DORIS_HOME="${teamcity_build_checkoutDir}/output"
|
||||
export DORIS_HOME
|
||||
stop_doris
|
||||
@ -1,85 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
CUR_DATE=`date +%Y%m%d-%H%M%S`
|
||||
|
||||
PPROF_TMPDIR="$DORIS_HOME/log/"
|
||||
|
||||
JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
|
||||
|
||||
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
|
||||
JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
|
||||
|
||||
# since 1.2, the JAVA_HOME need to be set to run BE process.
|
||||
# JAVA_HOME=/path/to/jdk/
|
||||
|
||||
# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
|
||||
# https://jemalloc.net/jemalloc.3.html
|
||||
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
|
||||
JEMALLOC_PROF_PRFIX=""
|
||||
|
||||
# INFO, WARNING, ERROR, FATAL
|
||||
sys_log_level = INFO
|
||||
|
||||
# ports for admin, web, heartbeat service
|
||||
be_port = 9060
|
||||
webserver_port = 8040
|
||||
heartbeat_service_port = 9050
|
||||
brpc_port = 8060
|
||||
arrow_flight_sql_port = -1
|
||||
|
||||
# HTTPS configures
|
||||
enable_https = false
|
||||
# path of certificate in PEM format.
|
||||
ssl_certificate_path = "$DORIS_HOME/conf/cert.pem"
|
||||
# path of private key in PEM format.
|
||||
ssl_private_key_path = "$DORIS_HOME/conf/key.pem"
|
||||
|
||||
|
||||
# Choose one if there are more than one ip except loopback address.
|
||||
# Note that there should at most one ip match this list.
|
||||
# If no ip match this rule, will choose one randomly.
|
||||
# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1
|
||||
# Default value is empty.
|
||||
# priority_networks = 10.10.10.0/24;192.168.0.0/16
|
||||
|
||||
# data root path, separate by ';'
|
||||
# You can specify the storage type for each root path, HDD (cold data) or SSD (hot data)
|
||||
# eg:
|
||||
# storage_root_path = /home/disk1/doris;/home/disk2/doris;/home/disk2/doris
|
||||
# storage_root_path = /home/disk1/doris,medium:SSD;/home/disk2/doris,medium:SSD;/home/disk2/doris,medium:HDD
|
||||
# /home/disk2/doris,medium:HDD(default)
|
||||
#
|
||||
# you also can specify the properties by setting '<property>:<value>', separate by ','
|
||||
# property 'medium' has a higher priority than the extension of path
|
||||
#
|
||||
# Default value is ${DORIS_HOME}/storage, you should create it by hand.
|
||||
# storage_root_path = ${DORIS_HOME}/storage
|
||||
|
||||
# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers
|
||||
# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
|
||||
|
||||
# Advanced configurations
|
||||
# sys_log_dir = ${DORIS_HOME}/log
|
||||
# sys_log_roll_mode = SIZE-MB-1024
|
||||
# sys_log_roll_num = 10
|
||||
# sys_log_verbose_modules = *
|
||||
# log_buffer_level = -1
|
||||
# palo_cgroups
|
||||
|
||||
priority_networks=172.16.0.0/24
|
||||
storage_root_path=/mnt/datadisk0/doris-storage
|
||||
@ -1,26 +0,0 @@
|
||||
[
|
||||
{
|
||||
"file": "docker/thirdparties/docker-compose/mysql/mysql-5.7.env",
|
||||
"replacements": {
|
||||
"DOCKER_MYSQL_57_EXTERNAL_PORT": 7111
|
||||
}
|
||||
},
|
||||
{
|
||||
"file": "docker/thirdparties/docker-compose/postgresql/postgresql-14.env",
|
||||
"replacements": {
|
||||
"DOCKER_PG_14_EXTERNAL_PORT": 7121
|
||||
}
|
||||
},
|
||||
{
|
||||
"file": "docker/thirdparties/docker-compose/hive/gen_env.sh",
|
||||
"replacements": {
|
||||
"FS_PORT": 7131,
|
||||
"HMS_PORT": 7141
|
||||
}
|
||||
}, {
|
||||
"file": "docker/thirdparties/start-thirdparties-docker.sh",
|
||||
"replacements": {
|
||||
"CONTAINER_UID": "doris-regression-fakeid-fakecommit"
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -1,78 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
#####################################################################
|
||||
## The uppercase properties are read and exported by bin/start_fe.sh.
|
||||
## To see all Frontend configurations,
|
||||
## see fe/src/org/apache/doris/common/Config.java
|
||||
#####################################################################
|
||||
|
||||
CUR_DATE=`date +%Y%m%d-%H%M%S`
|
||||
|
||||
# the output dir of stderr and stdout
|
||||
LOG_DIR = ${DORIS_HOME}/log
|
||||
|
||||
JAVA_OPTS="-Dsun.security.krb5.debug=true -Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE"
|
||||
|
||||
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
|
||||
JAVA_OPTS_FOR_JDK_9="-Dsun.security.krb5.debug=true -Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time"
|
||||
|
||||
##
|
||||
## the lowercase properties are read by main program.
|
||||
##
|
||||
|
||||
# INFO, WARN, ERROR, FATAL
|
||||
sys_log_level = INFO
|
||||
|
||||
# NORMAL, BRIEF, ASYNC
|
||||
sys_log_mode = NORMAL
|
||||
|
||||
# store metadata, must be created before start FE.
|
||||
# Default value is ${DORIS_HOME}/doris-meta
|
||||
# meta_dir = ${DORIS_HOME}/doris-meta
|
||||
|
||||
# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers
|
||||
# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
|
||||
|
||||
http_port = 8030
|
||||
rpc_port = 9020
|
||||
query_port = 9030
|
||||
edit_log_port = 9010
|
||||
arrow_flight_sql_port = -1
|
||||
|
||||
# Choose one if there are more than one ip except loopback address.
|
||||
# Note that there should at most one ip match this list.
|
||||
# If no ip match this rule, will choose one randomly.
|
||||
# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1
|
||||
# Default value is empty.
|
||||
# priority_networks = 10.10.10.0/24;192.168.0.0/16
|
||||
|
||||
# Advanced configurations
|
||||
# log_roll_size_mb = 1024
|
||||
# sys_log_dir = ${DORIS_HOME}/log
|
||||
# sys_log_roll_num = 10
|
||||
# sys_log_verbose_modules = org.apache.doris
|
||||
# audit_log_dir = ${DORIS_HOME}/log
|
||||
# audit_log_modules = slow_query, query
|
||||
# audit_log_roll_num = 10
|
||||
# meta_delay_toleration_second = 10
|
||||
# qe_max_connection = 1024
|
||||
# qe_query_timeout_second = 300
|
||||
# qe_slow_log_ms = 5000
|
||||
|
||||
priority_networks=172.16.0.0/24
|
||||
meta_dir=/mnt/datadisk0/doris-meta
|
||||
@ -1,43 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Example driver definitions, you should not use the example odbc driver
|
||||
# before you prepare env in your server
|
||||
|
||||
# Driver from the postgresql-odbc package
|
||||
# Setup from the unixODBC package
|
||||
[PostgreSQL]
|
||||
Description = ODBC for PostgreSQL
|
||||
Driver = /usr/lib/psqlodbc.so
|
||||
Setup = /usr/lib/libodbcpsqlS.so
|
||||
FileUsage = 1
|
||||
|
||||
|
||||
# Driver from the mysql-connector-odbc package
|
||||
# Setup from the unixODBC package
|
||||
[MySQL ODBC 8.0 Unicode Driver]
|
||||
Description = ODBC for MySQL
|
||||
Driver = /usr/lib64/libmyodbc8w.so
|
||||
FileUsage = 1
|
||||
|
||||
# Driver from the oracle-connector-odbc package
|
||||
# Setup from the unixODBC package
|
||||
[Oracle 19 ODBC driver]
|
||||
Description=Oracle ODBC driver for Oracle 19
|
||||
Driver=/usr/lib/libsqora.so.19.1
|
||||
|
||||
|
||||
@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Build Step: Command Line
|
||||
: <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
|
||||
if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/tpch/tpch-sf100/deploy.sh ]]; then
|
||||
cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/
|
||||
bash -x deploy.sh
|
||||
else
|
||||
echo "Build Step file missing: regression-test/pipeline/tpch/tpch-sf100/deploy.sh" && exit 1
|
||||
fi
|
||||
EOF
|
||||
|
||||
## deploy.sh content ##
|
||||
|
||||
# download_oss_file
|
||||
source ../../common/oss-utils.sh
|
||||
# start_doris_fe, get_doris_conf_value, start_doris_be, stop_doris,
|
||||
# print_doris_fe_log, print_doris_be_log, archive_doris_logs
|
||||
source ../../common/doris-utils.sh
|
||||
|
||||
echo "#### Check env"
|
||||
if [[ -z "${teamcity_build_checkoutDir}" ||
|
||||
-z "${pull_request_id}" ||
|
||||
-z "${commit_id}" ]]; then
|
||||
echo "ERROR: env teamcity_build_checkoutDir or pull_request_id or commit_id not set"
|
||||
exit 1
|
||||
fi
|
||||
if ${DEBUG:-false}; then
|
||||
pull_request_id="26465"
|
||||
commit_id="a532f7113f463e144e83918a37288f2649448482"
|
||||
fi
|
||||
|
||||
echo "#### Deploy Doris ####"
|
||||
DORIS_HOME="${teamcity_build_checkoutDir}/output"
|
||||
export DORIS_HOME
|
||||
exit_flag=0
|
||||
need_backup_doris_logs=false
|
||||
|
||||
echo "#### 1. try to kill old doris process and remove old doris binary"
|
||||
stop_doris && rm -rf output
|
||||
|
||||
echo "#### 2. download doris binary tar ball"
|
||||
cd "${teamcity_build_checkoutDir}" || exit 1
|
||||
if download_oss_file "${pull_request_id:-}_${commit_id:-}.tar.gz"; then
|
||||
if ! command -v pigz >/dev/null; then sudo apt install -y pigz; fi
|
||||
tar -I pigz -xf "${pull_request_id:-}_${commit_id:-}.tar.gz"
|
||||
if [[ -d output && -d output/fe && -d output/be ]]; then
|
||||
echo "INFO: be version: $(./output/be/lib/doris_be --version)"
|
||||
rm -rf "${pull_request_id}_${commit_id}.tar.gz"
|
||||
fi
|
||||
else
|
||||
echo "ERROR: download compiled binary failed" && exit 1
|
||||
fi
|
||||
|
||||
echo "#### 3. copy conf from regression-test/pipeline/tpch/tpch-sf100/conf/"
|
||||
rm -f "${DORIS_HOME}"/fe/conf/fe_custom.conf "${DORIS_HOME}"/be/conf/be_custom.conf
|
||||
if [[ -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf &&
|
||||
-f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf ]]; then
|
||||
cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf "${DORIS_HOME}"/fe/conf/
|
||||
cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf "${DORIS_HOME}"/be/conf/
|
||||
else
|
||||
echo "ERROR: doris conf file missing in ${teamcity_build_checkoutDir}/regression-test/pipeline/tpch/tpch-sf100/conf/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "#### 4. start Doris"
|
||||
meta_dir=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf meta_dir)
|
||||
storage_root_path=$(get_doris_conf_value "${DORIS_HOME}"/be/conf/be.conf storage_root_path)
|
||||
mkdir -p "${meta_dir}"
|
||||
mkdir -p "${storage_root_path}"
|
||||
if ! start_doris_fe; then
|
||||
echo "ERROR: Start doris fe failed."
|
||||
print_doris_fe_log
|
||||
need_backup_doris_logs=true
|
||||
exit_flag=1
|
||||
fi
|
||||
if ! start_doris_be; then
|
||||
echo "ERROR: Start doris be failed."
|
||||
print_doris_be_log
|
||||
need_backup_doris_logs=true
|
||||
exit_flag=1
|
||||
fi
|
||||
|
||||
# wait 10s for doris totally started, otherwize may encounter the error below,
|
||||
# ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find enough backend, please check the replication num,replication tag and storage medium.
|
||||
sleep 10s
|
||||
|
||||
echo "#### 5. set session variables"
|
||||
echo "TODO"
|
||||
|
||||
echo "#### 6. check if need backup doris logs"
|
||||
if ${need_backup_doris_logs}; then
|
||||
print_doris_fe_log
|
||||
print_doris_be_log
|
||||
if file_name=$(archive_doris_logs "${pull_request_id}_${commit_id}_doris_logs.tar.gz"); then
|
||||
upload_doris_log_to_oss "${file_name}"
|
||||
fi
|
||||
fi
|
||||
|
||||
exit "${exit_flag}"
|
||||
@ -1,69 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Build Step: Command Line
|
||||
: <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
|
||||
if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/tpch/tpch-sf100/prepare.sh ]]; then
|
||||
cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/
|
||||
bash -x prepare.sh
|
||||
else
|
||||
echo "Build Step file missing: regression-test/pipeline/tpch/tpch-sf100/prepare.sh" && exit 1
|
||||
fi
|
||||
EOF
|
||||
|
||||
## run.sh content ##
|
||||
|
||||
echo "#### Check env"
|
||||
if [[ -z "${commit_id_from_trigger}" || -z ${commit_id:-} || -z ${pull_request_id:-} ]]; then
|
||||
echo "ERROR: env commit_id_from_trigger or commit_id or pull_request_id not set" && exit 1
|
||||
else
|
||||
commit_id_from_checkout=${commit_id}
|
||||
fi
|
||||
if ${DEBUG:-false}; then commit_id_from_trigger=${commit_id}; fi
|
||||
|
||||
echo "Prepare to run tpch sf100 test"
|
||||
|
||||
echo "#### 1. check if need run"
|
||||
if [[ "${commit_id_from_trigger}" != "${commit_id_from_checkout}" ]]; then
|
||||
echo -e "目前是在 clickbench 流水线 compile 完后触发本 tpch 流水线的，
|
||||
有可能 pr 在 clickbench 流水线还在跑的时候新提交了commit，
|
||||
这时候 tpch 流水线 checkout 出来的 commit 就不是触发时的传过来的 commit了，
|
||||
这种情况不需要跑"
|
||||
echo -e "ERROR: PR(${pull_request_id}),
|
||||
the lastest commit id
|
||||
${commit_id_from_checkout}
|
||||
not equail to the commit_id_from_trigger
|
||||
${commit_id_from_trigger}
|
||||
commit_id_from_trigger is outdate"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "#### 2. check if depending files exist"
|
||||
if ! [[ -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf &&
|
||||
-f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf &&
|
||||
-f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/deploy.sh &&
|
||||
-f "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/run.sh &&
|
||||
-f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh &&
|
||||
-f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh &&
|
||||
-f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh &&
|
||||
-f "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh ]]; then
|
||||
echo "ERROR: depending files missing" && exit 1
|
||||
fi
|
||||
@ -1,131 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Build Step: Command Line
|
||||
: <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
|
||||
if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/tpch/tpch-sf100/run.sh ]]; then
|
||||
cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/
|
||||
bash -x run.sh
|
||||
else
|
||||
echo "Build Step file missing: regression-test/pipeline/tpch/tpch-sf100/run.sh" && exit 1
|
||||
fi
|
||||
EOF
|
||||
|
||||
## run.sh content ##
|
||||
|
||||
# check_tpch_table_rows, stop_doris, set_session_variable
|
||||
source ../../common/doris-utils.sh
|
||||
# create_an_issue_comment
|
||||
source ../../common/github-utils.sh
|
||||
# upload_doris_log_to_oss
|
||||
source ../../common/oss-utils.sh
|
||||
|
||||
echo "#### Check env"
|
||||
if [[ -z "${teamcity_build_checkoutDir}" ||
|
||||
-z "${pull_request_id}" ||
|
||||
-z "${commit_id}" ]]; then
|
||||
echo "ERROR: env teamcity_build_checkoutDir or pull_request_id or commit_id not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "#### Run tpch-sf100 test on Doris ####"
|
||||
DORIS_HOME="${teamcity_build_checkoutDir}/output"
|
||||
exit_flag=0
|
||||
|
||||
check_tpch_result() {
|
||||
log_file="$1"
|
||||
if ! grep '^Total cold run time' "${log_file}" || ! grep '^Total hot run time' "${log_file}"; then
|
||||
echo "ERROR: can not find 'Total hot run time' in '${log_file}'"
|
||||
return 1
|
||||
else
|
||||
cold_run_time=$(grep '^Total cold run time' "${log_file}" | awk '{print $5}')
|
||||
hot_run_time=$(grep '^Total hot run time' "${log_file}" | awk '{print $5}')
|
||||
fi
|
||||
# 单位是毫秒
|
||||
cold_run_time_threshold=${cold_run_time_threshold:-50000}
|
||||
hot_run_time_threshold=${hot_run_time_threshold:-42000}
|
||||
if [[ ${cold_run_time} -gt 50000 || ${hot_run_time} -gt 42000 ]]; then
|
||||
echo "ERROR:
|
||||
cold_run_time ${cold_run_time} is great than the threshold ${cold_run_time_threshold},
|
||||
or, hot_run_time ${hot_run_time} is great than the threshold ${hot_run_time_threshold}"
|
||||
return 1
|
||||
else
|
||||
echo "INFO:
|
||||
cold_run_time ${cold_run_time} is less than the threshold ${cold_run_time_threshold},
|
||||
or, hot_run_time ${hot_run_time} is less than the threshold ${hot_run_time_threshold}"
|
||||
fi
|
||||
}
|
||||
|
||||
(
|
||||
set -e
|
||||
shopt -s inherit_errexit
|
||||
|
||||
echo "#### 1. check if need to load data"
|
||||
SF="100" # SCALE FACTOR
|
||||
if ${DEBUG:-false}; then SF="1"; fi
|
||||
db_name="tpch_sf${SF}"
|
||||
sed -i "s|^export DB=.*$|export DB='${db_name}'|g" \
|
||||
"${teamcity_build_checkoutDir}"/tools/tpch-tools/conf/doris-cluster.conf
|
||||
if ! check_tpch_table_rows "${db_name}" "${SF}"; then
|
||||
echo "ERROR: check_tpch_table_rows failed." && exit 1
|
||||
fi
|
||||
|
||||
echo "#### 2. run tpch-sf${SF} query"
|
||||
set_session_variable runtime_filter_mode global
|
||||
sed -i "s|^SCALE_FACTOR=[0-9]\+$|SCALE_FACTOR=${SF}|g" "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh
|
||||
bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh | tee "${teamcity_build_checkoutDir}"/run-tpch-queries.log
|
||||
if ! check_tpch_result "${teamcity_build_checkoutDir}"/run-tpch-queries.log; then exit 1; fi
|
||||
line_end=$(sed -n '/^Total hot run time/=' "${teamcity_build_checkoutDir}"/run-tpch-queries.log)
|
||||
line_begin=$((line_end - 23))
|
||||
comment_body="Tpch sf${SF} test result on commit ${commit_id:-}, data reload: ${data_reload:-"false"}
|
||||
|
||||
run tpch-sf${SF} query with default conf and session variables
|
||||
$(sed -n "${line_begin},${line_end}p" "${teamcity_build_checkoutDir}"/run-tpch-queries.log)"
|
||||
|
||||
echo "#### 3. run tpch-sf${SF} query with runtime_filter_mode=off"
|
||||
set_session_variable runtime_filter_mode off
|
||||
bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh | tee "${teamcity_build_checkoutDir}"/run-tpch-queries.log
|
||||
if ! grep '^Total hot run time' "${teamcity_build_checkoutDir}"/run-tpch-queries.log >/dev/null; then exit 1; fi
|
||||
line_end=$(sed -n '/^Total hot run time/=' "${teamcity_build_checkoutDir}"/run-tpch-queries.log)
|
||||
line_begin=$((line_end - 23))
|
||||
comment_body="${comment_body}
|
||||
|
||||
run tpch-sf${SF} query with default conf and set session variable runtime_filter_mode=off
|
||||
$(sed -n "${line_begin},${line_end}p" "${teamcity_build_checkoutDir}"/run-tpch-queries.log)"
|
||||
|
||||
echo "#### 4. comment result on tpch"
|
||||
comment_body=$(echo "${comment_body}" | sed -e ':a;N;$!ba;s/\t/\\t/g;s/\n/\\n/g') # 将所有的 Tab字符替换为\t 换行符替换为\n
|
||||
create_an_issue_comment_tpch "${pull_request_id:-}" "${comment_body}"
|
||||
|
||||
stop_doris
|
||||
)
|
||||
exit_flag="$?"
|
||||
|
||||
echo "#### 5. check if need backup doris logs"
|
||||
if [[ ${exit_flag} != "0" ]]; then
|
||||
print_doris_fe_log
|
||||
print_doris_be_log
|
||||
if file_name=$(archive_doris_logs "${pull_request_id}_${commit_id}_doris_logs.tar.gz"); then
|
||||
upload_doris_log_to_oss "${file_name}"
|
||||
fi
|
||||
fi
|
||||
|
||||
exit "${exit_flag}"
|
||||
Reference in New Issue
Block a user