database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Added: change Doris build.sh to get environment variables from

Browse Source 
custom_env.sh, and add run-ut.sh and run-fe-ut.sh
This commit is contained in:
zhaochun
2018-10-30 22:21:33 +08:00
parent ae9ce81453
commit 765c91bbc2
10 changed files with 388 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/util/metrics.cpp
View File

@ -25,7 +25,7 @@ std::ostream& operator<<(std::ostream& os, MetricType type) {
os << "counter";
break;
case MetricType::GAUGE:
os << "guage";
os << "gauge";
break;
case MetricType::HISTOGRAM:
os << "histogram";

4
be/src/util/system_metrics.cpp
View File

@ -203,7 +203,7 @@ void SystemMetrics::_install_disk_metrics(MetricRegistry* registry,
void SystemMetrics::_update_disk_metrics() {
#ifdef BE_TEST
FILE* fp = fopen(k_ut_fd_path, "r");
FILE* fp = fopen(k_ut_diskstats_path, "r");
#else
FILE* fp = fopen("/proc/diskstats", "r");
#endif
@ -398,7 +398,7 @@ void SystemMetrics::_install_fd_metrics(MetricRegistry* registry) {
void SystemMetrics::_update_fd_metrics() {
#ifdef BE_TEST
FILE* fp = fopen(k_ut_diskstats_path, "r");
FILE* fp = fopen(k_ut_fd_path, "r");
#else
FILE* fp = fopen("/proc/sys/fs/file-nr", "r");
#endif

6
be/test/http/metrics_action_test.cpp
View File

@ -50,9 +50,9 @@ TEST_F(MetricsActionTest, prometheus_output) {
MetricLabels().add("type", "put").add("path", "/sports"),
&put_requests_total);
s_expect_response =
"# TYPE test_cpu_idle GAUGE\n"
"# TYPE test_cpu_idle gauge\n"
"test_cpu_idle 50\n"
"# TYPE test_requests_total COUNTER\n"
"# TYPE test_requests_total counter\n"
"test_requests_total{path=\"/sports\",type=\"put\"} 2345\n";
HttpRequest request(nullptr);
MetricsAction action(&registry);
@ -65,7 +65,7 @@ TEST_F(MetricsActionTest, prometheus_no_prefix) {
cpu_idle.set_value(50);
registry.register_metric("cpu_idle", &cpu_idle);
s_expect_response =
"# TYPE cpu_idle GAUGE\n"
"# TYPE cpu_idle gauge\n"
"cpu_idle 50\n";
HttpRequest request(nullptr);
MetricsAction action(&registry);

6
be/test/olap/CMakeLists.txt
View File

@ -19,9 +19,9 @@ ADD_BE_TEST(in_list_predicate_test)
ADD_BE_TEST(null_predicate_test)
ADD_BE_TEST(file_helper_test)
ADD_BE_TEST(file_utils_test)
ADD_BE_TEST(delete_handler_test)
ADD_BE_TEST(column_reader_test)
ADD_BE_TEST(row_cursor_test)
# ADD_BE_TEST(delete_handler_test)
# ADD_BE_TEST(column_reader_test)
# ADD_BE_TEST(row_cursor_test)
## deleted
# ADD_BE_TEST(olap_reader_test)

6
be/test/util/system_metrics_test.cpp
View File

@ -284,12 +284,6 @@ TEST_F(SystemMetricsTest, no_proc_file) {
}
int main(int argc, char** argv) {
std::string conffile = std::string(getenv("DORIS_HOME")) + "/conf/be.conf";
if (!palo::config::init(conffile.c_str(), false)) {
fprintf(stderr, "error read config file. \n");
return -1;
}
palo::init_glog("be-test");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

43
build.sh
View File

@ -28,44 +28,13 @@ set -eo pipefail
ROOT=`dirname "$0"`
ROOT=`cd "$ROOT"; pwd`
export DORIS_HOME=$ROOT
export DORIS_HOME=${ROOT}
. ${DORIS_HOME}/env.sh
PARALLEL=8
# Check java version
if [ -z ${JAVA_HOME} ]; then
echo "Error: JAVA_HOME is not set, use thirdparty/installed/jdk1.8.0_131"
export JAVA_HOME=${DORIS_HOME}/thirdparty/installed/jdk1.8.0_131
fi
JAVA=${JAVA_HOME}/bin/java
JAVA_VER=$(${JAVA} -version 2>&1 | sed 's/.* version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q' | cut -f1 -d " ")
if [ $JAVA_VER -lt 18 ]; then
echo "Require JAVA with JDK version at least 1.8"
exit 1
fi
MVN=mvn
# Check maven
if ! ${MVN} --version; then
echo "mvn is not found"
exit 1
fi
# check python
export PYTHON=python
if ! ${PYTHON} --version; then
export PYTHON=python2.7
if ! ${PYTHON} --version; then
echo "python is not found"
exit
fi
fi
if [ -z ${DORIS_THIRDPARTY} ]; then
export DORIS_THIRDPARTY=${DORIS_HOME}/thirdparty
fi
# Check args
usage() {
echo "
@ -210,4 +179,8 @@ echo "***************************************"
echo "Successfully build Palo."
echo "***************************************"
if [[ ! -z ${DORIS_POST_BUILD_HOOK} ]]; then
eval ${DORIS_POST_BUILD_HOOK}
fi
exit 0

62
env.sh Executable file
View File

@ -0,0 +1,62 @@
#!/usr/bin/env bash
# Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# check DORIS_HOME
if [[ -z ${DORIS_HOME} ]]; then
echo "Error: DORIS_HOME is not set"
exit 1
fi
# include custom environment variables
if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then
source ${DORIS_HOME}/custom_env.sh
fi
# check java version
if [ -z ${JAVA_HOME} ]; then
echo "Error: JAVA_HOME is not set, use thirdparty/installed/jdk1.8.0_131"
export JAVA_HOME=${DORIS_HOME}/thirdparty/installed/jdk1.8.0_131
fi
export JAVA=${JAVA_HOME}/bin/java
JAVA_VER=$(${JAVA} -version 2>&1 | sed 's/.* version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q' | cut -f1 -d " ")
if [ $JAVA_VER -lt 18 ]; then
echo "Error: require JAVA with JDK version at least 1.8"
exit 1
fi
# check maven
export MVN=mvn
if ! ${MVN} --version; then
echo "Error: mvn is not found"
exit 1
fi
# check python
export PYTHON=python
if ! ${PYTHON} --version; then
export PYTHON=python2.7
if ! ${PYTHON} --version; then
echo "Error: python is not found"
exit
fi
fi
# set DORIS_THIRDPARTY
if [ -z ${DORIS_THIRDPARTY} ]; then
export DORIS_THIRDPARTY=${DORIS_HOME}/thirdparty
fi

22
fs_brokers/apache_hdfs_broker/build.sh
View File

@ -16,30 +16,16 @@
# under the License.
set -e
ROOT=`dirname "$0"`
ROOT=`cd "$ROOT"; pwd`
# check java version
if [ -z $JAVA_HOME ]; then
echo "Error: JAVA_HOME is not set, use thirdparty/installed/jdk1.8.0_131"
export JAVA_HOME=${ROOT}/../../thirdparty/installed/jdk1.8.0_131
fi
JAVA=${JAVA_HOME}/bin/java
JAVA_VER=$(${JAVA} -version 2>&1 | sed 's/.* version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q' | cut -f1 -d " ")
if [ $JAVA_VER -lt 18 ]; then
echo "Error: java version is too old" $JAVA_VER" need jdk 1.8."
exit 1
fi
export DORIS_HOME=${ROOT}/../..
. ${DORIS_HOME}/env.sh
export BROKER_HOME=$ROOT
MVN=mvn
# Check ant
if ! ${MVN} --version; then
echo "mvn is not found"
exit 1
fi
# prepare thrift
mkdir -p ${BROKER_HOME}/src/main/resources/thrift
mkdir -p ${BROKER_HOME}/src/main/thrift

86
run-fe-ut.sh Executable file
View File

@ -0,0 +1,86 @@
#!/usr/bin/env bash
# Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -eo pipefail
ROOT=`dirname "$0"`
ROOT=`cd "$ROOT"; pwd`
export DORIS_HOME=${ROOT}
. ${DORIS_HOME}/env.sh
# Check args
usage() {
echo "
Usage: $0 <options>
Optional options:
--clean clean and build ut
--run build and run ut
Eg.
$0 build and run ut
$0 --coverage build and run coverage statistic
"
exit 1
}
OPTS=$(getopt \
-n $0 \
-o '' \
-l 'coverage' \
-- "$@")
if [ $? != 0 ] ; then
usage
fi
eval set -- "$OPTS"
COVERAGE=
if [ $# == 1 ] ; then
#default
COVERAGE=0
else
COVERAGE=0
while true; do
case "$1" in
--coverage) COVERAGE=1 ; shift ;;
--) shift ; break ;;
*) ehco "Internal error" ; exit 1 ;;
esac
done
fi
echo "Build Frontend UT"
rm ${DORIS_HOME}/fe/build/ -rf
rm ${DORIS_HOME}/fe/output/ -rf
echo "******************************"
echo " Runing PaloBe Unittest "
echo "******************************"
cd ${DORIS_HOME}/fe/
mkdir -p build/compile
if [ ${COVERAGE} -eq 1 ]; then
echo "Run coverage statistic"
ant cover-test
else
echo "Run Frontend UT"
$MVN test
fi

219
run-ut.sh Executable file
View File

@ -0,0 +1,219 @@
#!/usr/bin/env bash
# Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -eo pipefail
ROOT=`dirname "$0"`
ROOT=`cd "$ROOT"; pwd`
export DORIS_HOME=${ROOT}
. ${DORIS_HOME}/env.sh
PARALLEL=8
# Check args
usage() {
echo "
Usage: $0 <options>
Optional options:
--clean clean and build ut
--run build and run ut
Eg.
$0 build ut
$0 --run build and run ut
$0 --clean clean and build ut
$0 --clean --run clean, build and run ut
"
exit 1
}
OPTS=$(getopt \
-n $0 \
-o '' \
-l 'run' \
-l 'clean' \
-- "$@")
if [ $? != 0 ] ; then
usage
fi
eval set -- "$OPTS"
CLEAN=
RUN=
if [ $# == 1 ] ; then
#default
CLEAN=0
RUN=0
else
CLEAN=0
RUN=0
while true; do
case "$1" in
--clean) CLEAN=1 ; shift ;;
--run) RUN=1 ; shift ;;
--) shift ; break ;;
*) ehco "Internal error" ; exit 1 ;;
esac
done
fi
echo "Build Backend UT"
if [ ${CLEAN} -eq 1 ]; then
rm ${DORIS_HOME}/be/build/ -rf
rm ${DORIS_HOME}/be/output/ -rf
fi
if [ ! -d ${DORIS_HOME}/be/build ]; then
mkdir -p ${DORIS_HOME}/be/build/
fi
cd ${DORIS_HOME}/be/build/
cmake ../ -DMAKE_TEST=ON
make -j8
if [ ${RUN} -ne 1 ]; then
echo "Finished"
exit 0
fi
echo "******************************"
echo " Runing PaloBe Unittest "
echo "******************************"
cd ${DORIS_HOME}
export DORIS_TEST_BINARY_DIR=${DORIS_HOME}/be/build
export TERM=xterm
export UDF_RUNTIME_DIR=${DORIS_HOME}/lib/udf-runtime
export LOG_DIR=${DORIS_HOME}/log
for i in `sed 's/ //g' $DORIS_HOME/conf/be.conf | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`; do
eval "export $i";
done
mkdir -p $LOG_DIR
mkdir -p ${UDF_RUNTIME_DIR}
rm -f ${UDF_RUNTIME_DIR}/*
if [ ${RUN} -ne 1 ]; then
echo "Finished"
exit 0
fi
echo "******************************"
echo " Runing PaloBe Unittest "
echo "******************************"
export DORIS_TEST_BINARY_DIR=${DORIS_TEST_BINARY_DIR}/test/
# prepare util test_data
if [ -d ${DORIS_TEST_BINARY_DIR}/util/test_data ]; then
rm -rf ${DORIS_TEST_BINARY_DIR}/util/test_data
fi
cp -r ${DORIS_HOME}/be/test/util/test_data ${DORIS_TEST_BINARY_DIR}/util/
# Running Util Unittest
${DORIS_TEST_BINARY_DIR}/util/bit_util_test
${DORIS_TEST_BINARY_DIR}/util/path_trie_test
${DORIS_TEST_BINARY_DIR}/util/count_down_latch_test
${DORIS_TEST_BINARY_DIR}/util/lru_cache_util_test
${DORIS_TEST_BINARY_DIR}/util/filesystem_util_test
${DORIS_TEST_BINARY_DIR}/util/internal_queue_test
${DORIS_TEST_BINARY_DIR}/util/cidr_test
${DORIS_TEST_BINARY_DIR}/util/new_metrics_test
${DORIS_TEST_BINARY_DIR}/util/palo_metrics_test
${DORIS_TEST_BINARY_DIR}/util/system_metrics_test
${DORIS_TEST_BINARY_DIR}/util/core_local_test
# ${DORIS_TEST_BINARY_DIR}/util/arena_test
# ${DORIS_TEST_BINARY_DIR}/util/types_test
# ${DORIS_TEST_BINARY_DIR}/util/json_util_test
# ${DORIS_TEST_BINARY_DIR}/util/byte_buffer_test2
# ${DORIS_TEST_BINARY_DIR}/util/uid_util_test
## Running common Unittest
${DORIS_TEST_BINARY_DIR}/common/resource_tls_test
## Running exec unit test
${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_uncompressed_test
${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_gzip_test
${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_bzip_test
${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_lz4frame_test
${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_lzop_test
# ${DORIS_TEST_BINARY_DIR}/exec/broker_scanner_test
# ${DORIS_TEST_BINARY_DIR}/exec/broker_scan_node_test
# ${DORIS_TEST_BINARY_DIR}/exec/olap_table_info_test
# ${DORIS_TEST_BINARY_DIR}/exec/olap_table_sink_test
## Running runtime Unittest
${DORIS_TEST_BINARY_DIR}/runtime/fragment_mgr_test
${DORIS_TEST_BINARY_DIR}/runtime/decimal_value_test
${DORIS_TEST_BINARY_DIR}/runtime/datetime_value_test
${DORIS_TEST_BINARY_DIR}/runtime/large_int_value_test
${DORIS_TEST_BINARY_DIR}/runtime/string_value_test
${DORIS_TEST_BINARY_DIR}/runtime/free_list_test
${DORIS_TEST_BINARY_DIR}/runtime/string_buffer_test
# ${DORIS_TEST_BINARY_DIR}/runtime/stream_load_pipe_test
# ${DORIS_TEST_BINARY_DIR}/runtime/tablet_writer_mgr_test
## Running expr Unittest
${DORIS_TEST_BINARY_DIR}/runtime/snapshot_loader_test
# Running http
${DORIS_TEST_BINARY_DIR}/http/metrics_action_test
# ${DORIS_TEST_BINARY_DIR}/http/http_utils_test
#${DORIS_TEST_BINARY_DIR}/http/stream_load_test
# Running OLAPEngine Unittest
#${DORIS_TEST_BINARY_DIR}/olap/command_executor_test
${DORIS_TEST_BINARY_DIR}/olap/bit_field_test
${DORIS_TEST_BINARY_DIR}/olap/byte_buffer_test
${DORIS_TEST_BINARY_DIR}/olap/run_length_byte_test
${DORIS_TEST_BINARY_DIR}/olap/run_length_integer_test
${DORIS_TEST_BINARY_DIR}/olap/stream_index_test
${DORIS_TEST_BINARY_DIR}/olap/lru_cache_test
${DORIS_TEST_BINARY_DIR}/olap/bloom_filter_test
${DORIS_TEST_BINARY_DIR}/olap/bloom_filter_index_test
${DORIS_TEST_BINARY_DIR}/olap/row_block_test
${DORIS_TEST_BINARY_DIR}/olap/comparison_predicate_test
${DORIS_TEST_BINARY_DIR}/olap/in_list_predicate_test
${DORIS_TEST_BINARY_DIR}/olap/null_predicate_test
${DORIS_TEST_BINARY_DIR}/olap/file_helper_test
${DORIS_TEST_BINARY_DIR}/olap/file_utils_test
# ${DORIS_TEST_BINARY_DIR}/olap/delete_handler_test
# ${DORIS_TEST_BINARY_DIR}/olap/column_reader_test
# ${DORIS_TEST_BINARY_DIR}/olap/row_cursor_test
# ${DORIS_TEST_BINARY_DIR}/olap/skiplist_test
# ${DORIS_TEST_BINARY_DIR}/olap/serialize_test
# ${DORIS_TEST_BINARY_DIR}/olap/olap_header_manager_test
# ${DORIS_TEST_BINARY_DIR}/olap/olap_meta_test
# ${DORIS_TEST_BINARY_DIR}/olap/delta_writer_test
## Running agent unittest
# Prepare agent testdata
if [ -d ${DORIS_TEST_BINARY_DIR}/agent/test_data ]; then
rm -rf ${DORIS_TEST_BINARY_DIR}/agent/test_data
fi
cp -r ${DORIS_HOME}/be/test/agent/test_data ${DORIS_TEST_BINARY_DIR}/agent/
cd ${DORIS_TEST_BINARY_DIR}/agent
# ./agent_server_test
./file_downloader_test
#./heartbeat_server_test
#./pusher_test
./utils_test
#./task_worker_pool_test