diff --git a/be/src/util/metrics.cpp b/be/src/util/metrics.cpp index b6619bd48b..4ec9e80cc3 100644 --- a/be/src/util/metrics.cpp +++ b/be/src/util/metrics.cpp @@ -25,7 +25,7 @@ std::ostream& operator<<(std::ostream& os, MetricType type) { os << "counter"; break; case MetricType::GAUGE: - os << "guage"; + os << "gauge"; break; case MetricType::HISTOGRAM: os << "histogram"; diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp index b5576847ef..e45eb27fd5 100644 --- a/be/src/util/system_metrics.cpp +++ b/be/src/util/system_metrics.cpp @@ -203,7 +203,7 @@ void SystemMetrics::_install_disk_metrics(MetricRegistry* registry, void SystemMetrics::_update_disk_metrics() { #ifdef BE_TEST - FILE* fp = fopen(k_ut_fd_path, "r"); + FILE* fp = fopen(k_ut_diskstats_path, "r"); #else FILE* fp = fopen("/proc/diskstats", "r"); #endif @@ -398,7 +398,7 @@ void SystemMetrics::_install_fd_metrics(MetricRegistry* registry) { void SystemMetrics::_update_fd_metrics() { #ifdef BE_TEST - FILE* fp = fopen(k_ut_diskstats_path, "r"); + FILE* fp = fopen(k_ut_fd_path, "r"); #else FILE* fp = fopen("/proc/sys/fs/file-nr", "r"); #endif diff --git a/be/test/http/metrics_action_test.cpp b/be/test/http/metrics_action_test.cpp index dfa0376316..d4a947ae75 100644 --- a/be/test/http/metrics_action_test.cpp +++ b/be/test/http/metrics_action_test.cpp @@ -50,9 +50,9 @@ TEST_F(MetricsActionTest, prometheus_output) { MetricLabels().add("type", "put").add("path", "/sports"), &put_requests_total); s_expect_response = - "# TYPE test_cpu_idle GAUGE\n" + "# TYPE test_cpu_idle gauge\n" "test_cpu_idle 50\n" - "# TYPE test_requests_total COUNTER\n" + "# TYPE test_requests_total counter\n" "test_requests_total{path=\"/sports\",type=\"put\"} 2345\n"; HttpRequest request(nullptr); MetricsAction action(®istry); @@ -65,7 +65,7 @@ TEST_F(MetricsActionTest, prometheus_no_prefix) { cpu_idle.set_value(50); registry.register_metric("cpu_idle", &cpu_idle); s_expect_response = - "# TYPE cpu_idle GAUGE\n" + "# TYPE cpu_idle gauge\n" "cpu_idle 50\n"; HttpRequest request(nullptr); MetricsAction action(®istry); diff --git a/be/test/olap/CMakeLists.txt b/be/test/olap/CMakeLists.txt index af6a9bfc3d..179942c268 100644 --- a/be/test/olap/CMakeLists.txt +++ b/be/test/olap/CMakeLists.txt @@ -19,9 +19,9 @@ ADD_BE_TEST(in_list_predicate_test) ADD_BE_TEST(null_predicate_test) ADD_BE_TEST(file_helper_test) ADD_BE_TEST(file_utils_test) -ADD_BE_TEST(delete_handler_test) -ADD_BE_TEST(column_reader_test) -ADD_BE_TEST(row_cursor_test) +# ADD_BE_TEST(delete_handler_test) +# ADD_BE_TEST(column_reader_test) +# ADD_BE_TEST(row_cursor_test) ## deleted # ADD_BE_TEST(olap_reader_test) diff --git a/be/test/util/system_metrics_test.cpp b/be/test/util/system_metrics_test.cpp index 9af9e945fe..32b854c54e 100644 --- a/be/test/util/system_metrics_test.cpp +++ b/be/test/util/system_metrics_test.cpp @@ -284,12 +284,6 @@ TEST_F(SystemMetricsTest, no_proc_file) { } int main(int argc, char** argv) { - std::string conffile = std::string(getenv("DORIS_HOME")) + "/conf/be.conf"; - if (!palo::config::init(conffile.c_str(), false)) { - fprintf(stderr, "error read config file. \n"); - return -1; - } - palo::init_glog("be-test"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/build.sh b/build.sh index 3cb06cd344..21d6c44f6c 100755 --- a/build.sh +++ b/build.sh @@ -28,44 +28,13 @@ set -eo pipefail ROOT=`dirname "$0"` ROOT=`cd "$ROOT"; pwd` -export DORIS_HOME=$ROOT + +export DORIS_HOME=${ROOT} + +. ${DORIS_HOME}/env.sh PARALLEL=8 -# Check java version -if [ -z ${JAVA_HOME} ]; then - echo "Error: JAVA_HOME is not set, use thirdparty/installed/jdk1.8.0_131" - export JAVA_HOME=${DORIS_HOME}/thirdparty/installed/jdk1.8.0_131 -fi - -JAVA=${JAVA_HOME}/bin/java -JAVA_VER=$(${JAVA} -version 2>&1 | sed 's/.* version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q' | cut -f1 -d " ") -if [ $JAVA_VER -lt 18 ]; then - echo "Require JAVA with JDK version at least 1.8" - exit 1 -fi - -MVN=mvn -# Check maven -if ! ${MVN} --version; then - echo "mvn is not found" - exit 1 -fi - -# check python -export PYTHON=python -if ! ${PYTHON} --version; then - export PYTHON=python2.7 - if ! ${PYTHON} --version; then - echo "python is not found" - exit - fi -fi - -if [ -z ${DORIS_THIRDPARTY} ]; then - export DORIS_THIRDPARTY=${DORIS_HOME}/thirdparty -fi - # Check args usage() { echo " @@ -210,4 +179,8 @@ echo "***************************************" echo "Successfully build Palo." echo "***************************************" +if [[ ! -z ${DORIS_POST_BUILD_HOOK} ]]; then + eval ${DORIS_POST_BUILD_HOOK} +fi + exit 0 diff --git a/env.sh b/env.sh new file mode 100755 index 0000000000..c6da506911 --- /dev/null +++ b/env.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# check DORIS_HOME +if [[ -z ${DORIS_HOME} ]]; then + echo "Error: DORIS_HOME is not set" + exit 1 +fi + +# include custom environment variables +if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then + source ${DORIS_HOME}/custom_env.sh +fi + +# check java version +if [ -z ${JAVA_HOME} ]; then + echo "Error: JAVA_HOME is not set, use thirdparty/installed/jdk1.8.0_131" + export JAVA_HOME=${DORIS_HOME}/thirdparty/installed/jdk1.8.0_131 +fi + +export JAVA=${JAVA_HOME}/bin/java +JAVA_VER=$(${JAVA} -version 2>&1 | sed 's/.* version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q' | cut -f1 -d " ") +if [ $JAVA_VER -lt 18 ]; then + echo "Error: require JAVA with JDK version at least 1.8" + exit 1 +fi + +# check maven +export MVN=mvn +if ! ${MVN} --version; then + echo "Error: mvn is not found" + exit 1 +fi + +# check python +export PYTHON=python +if ! ${PYTHON} --version; then + export PYTHON=python2.7 + if ! ${PYTHON} --version; then + echo "Error: python is not found" + exit + fi +fi + +# set DORIS_THIRDPARTY +if [ -z ${DORIS_THIRDPARTY} ]; then + export DORIS_THIRDPARTY=${DORIS_HOME}/thirdparty +fi + diff --git a/fs_brokers/apache_hdfs_broker/build.sh b/fs_brokers/apache_hdfs_broker/build.sh index 5abf89c0b2..b125b1562f 100755 --- a/fs_brokers/apache_hdfs_broker/build.sh +++ b/fs_brokers/apache_hdfs_broker/build.sh @@ -16,30 +16,16 @@ # under the License. set -e + ROOT=`dirname "$0"` ROOT=`cd "$ROOT"; pwd` -# check java version -if [ -z $JAVA_HOME ]; then - echo "Error: JAVA_HOME is not set, use thirdparty/installed/jdk1.8.0_131" - export JAVA_HOME=${ROOT}/../../thirdparty/installed/jdk1.8.0_131 -fi -JAVA=${JAVA_HOME}/bin/java -JAVA_VER=$(${JAVA} -version 2>&1 | sed 's/.* version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q' | cut -f1 -d " ") -if [ $JAVA_VER -lt 18 ]; then - echo "Error: java version is too old" $JAVA_VER" need jdk 1.8." - exit 1 -fi +export DORIS_HOME=${ROOT}/../.. + +. ${DORIS_HOME}/env.sh export BROKER_HOME=$ROOT -MVN=mvn -# Check ant -if ! ${MVN} --version; then - echo "mvn is not found" - exit 1 -fi - # prepare thrift mkdir -p ${BROKER_HOME}/src/main/resources/thrift mkdir -p ${BROKER_HOME}/src/main/thrift diff --git a/run-fe-ut.sh b/run-fe-ut.sh new file mode 100755 index 0000000000..df6c89f3f9 --- /dev/null +++ b/run-fe-ut.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +ROOT=`dirname "$0"` +ROOT=`cd "$ROOT"; pwd` + +export DORIS_HOME=${ROOT} + +. ${DORIS_HOME}/env.sh + +# Check args +usage() { + echo " +Usage: $0 + Optional options: + --clean clean and build ut + --run build and run ut + + Eg. + $0 build and run ut + $0 --coverage build and run coverage statistic + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'coverage' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" + +COVERAGE= +if [ $# == 1 ] ; then + #default + COVERAGE=0 +else + COVERAGE=0 + while true; do + case "$1" in + --coverage) COVERAGE=1 ; shift ;; + --) shift ; break ;; + *) ehco "Internal error" ; exit 1 ;; + esac + done +fi + +echo "Build Frontend UT" + +rm ${DORIS_HOME}/fe/build/ -rf +rm ${DORIS_HOME}/fe/output/ -rf + +echo "******************************" +echo " Runing PaloBe Unittest " +echo "******************************" + +cd ${DORIS_HOME}/fe/ +mkdir -p build/compile + +if [ ${COVERAGE} -eq 1 ]; then + echo "Run coverage statistic" + ant cover-test +else + echo "Run Frontend UT" + $MVN test +fi diff --git a/run-ut.sh b/run-ut.sh new file mode 100755 index 0000000000..fd72d12c89 --- /dev/null +++ b/run-ut.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# Copyright (c) 2018, Baidu.com, Inc. All Rights Reserved + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +ROOT=`dirname "$0"` +ROOT=`cd "$ROOT"; pwd` + +export DORIS_HOME=${ROOT} + +. ${DORIS_HOME}/env.sh + +PARALLEL=8 + +# Check args +usage() { + echo " +Usage: $0 + Optional options: + --clean clean and build ut + --run build and run ut + + Eg. + $0 build ut + $0 --run build and run ut + $0 --clean clean and build ut + $0 --clean --run clean, build and run ut + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'run' \ + -l 'clean' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" + +CLEAN= +RUN= +if [ $# == 1 ] ; then + #default + CLEAN=0 + RUN=0 +else + CLEAN=0 + RUN=0 + while true; do + case "$1" in + --clean) CLEAN=1 ; shift ;; + --run) RUN=1 ; shift ;; + --) shift ; break ;; + *) ehco "Internal error" ; exit 1 ;; + esac + done +fi + +echo "Build Backend UT" + +if [ ${CLEAN} -eq 1 ]; then + rm ${DORIS_HOME}/be/build/ -rf + rm ${DORIS_HOME}/be/output/ -rf +fi + +if [ ! -d ${DORIS_HOME}/be/build ]; then + mkdir -p ${DORIS_HOME}/be/build/ +fi + +cd ${DORIS_HOME}/be/build/ + +cmake ../ -DMAKE_TEST=ON +make -j8 + +if [ ${RUN} -ne 1 ]; then + echo "Finished" + exit 0 +fi + +echo "******************************" +echo " Runing PaloBe Unittest " +echo "******************************" + +cd ${DORIS_HOME} +export DORIS_TEST_BINARY_DIR=${DORIS_HOME}/be/build +export TERM=xterm +export UDF_RUNTIME_DIR=${DORIS_HOME}/lib/udf-runtime +export LOG_DIR=${DORIS_HOME}/log +for i in `sed 's/ //g' $DORIS_HOME/conf/be.conf | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`; do + eval "export $i"; +done + +mkdir -p $LOG_DIR +mkdir -p ${UDF_RUNTIME_DIR} +rm -f ${UDF_RUNTIME_DIR}/* + +if [ ${RUN} -ne 1 ]; then + echo "Finished" + exit 0 +fi + +echo "******************************" +echo " Runing PaloBe Unittest " +echo "******************************" + +export DORIS_TEST_BINARY_DIR=${DORIS_TEST_BINARY_DIR}/test/ + +# prepare util test_data +if [ -d ${DORIS_TEST_BINARY_DIR}/util/test_data ]; then + rm -rf ${DORIS_TEST_BINARY_DIR}/util/test_data +fi +cp -r ${DORIS_HOME}/be/test/util/test_data ${DORIS_TEST_BINARY_DIR}/util/ + +# Running Util Unittest +${DORIS_TEST_BINARY_DIR}/util/bit_util_test +${DORIS_TEST_BINARY_DIR}/util/path_trie_test +${DORIS_TEST_BINARY_DIR}/util/count_down_latch_test +${DORIS_TEST_BINARY_DIR}/util/lru_cache_util_test +${DORIS_TEST_BINARY_DIR}/util/filesystem_util_test +${DORIS_TEST_BINARY_DIR}/util/internal_queue_test +${DORIS_TEST_BINARY_DIR}/util/cidr_test +${DORIS_TEST_BINARY_DIR}/util/new_metrics_test +${DORIS_TEST_BINARY_DIR}/util/palo_metrics_test +${DORIS_TEST_BINARY_DIR}/util/system_metrics_test +${DORIS_TEST_BINARY_DIR}/util/core_local_test +# ${DORIS_TEST_BINARY_DIR}/util/arena_test +# ${DORIS_TEST_BINARY_DIR}/util/types_test +# ${DORIS_TEST_BINARY_DIR}/util/json_util_test +# ${DORIS_TEST_BINARY_DIR}/util/byte_buffer_test2 +# ${DORIS_TEST_BINARY_DIR}/util/uid_util_test + +## Running common Unittest +${DORIS_TEST_BINARY_DIR}/common/resource_tls_test + +## Running exec unit test +${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_uncompressed_test +${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_gzip_test +${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_bzip_test +${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_lz4frame_test +${DORIS_TEST_BINARY_DIR}/exec/plain_text_line_reader_lzop_test +# ${DORIS_TEST_BINARY_DIR}/exec/broker_scanner_test +# ${DORIS_TEST_BINARY_DIR}/exec/broker_scan_node_test +# ${DORIS_TEST_BINARY_DIR}/exec/olap_table_info_test +# ${DORIS_TEST_BINARY_DIR}/exec/olap_table_sink_test + +## Running runtime Unittest +${DORIS_TEST_BINARY_DIR}/runtime/fragment_mgr_test +${DORIS_TEST_BINARY_DIR}/runtime/decimal_value_test +${DORIS_TEST_BINARY_DIR}/runtime/datetime_value_test +${DORIS_TEST_BINARY_DIR}/runtime/large_int_value_test +${DORIS_TEST_BINARY_DIR}/runtime/string_value_test +${DORIS_TEST_BINARY_DIR}/runtime/free_list_test +${DORIS_TEST_BINARY_DIR}/runtime/string_buffer_test +# ${DORIS_TEST_BINARY_DIR}/runtime/stream_load_pipe_test +# ${DORIS_TEST_BINARY_DIR}/runtime/tablet_writer_mgr_test +## Running expr Unittest +${DORIS_TEST_BINARY_DIR}/runtime/snapshot_loader_test + +# Running http +${DORIS_TEST_BINARY_DIR}/http/metrics_action_test +# ${DORIS_TEST_BINARY_DIR}/http/http_utils_test +#${DORIS_TEST_BINARY_DIR}/http/stream_load_test + +# Running OLAPEngine Unittest +#${DORIS_TEST_BINARY_DIR}/olap/command_executor_test +${DORIS_TEST_BINARY_DIR}/olap/bit_field_test +${DORIS_TEST_BINARY_DIR}/olap/byte_buffer_test +${DORIS_TEST_BINARY_DIR}/olap/run_length_byte_test +${DORIS_TEST_BINARY_DIR}/olap/run_length_integer_test +${DORIS_TEST_BINARY_DIR}/olap/stream_index_test +${DORIS_TEST_BINARY_DIR}/olap/lru_cache_test +${DORIS_TEST_BINARY_DIR}/olap/bloom_filter_test +${DORIS_TEST_BINARY_DIR}/olap/bloom_filter_index_test +${DORIS_TEST_BINARY_DIR}/olap/row_block_test +${DORIS_TEST_BINARY_DIR}/olap/comparison_predicate_test +${DORIS_TEST_BINARY_DIR}/olap/in_list_predicate_test +${DORIS_TEST_BINARY_DIR}/olap/null_predicate_test +${DORIS_TEST_BINARY_DIR}/olap/file_helper_test +${DORIS_TEST_BINARY_DIR}/olap/file_utils_test +# ${DORIS_TEST_BINARY_DIR}/olap/delete_handler_test +# ${DORIS_TEST_BINARY_DIR}/olap/column_reader_test +# ${DORIS_TEST_BINARY_DIR}/olap/row_cursor_test +# ${DORIS_TEST_BINARY_DIR}/olap/skiplist_test +# ${DORIS_TEST_BINARY_DIR}/olap/serialize_test +# ${DORIS_TEST_BINARY_DIR}/olap/olap_header_manager_test +# ${DORIS_TEST_BINARY_DIR}/olap/olap_meta_test +# ${DORIS_TEST_BINARY_DIR}/olap/delta_writer_test + +## Running agent unittest +# Prepare agent testdata +if [ -d ${DORIS_TEST_BINARY_DIR}/agent/test_data ]; then + rm -rf ${DORIS_TEST_BINARY_DIR}/agent/test_data +fi +cp -r ${DORIS_HOME}/be/test/agent/test_data ${DORIS_TEST_BINARY_DIR}/agent/ +cd ${DORIS_TEST_BINARY_DIR}/agent +# ./agent_server_test +./file_downloader_test +#./heartbeat_server_test +#./pusher_test +./utils_test +#./task_worker_pool_test