diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 3ae2b6d44c..2ccc60e387 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -484,6 +484,7 @@ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} # Set libraries for test set (TEST_LINK_LIBS ${DORIS_LINK_LIBS} ${WL_START_GROUP} + Test_util gmock gtest ${WL_END_GROUP} @@ -555,6 +556,7 @@ FUNCTION(ADD_BE_PLUGIN PLUGIN_NAME) ENDFUNCTION() if (${MAKE_TEST} STREQUAL "ON") + add_subdirectory(${TEST_DIR}/test_util) add_subdirectory(${TEST_DIR}/agent) add_subdirectory(${TEST_DIR}/common) add_subdirectory(${TEST_DIR}/env) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 5e9259a2b6..b2d382e950 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -248,7 +248,7 @@ Status ColumnReader::get_row_ranges_by_bloom_filter(CondColumn* cond_column, int64_t idx = from; int64_t to = row_ranges->get_range_to(i); auto iter = _ordinal_index->seek_at_or_before(from); - while (idx < to) { + while (idx < to && iter.valid()) { page_ids.insert(iter.page_index()); idx = iter.last_ordinal() + 1; iter.next(); @@ -444,7 +444,7 @@ Status FileColumnIterator::seek_to_first() { Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) { // if current page contains this row, we don't need to seek - if (_page == nullptr || !_page->contains(ord)) { + if (_page == nullptr || !_page->contains(ord) || !_page_iter.valid()) { RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter)); RETURN_IF_ERROR(_read_data_page(_page_iter)); } diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index a85b0c0e76..b7aa5e5dac 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1028,7 +1028,9 @@ OLAPStatus TabletManager::start_trash_sweep() { int32_t clean_num = 0; do { +#ifndef BE_TEST sleep(1); +#endif clean_num = 0; // should get write lock here, because it will remove tablet from shut_down_tablets // and get tablet will access shut_down_tablets diff --git a/be/test/exprs/string_functions_test.cpp b/be/test/exprs/string_functions_test.cpp index eaa2cb749d..92c92e71e4 100644 --- a/be/test/exprs/string_functions_test.cpp +++ b/be/test/exprs/string_functions_test.cpp @@ -25,6 +25,7 @@ #include "exprs/anyval_util.h" #include "testutil/function_utils.h" #include "util/logging.h" +#include "test_util/test_util.h" namespace doris { @@ -47,7 +48,7 @@ TEST_F(StringFunctionsTest, do_money_format_bench) { doris_udf::FunctionContext* context = new doris_udf::FunctionContext(); StringVal expected = AnyValUtil::from_string_temp(context, std::string("9,223,372,036,854,775,807.00")); - for (int i = 0; i < 10000000; i++) { + for (int i = 0; i < LOOP_LESS_OR_MORE(10, 10000000); i++) { StringVal result = StringFunctions::do_money_format(context, "922337203685477580700"); // cent ASSERT_EQ(expected, result); diff --git a/be/test/exprs/topn_function_test.cpp b/be/test/exprs/topn_function_test.cpp index f13c2e2a8b..7f411333f8 100644 --- a/be/test/exprs/topn_function_test.cpp +++ b/be/test/exprs/topn_function_test.cpp @@ -20,6 +20,7 @@ #include "util/topn_counter.h" #include "testutil/function_utils.h" #include "zipf_distribution.h" +#include "test_util/test_util.h" #include #include @@ -28,7 +29,7 @@ namespace doris { static const uint32_t TOPN_NUM = 100; -static const uint32_t TOTAL_RECORDS = 1000000; +static const uint32_t TOTAL_RECORDS = LOOP_LESS_OR_MORE(1000, 1000000); static const uint32_t PARALLEL = 10; std::string gen_random(const int len) { @@ -149,7 +150,9 @@ void test_topn_accuracy(FunctionContext* ctx, int key_space, int space_expand_ra } TEST_F(TopNFunctionsTest, topn_accuracy) { - std::vector key_space_vec ({1000, 10000, 100000, 500000}); + std::vector small_key_space({100}); + std::vector large_key_space({1000, 10000, 100000, 500000}); + std::vector key_space_vec(LOOP_LESS_OR_MORE(small_key_space, large_key_space)); std::vector space_expand_rate_vec({20, 50, 100}); std::vector zipf_distribution_exponent_vec({0.5, 0.6, 1.0}); for (auto ket_space : key_space_vec) { diff --git a/be/test/olap/file_helper_test.cpp b/be/test/olap/file_helper_test.cpp index f4ccd25549..e22bdc1939 100644 --- a/be/test/olap/file_helper_test.cpp +++ b/be/test/olap/file_helper_test.cpp @@ -27,6 +27,7 @@ #include "gtest/gtest.h" #include "olap/olap_define.h" #include "util/logging.h" +#include "test_util/test_util.h" #ifndef BE_TEST #define BE_TEST @@ -93,7 +94,7 @@ TEST_F(FileHandlerTest, TestWrite) { char* large_bytes2[(1 << 10)]; memset(large_bytes2, 0, sizeof(char) * ((1 << 12))); int i = 1; - while (i < 1 << 17) { + while (i < LOOP_LESS_OR_MORE(1 << 10, 1 << 17)) { file_handler.write(large_bytes2, ((1 << 12))); ++i; } diff --git a/be/test/olap/lru_cache_test.cpp b/be/test/olap/lru_cache_test.cpp index c143acc53f..b2b62acf6c 100644 --- a/be/test/olap/lru_cache_test.cpp +++ b/be/test/olap/lru_cache_test.cpp @@ -22,6 +22,7 @@ #include #include "util/logging.h" +#include "test_util/test_util.h" using namespace doris; using namespace std; @@ -293,7 +294,7 @@ TEST_F(CacheTest, NewId) { } TEST_F(CacheTest, SimpleBenchmark) { - for (int i = 0; i < kCacheSize * 10000; i++) { + for (int i = 0; i < kCacheSize * LOOP_LESS_OR_MORE(10, 10000); i++) { Insert(1000 + i, 2000 + i, 1); ASSERT_EQ(2000 + i, Lookup(1000 + i)); } diff --git a/be/test/olap/memory/column_delta_test.cpp b/be/test/olap/memory/column_delta_test.cpp index 2193e09d2b..6b1f01f130 100644 --- a/be/test/olap/memory/column_delta_test.cpp +++ b/be/test/olap/memory/column_delta_test.cpp @@ -23,13 +23,14 @@ #include #include "olap/memory/column.h" +#include "test_util/test_util.h" namespace doris { namespace memory { TEST(ColumnDelta, Index) { - const int BaseSize = 256001; - const int NumUpdate = 10000; + const int BaseSize = LOOP_LESS_OR_MORE(2560, 256001); + const int NumUpdate = LOOP_LESS_OR_MORE(100, 10000); srand(1); scoped_refptr delta(new ColumnDelta()); std::map updates; diff --git a/be/test/olap/memory/column_test.cpp b/be/test/olap/memory/column_test.cpp index 90411ed22d..d5c3dab288 100644 --- a/be/test/olap/memory/column_test.cpp +++ b/be/test/olap/memory/column_test.cpp @@ -21,16 +21,17 @@ #include "olap/memory/column_reader.h" #include "olap/memory/column_writer.h" +#include "test_util/test_util.h" namespace doris { namespace memory { +static const size_t InsertCount = LOOP_LESS_OR_MORE(1000, 1000000); +static const size_t UpdateTime = 70; +static const size_t UpdateCount = LOOP_LESS_OR_MORE(10, 10000); + template struct ColumnTest { - static const size_t InsertCount = 1000000; - static const size_t UpdateTime = 70; - static const size_t UpdateCount = 10000; - static bool is_null(CppType v) { return ((int64_t)v) % 10 == 0; } static void test_not_null() { @@ -46,7 +47,9 @@ struct ColumnTest { scoped_refptr newc; ASSERT_TRUE(writer->finalize(2).ok()); ASSERT_TRUE(writer->get_new_column(&newc).ok()); - EXPECT_TRUE(c.get() != newc.get()); + if (!AllowSlowTests()) { + EXPECT_TRUE(c.get() != newc.get()); + } std::unique_ptr readc; ASSERT_TRUE(newc->create_reader(2, &readc).ok()); for (uint32_t i = 0; i < values.size(); i++) { @@ -73,7 +76,9 @@ struct ColumnTest { scoped_refptr newc; ASSERT_TRUE(writer->finalize(2).ok()); ASSERT_TRUE(writer->get_new_column(&newc).ok()); - EXPECT_TRUE(c.get() != newc.get()); + if (!AllowSlowTests()) { + EXPECT_TRUE(c.get() != newc.get()); + } std::unique_ptr readc; ASSERT_TRUE(newc->create_reader(2, &readc).ok()); for (uint32_t i = 0; i < values.size(); i++) { diff --git a/be/test/olap/memory/mem_tablet_test.cpp b/be/test/olap/memory/mem_tablet_test.cpp index 9f0bff9619..ad4b2975c0 100644 --- a/be/test/olap/memory/mem_tablet_test.cpp +++ b/be/test/olap/memory/mem_tablet_test.cpp @@ -22,6 +22,7 @@ #include "olap/memory/mem_tablet_scan.h" #include "olap/memory/write_txn.h" #include "olap/tablet_meta.h" +#include "test_util/test_util.h" namespace doris { namespace memory { @@ -34,9 +35,9 @@ struct TData { }; TEST(MemTablet, writescan) { - const int num_insert = 2000000; - const int insert_per_write = 500000; - const int num_update = 10000; + const int num_insert = LOOP_LESS_OR_MORE(2000, 2000000); + const int insert_per_write = LOOP_LESS_OR_MORE(500, 500000); + const int num_update = LOOP_LESS_OR_MORE(10, 10000); const int update_time = 3; scoped_refptr sc; ASSERT_TRUE(Schema::create("id int,uv int,pv int,city tinyint null", &sc).ok()); diff --git a/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp b/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp index 5074c9454d..54614c9d3c 100644 --- a/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp @@ -31,6 +31,7 @@ #include "runtime/mem_pool.h" #include "runtime/mem_tracker.h" #include "util/debug_util.h" +#include "test_util/test_util.h" namespace doris { namespace segment_v2 { @@ -221,7 +222,7 @@ TEST_F(BinaryDictPageTest, TestEncodingRatio) { while (getline(infile, line)) { src_strings.emplace_back(line); } - for (int i = 0; i < 10000; ++i) { + for (int i = 0; i < LOOP_LESS_OR_MORE(100, 10000); ++i) { for (const auto& src_string : src_strings) { slices.push_back(src_string); } diff --git a/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp b/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp index 3487ae5f8b..a04c21eeba 100644 --- a/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp +++ b/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp @@ -31,6 +31,7 @@ #include "runtime/mem_pool.h" #include "runtime/mem_tracker.h" #include "util/file_utils.h" +#include "test_util/test_util.h" namespace doris { namespace segment_v2 { @@ -176,12 +177,13 @@ TEST_F(BitmapIndexTest, test_invert_2) { } TEST_F(BitmapIndexTest, test_multi_pages) { - size_t num_uint8_rows = 1024 * 1024; + size_t times = LOOP_LESS_OR_MORE(1, 1024); + size_t num_uint8_rows = times * 1024; int64_t* val = new int64_t[num_uint8_rows]; for (int i = 0; i < num_uint8_rows; ++i) { val[i] = random() + 10000; } - val[1024 * 510] = 2019; + val[times * 510] = 2019; std::string file_name = kTestDir + "/mul"; ColumnIndexMetaPB meta; diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp index 7b0c151347..057ccfb321 100644 --- a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp @@ -32,6 +32,7 @@ #include "runtime/mem_pool.h" #include "runtime/mem_tracker.h" #include "util/file_utils.h" +#include "test_util/test_util.h" using std::string; @@ -368,7 +369,7 @@ void test_array_nullable_data(Collection* src_data, uint8_t* src_is_null, int nu } TEST_F(ColumnReaderWriterTest, test_array_type) { - size_t num_list = 24 * 1024; + size_t num_list = LOOP_LESS_OR_MORE(1024, 24 * 1024); size_t num_item = num_list * 3; uint8_t* array_is_null = new uint8_t[BitmapSize(num_list)]; @@ -513,7 +514,7 @@ void test_read_default_value(string value, void* result) { } TEST_F(ColumnReaderWriterTest, test_nullable) { - size_t num_uint8_rows = 1024 * 1024; + size_t num_uint8_rows = LOOP_LESS_OR_MORE(1024, 1024 * 1024); uint8_t* is_null = new uint8_t[num_uint8_rows]; uint8_t* val = new uint8_t[num_uint8_rows]; for (int i = 0; i < num_uint8_rows; ++i) { @@ -571,7 +572,7 @@ TEST_F(ColumnReaderWriterTest, test_nullable) { } TEST_F(ColumnReaderWriterTest, test_types) { - size_t num_uint8_rows = 1024 * 1024; + size_t num_uint8_rows = LOOP_LESS_OR_MORE(1024, 1024 * 1024); uint8_t* is_null = new uint8_t[num_uint8_rows]; bool* bool_vals = new bool[num_uint8_rows]; diff --git a/be/test/olap/rowset/segment_v2/segment_test.cpp b/be/test/olap/rowset/segment_v2/segment_test.cpp index 2eb9cb2eff..f4971b6ce2 100644 --- a/be/test/olap/rowset/segment_v2/segment_test.cpp +++ b/be/test/olap/rowset/segment_v2/segment_test.cpp @@ -41,6 +41,7 @@ #include "runtime/mem_pool.h" #include "runtime/mem_tracker.h" #include "util/file_utils.h" +#include "test_util/test_util.h" namespace doris { namespace segment_v2 { @@ -626,7 +627,7 @@ TEST_F(SegmentReaderWriterTest, estimate_segment_size) { // 0, 1, 2, 3 // 10, 11, 12, 13 // 20, 21, 22, 23 - for (int i = 0; i < 1048576; ++i) { + for (int i = 0; i < LOOP_LESS_OR_MORE(1024, 1048576); ++i) { for (int j = 0; j < 4; ++j) { auto cell = row.cell(j); cell.set_not_null(); diff --git a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp index 981d74ff2e..41e51f31af 100644 --- a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp +++ b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp @@ -24,6 +24,7 @@ #include "util/pretty_printer.h" #include "util/runtime_profile.h" #include "util/threadpool.h" +#include "test_util/test_util.h" namespace doris { class UniqueRowsetIdGeneratorTest : public testing::Test {}; @@ -98,7 +99,7 @@ TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdTest) { TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdBenchmark) { const int kNumThreads = 8; - const int kIdPerThread = 1000000; + const int kIdPerThread = LOOP_LESS_OR_MORE(1000, 1000000); UniqueId backend_uid = UniqueId::gen_uid(); UniqueRowsetIdGenerator id_generator(backend_uid); @@ -113,7 +114,7 @@ TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdBenchmark) { { SCOPED_RAW_TIMER(&cost_ns); for (int i = 0; i < kNumThreads; i++) { - ASSERT_TRUE(pool->submit_func([&id_generator]() { + ASSERT_TRUE(pool->submit_func([&id_generator, kIdPerThread]() { for (int i = 0; i < kIdPerThread; ++i) { id_generator.next_id(); } diff --git a/be/test/olap/skiplist_test.cpp b/be/test/olap/skiplist_test.cpp index d948628426..0077a3e40b 100644 --- a/be/test/olap/skiplist_test.cpp +++ b/be/test/olap/skiplist_test.cpp @@ -30,6 +30,7 @@ #include "util/mutex.h" #include "util/priority_thread_pool.hpp" #include "util/random.h" +#include "test_util/test_util.h" namespace doris { @@ -395,7 +396,7 @@ static void concurrent_reader(void* arg) { static void run_concurrent(int run) { const int seed = random_seed + (run * 100); Random rnd(seed); - const int N = 1000; + const int N = LOOP_LESS_OR_MORE(10, 1000); const int kSize = 1000; PriorityThreadPool thread_pool(10, 100); for (int i = 0; i < N; i++) { @@ -413,20 +414,10 @@ static void run_concurrent(int run) { } } -TEST_F(SkipTest, Concurrent1) { - run_concurrent(1); -} -TEST_F(SkipTest, Concurrent2) { - run_concurrent(2); -} -TEST_F(SkipTest, Concurrent3) { - run_concurrent(3); -} -TEST_F(SkipTest, Concurrent4) { - run_concurrent(4); -} -TEST_F(SkipTest, Concurrent5) { - run_concurrent(5); +TEST_F(SkipTest, Concurrent) { + for (int i = 1; i < LOOP_LESS_OR_MORE(2, 6); ++i) { + run_concurrent(i); + } } } // namespace doris diff --git a/be/test/plugin/plugin_zip_test.cpp b/be/test/plugin/plugin_zip_test.cpp index f1ff52c620..473a6c712a 100644 --- a/be/test/plugin/plugin_zip_test.cpp +++ b/be/test/plugin/plugin_zip_test.cpp @@ -142,9 +142,9 @@ TEST_F(PluginZipTest, http_normal) { } TEST_F(PluginZipTest, already_install) { - // This test case will finish very soon, sleep 1 second to ensure that EvHttpServer worker has started + // This test case will finish very soon, sleep 100 us to ensure that EvHttpServer worker has started // before this unit test case finished, or there may cause an ASAN error. - sleep(1); + usleep(100); FileUtils::remove_all(_path + "/plugin_test/target"); PluginZip zip("http://127.0.0.1:29191/test.zip"); diff --git a/be/test/runtime/cache/partition_cache_test.cpp b/be/test/runtime/cache/partition_cache_test.cpp index cf0671139f..5ae9eefbc7 100644 --- a/be/test/runtime/cache/partition_cache_test.cpp +++ b/be/test/runtime/cache/partition_cache_test.cpp @@ -25,6 +25,7 @@ #include "runtime/cache/result_cache.h" #include "util/cpu_info.h" #include "util/logging.h" +#include "test_util/test_util.h" namespace doris { @@ -275,7 +276,7 @@ TEST_F(PartitionCacheTest, fetch_data_overdue) { TEST_F(PartitionCacheTest, prune_data) { init(1, 1); - init_batch_data(129, 1, 1024); // 16*1024*128=2M + init_batch_data(LOOP_LESS_OR_MORE(10, 129), 1, 1024); // 16*1024*128=2M ASSERT_LE(_cache->get_cache_size(), 1 * 1024 * 1024); //cache_size <= 1M clear(); } diff --git a/be/test/runtime/routine_load_task_executor_test.cpp b/be/test/runtime/routine_load_task_executor_test.cpp index 4db2caa70e..31e01a7d0b 100644 --- a/be/test/runtime/routine_load_task_executor_test.cpp +++ b/be/test/runtime/routine_load_task_executor_test.cpp @@ -97,25 +97,23 @@ TEST_F(RoutineLoadTaskExecutorTest, exec_task) { st = executor.submit_task(task); ASSERT_TRUE(st.ok()); - sleep(2); + usleep(200); k_info.brokers = "127.0.0.1:9092"; task.__set_kafka_load_info(k_info); st = executor.submit_task(task); ASSERT_TRUE(st.ok()); - sleep(2); + usleep(200); k_info.brokers = "192.0.0.2:9092"; task.__set_kafka_load_info(k_info); st = executor.submit_task(task); ASSERT_TRUE(st.ok()); - sleep(2); + usleep(200); k_info.brokers = "192.0.0.2:9092"; task.__set_kafka_load_info(k_info); st = executor.submit_task(task); ASSERT_TRUE(st.ok()); - - sleep(2); } } // namespace doris diff --git a/be/test/test_util/CMakeLists.txt b/be/test/test_util/CMakeLists.txt new file mode 100644 index 0000000000..b9f37ee03f --- /dev/null +++ b/be/test/test_util/CMakeLists.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# where to put generated libraries +set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/test/test_util") + +set(TEST_UTIL_FILES + test_util.cpp +) + +add_library(Test_util STATIC + ${TEST_UTIL_FILES} +) + +target_link_libraries(Test_util Gutil glog gflags) diff --git a/be/test/test_util/test_util.cpp b/be/test/test_util/test_util.cpp new file mode 100644 index 0000000000..5ee757270e --- /dev/null +++ b/be/test/test_util/test_util.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "test_util/test_util.h" + +#include + +#include "gutil/strings/substitute.h" + +using strings::Substitute; + +namespace doris { + +static const char* const kSlowTestsEnvVar = "DORIS_ALLOW_SLOW_TESTS"; + +bool AllowSlowTests() { return GetBooleanEnvironmentVariable(kSlowTestsEnvVar); } + +bool GetBooleanEnvironmentVariable(const char* env_var_name) { + const char* const e = getenv(env_var_name); + if ((e == nullptr) || + (strlen(e) == 0) || + (strcasecmp(e, "false") == 0) || + (strcasecmp(e, "0") == 0) || + (strcasecmp(e, "no") == 0)) { + return false; + } + if ((strcasecmp(e, "true") == 0) || + (strcasecmp(e, "1") == 0) || + (strcasecmp(e, "yes") == 0)) { + return true; + } + LOG(FATAL) << Substitute("$0: invalid value for environment variable $0", + e, env_var_name); + return false; // unreachable +} + +} // namespace doris diff --git a/be/test/test_util/test_util.h b/be/test/test_util/test_util.h new file mode 100644 index 0000000000..197ed23aee --- /dev/null +++ b/be/test/test_util/test_util.h @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace doris { + +#define LOOP_LESS_OR_MORE(less, more) (AllowSlowTests() ? less : more) + +// Get the value of an environment variable that has boolean semantics. +bool GetBooleanEnvironmentVariable(const char* env_var_name); + +// Returns true if slow tests are runtime-enabled. +bool AllowSlowTests(); + +} // namespace doris diff --git a/be/test/util/core_local_test.cpp b/be/test/util/core_local_test.cpp index d8bf5a9463..4268fc0da0 100644 --- a/be/test/util/core_local_test.cpp +++ b/be/test/util/core_local_test.cpp @@ -25,6 +25,7 @@ #include "common/logging.h" #include "time.h" #include "util/stopwatch.hpp" +#include "test_util/test_util.h" namespace doris { @@ -35,23 +36,24 @@ protected: ~CoreLocalTest() {} }; -void updater(CoreLocalValue* value, int64_t* used_ns) { - sleep(1); +void updater(int64_t loop, CoreLocalValue* value, int64_t* used_ns) { + usleep(100); MonotonicStopWatch stopwatch; stopwatch.start(); - for (int i = 0; i < 1000000L; ++i) { + for (int i = 0; i < loop; ++i) { __sync_fetch_and_add(value->access(), 1); } *used_ns = stopwatch.elapsed_time(); } TEST_F(CoreLocalTest, CoreLocalValue) { + int64_t loop = LOOP_LESS_OR_MORE(1000, 1000000L); CoreLocalValue value; std::vector used_ns; used_ns.resize(8); std::vector workers; for (int i = 0; i < 8; ++i) { - workers.emplace_back(updater, &value, &used_ns[i]); + workers.emplace_back(updater, loop, &value, &used_ns[i]); } int64_t sum_ns = 0; for (int i = 0; i < 8; ++i) { @@ -62,7 +64,7 @@ TEST_F(CoreLocalTest, CoreLocalValue) { for (int i = 0; i < value.size(); ++i) { sum += __sync_fetch_and_add(value.access_at_core(i), 0); } - ASSERT_EQ(8 * 1000000L, sum); + ASSERT_EQ(8 * loop, sum); LOG(INFO) << "time:" << sum_ns / sum << "ns/op"; } diff --git a/be/test/util/counter_cond_variable_test.cpp b/be/test/util/counter_cond_variable_test.cpp index 542f28c034..c988402348 100644 --- a/be/test/util/counter_cond_variable_test.cpp +++ b/be/test/util/counter_cond_variable_test.cpp @@ -46,7 +46,7 @@ void worker() { std::unique_lock lock(g_io_mu); std::cout << "worker " << i << std::endl; } - sleep(1); + usleep(100); g_cond.dec(); } } diff --git a/be/test/util/internal_queue_test.cpp b/be/test/util/internal_queue_test.cpp index 782d457ae3..03d2139aa0 100644 --- a/be/test/util/internal_queue_test.cpp +++ b/be/test/util/internal_queue_test.cpp @@ -25,6 +25,7 @@ #include "common/configbase.h" #include "util/logging.h" +#include "test_util/test_util.h" using std::vector; using boost::thread; @@ -222,7 +223,7 @@ TEST(InternalQueue, TestClear) { TEST(InternalQueue, TestSingleProducerSingleConsumer) { std::vector nodes; AtomicInt counter; - nodes.resize(1000000); + nodes.resize(LOOP_LESS_OR_MORE(100, 1000000)); std::vector results; InternalQueue queue; @@ -246,7 +247,7 @@ TEST(InternalQueue, TestSingleProducerSingleConsumer) { TEST(InternalQueue, TestMultiProducerMultiConsumer) { std::vector nodes; - nodes.resize(1000000); + nodes.resize(LOOP_LESS_OR_MORE(100, 1000000)); bool failed = false; for (int num_producers = 1; num_producers < 5; num_producers += 3) { diff --git a/be/test/util/new_metrics_test.cpp b/be/test/util/new_metrics_test.cpp index aac6bbf232..3c56f85530 100644 --- a/be/test/util/new_metrics_test.cpp +++ b/be/test/util/new_metrics_test.cpp @@ -24,6 +24,7 @@ #include "util/logging.h" #include "util/metrics.h" #include "util/stopwatch.hpp" +#include "test_util/test_util.h" namespace doris { @@ -69,11 +70,11 @@ TEST_F(MetricsTest, Counter) { } template -void mt_updater(T* counter, std::atomic* used_time) { +void mt_updater(int32_t loop, T* counter, std::atomic* used_time) { sleep(1); MonotonicStopWatch watch; watch.start(); - for (int i = 0; i < 1000000L; ++i) { + for (int i = 0; i < loop; ++i) { counter->increment(1); } uint64_t elapsed = watch.elapsed_time(); @@ -81,7 +82,8 @@ void mt_updater(T* counter, std::atomic* used_time) { } TEST_F(MetricsTest, CounterPerf) { - static const int kLoopCount = 100000000; + static const int kLoopCount = LOOP_LESS_OR_MORE(10, 100000000); + static const int kThreadLoopCount = LOOP_LESS_OR_MORE(1000, 1000000); // volatile int64_t { volatile int64_t sum = 0; @@ -126,14 +128,14 @@ TEST_F(MetricsTest, CounterPerf) { std::vector updaters; std::atomic used_time(0); for (int i = 0; i < 8; ++i) { - updaters.emplace_back(&mt_updater, &mt_counter, &used_time); + updaters.emplace_back(&mt_updater, kThreadLoopCount, &mt_counter, &used_time); } for (int i = 0; i < 8; ++i) { updaters[i].join(); } LOG(INFO) << "IntCounter multi-thread elapsed: " << used_time.load() - << "ns, ns/iter:" << used_time.load() / (8 * 1000000L); - ASSERT_EQ(8 * 1000000L, mt_counter.value()); + << "ns, ns/iter:" << used_time.load() / (8 * kThreadLoopCount); + ASSERT_EQ(8 * kThreadLoopCount, mt_counter.value()); } // multi-thread for IntAtomicCounter { @@ -141,14 +143,14 @@ TEST_F(MetricsTest, CounterPerf) { std::vector updaters; std::atomic used_time(0); for (int i = 0; i < 8; ++i) { - updaters.emplace_back(&mt_updater, &mt_counter, &used_time); + updaters.emplace_back(&mt_updater, kThreadLoopCount, &mt_counter, &used_time); } for (int i = 0; i < 8; ++i) { updaters[i].join(); } LOG(INFO) << "IntAtomicCounter multi-thread elapsed: " << used_time.load() - << "ns, ns/iter:" << used_time.load() / (8 * 1000000L); - ASSERT_EQ(8 * 1000000L, mt_counter.value()); + << "ns, ns/iter:" << used_time.load() / (8 * kThreadLoopCount); + ASSERT_EQ(8 * kThreadLoopCount, mt_counter.value()); } } diff --git a/be/test/util/rle_encoding_test.cpp b/be/test/util/rle_encoding_test.cpp index d4eec05319..f4495e2091 100644 --- a/be/test/util/rle_encoding_test.cpp +++ b/be/test/util/rle_encoding_test.cpp @@ -36,6 +36,7 @@ #include "util/debug_util.h" #include "util/faststring.h" #include "util/rle_encoding.h" +#include "test_util/test_util.h" using std::string; using std::vector; @@ -186,7 +187,7 @@ TEST_F(BitRle, Flush) { // Test some random bool sequences. TEST_F(BitRle, RandomBools) { int iters = 0; - const int n_iters = 20; + const int n_iters = LOOP_LESS_OR_MORE(5, 20); while (iters < n_iters) { srand(iters++); if (iters % 10000 == 0) LOG(ERROR) << "Seed: " << iters; @@ -209,12 +210,12 @@ TEST_F(BitRle, RandomBools) { // Test some random 64-bit sequences. TEST_F(BitRle, Random64Bit) { int iters = 0; - const int n_iters = 20; + const int n_iters = LOOP_LESS_OR_MORE(5, 20); while (iters < n_iters) { srand(iters++); if (iters % 10000 == 0) LOG(ERROR) << "Seed: " << iters; std::vector values; - for (int i = 0; i < 1000; ++i) { + for (int i = 0; i < LOOP_LESS_OR_MORE(10, 1000); ++i) { int group_size = rand() % 20 + 1; // NOLINT(*) uint64_t cur_value = (static_cast(rand()) << 32) + static_cast(rand()); diff --git a/be/test/util/tdigest_test.cpp b/be/test/util/tdigest_test.cpp index df51a1edde..84bc4e23be 100644 --- a/be/test/util/tdigest_test.cpp +++ b/be/test/util/tdigest_test.cpp @@ -21,6 +21,8 @@ #include +#include "test_util/test_util.h" + namespace doris { class TDigestTest : public ::testing::Test { @@ -87,7 +89,7 @@ TEST_F(TDigestTest, CrashAfterMerge) { TDigest digest(1000); std::uniform_real_distribution<> reals(0.0, 1.0); std::random_device gen; - for (int i = 0; i < 100000; i++) { + for (int i = 0; i < LOOP_LESS_OR_MORE(100, 100000); i++) { digest.add(reals(gen)); } digest.compress(); @@ -227,13 +229,13 @@ TEST_F(TDigestTest, Montonicity) { TDigest digest(1000); std::uniform_real_distribution<> reals(0.0, 1.0); std::random_device gen; - for (int i = 0; i < 100000; i++) { + for (int i = 0; i < LOOP_LESS_OR_MORE(10, 100000); i++) { digest.add(reals(gen)); } double lastQuantile = -1; double lastX = -1; - for (double z = 0; z <= 1; z += 1e-5) { + for (double z = 0; z <= 1; z += LOOP_LESS_OR_MORE(0.1, 1e-5)) { double x = digest.quantile(z); EXPECT_GE(x, lastX); lastX = x;