database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[UT] Speed up BE unit test (#5131)

Browse Source 
There are some long loops and sleeps in unit tests, it will cost a
very long time to run all unit tests, especially run in TSAN mode.
This patch speed up unit tests by shortening long loops and sleeps,
on my environment all unit tests finished in 1 minite. It's useful
to do basic functional unit tests.
You can switch to run in this mode by adding a new environment variable
'DORIS_ALLOW_SLOW_TESTS'. For example, you can set:
export DORIS_ALLOW_SLOW_TESTS=1
and also you can disable it by setting:
export DORIS_ALLOW_SLOW_TESTS=0
This commit is contained in:
Yingchun Lai
2020-12-27 22:19:56 +08:00
committed by GitHub
parent 85076b5678
commit 11c0aafa5c
28 changed files with 204 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/CMakeLists.txt
View File

@ -484,6 +484,7 @@ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
# Set libraries for test
set (TEST_LINK_LIBS ${DORIS_LINK_LIBS}
${WL_START_GROUP}
Test_util
gmock
gtest
${WL_END_GROUP}
@ -555,6 +556,7 @@ FUNCTION(ADD_BE_PLUGIN PLUGIN_NAME)
ENDFUNCTION()
if (${MAKE_TEST} STREQUAL "ON")
add_subdirectory(${TEST_DIR}/test_util)
add_subdirectory(${TEST_DIR}/agent)
add_subdirectory(${TEST_DIR}/common)
add_subdirectory(${TEST_DIR}/env)

4
be/src/olap/rowset/segment_v2/column_reader.cpp
View File

@ -248,7 +248,7 @@ Status ColumnReader::get_row_ranges_by_bloom_filter(CondColumn* cond_column,
int64_t idx = from;
int64_t to = row_ranges->get_range_to(i);
auto iter = _ordinal_index->seek_at_or_before(from);
while (idx < to) {
while (idx < to && iter.valid()) {
page_ids.insert(iter.page_index());
idx = iter.last_ordinal() + 1;
iter.next();
@ -444,7 +444,7 @@ Status FileColumnIterator::seek_to_first() {
Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) {
// if current page contains this row, we don't need to seek
if (_page == nullptr || !_page->contains(ord)) {
if (_page == nullptr || !_page->contains(ord) || !_page_iter.valid()) {
RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter));
RETURN_IF_ERROR(_read_data_page(_page_iter));
}

2
be/src/olap/tablet_manager.cpp
View File

@ -1028,7 +1028,9 @@ OLAPStatus TabletManager::start_trash_sweep() {
int32_t clean_num = 0;
do {
#ifndef BE_TEST
sleep(1);
#endif
clean_num = 0;
// should get write lock here, because it will remove tablet from shut_down_tablets
// and get tablet will access shut_down_tablets

3
be/test/exprs/string_functions_test.cpp
View File

@ -25,6 +25,7 @@
#include "exprs/anyval_util.h"
#include "testutil/function_utils.h"
#include "util/logging.h"
#include "test_util/test_util.h"
namespace doris {
@ -47,7 +48,7 @@ TEST_F(StringFunctionsTest, do_money_format_bench) {
doris_udf::FunctionContext* context = new doris_udf::FunctionContext();
StringVal expected =
AnyValUtil::from_string_temp(context, std::string("9,223,372,036,854,775,807.00"));
for (int i = 0; i < 10000000; i++) {
for (int i = 0; i < LOOP_LESS_OR_MORE(10, 10000000); i++) {
StringVal result =
StringFunctions::do_money_format(context, "922337203685477580700"); // cent
ASSERT_EQ(expected, result);

7
be/test/exprs/topn_function_test.cpp
View File

@ -20,6 +20,7 @@
#include "util/topn_counter.h"
#include "testutil/function_utils.h"
#include "zipf_distribution.h"
#include "test_util/test_util.h"
#include <gtest/gtest.h>
#include <unordered_map>
@ -28,7 +29,7 @@
namespace doris {
static const uint32_t TOPN_NUM = 100;
static const uint32_t TOTAL_RECORDS = 1000000;
static const uint32_t TOTAL_RECORDS = LOOP_LESS_OR_MORE(1000, 1000000);
static const uint32_t PARALLEL = 10;
std::string gen_random(const int len) {
@ -149,7 +150,9 @@ void test_topn_accuracy(FunctionContext* ctx, int key_space, int space_expand_ra
}
TEST_F(TopNFunctionsTest, topn_accuracy) {
std::vector<int> key_space_vec ({1000, 10000, 100000, 500000});
std::vector<int> small_key_space({100});
std::vector<int> large_key_space({1000, 10000, 100000, 500000});
std::vector<int> key_space_vec(LOOP_LESS_OR_MORE(small_key_space, large_key_space));
std::vector<int> space_expand_rate_vec({20, 50, 100});
std::vector<double> zipf_distribution_exponent_vec({0.5, 0.6, 1.0});
for (auto ket_space : key_space_vec) {

3
be/test/olap/file_helper_test.cpp
View File

@ -27,6 +27,7 @@
#include "gtest/gtest.h"
#include "olap/olap_define.h"
#include "util/logging.h"
#include "test_util/test_util.h"
#ifndef BE_TEST
#define BE_TEST
@ -93,7 +94,7 @@ TEST_F(FileHandlerTest, TestWrite) {
char* large_bytes2[(1 << 10)];
memset(large_bytes2, 0, sizeof(char) * ((1 << 12)));
int i = 1;
while (i < 1 << 17) {
while (i < LOOP_LESS_OR_MORE(1 << 10, 1 << 17)) {
file_handler.write(large_bytes2, ((1 << 12)));
++i;
}

3
be/test/olap/lru_cache_test.cpp
View File

@ -22,6 +22,7 @@
#include <vector>
#include "util/logging.h"
#include "test_util/test_util.h"
using namespace doris;
using namespace std;
@ -293,7 +294,7 @@ TEST_F(CacheTest, NewId) {
}
TEST_F(CacheTest, SimpleBenchmark) {
for (int i = 0; i < kCacheSize * 10000; i++) {
for (int i = 0; i < kCacheSize * LOOP_LESS_OR_MORE(10, 10000); i++) {
Insert(1000 + i, 2000 + i, 1);
ASSERT_EQ(2000 + i, Lookup(1000 + i));
}

5
be/test/olap/memory/column_delta_test.cpp
View File

@ -23,13 +23,14 @@
#include <vector>
#include "olap/memory/column.h"
#include "test_util/test_util.h"
namespace doris {
namespace memory {
TEST(ColumnDelta, Index) {
const int BaseSize = 256001;
const int NumUpdate = 10000;
const int BaseSize = LOOP_LESS_OR_MORE(2560, 256001);
const int NumUpdate = LOOP_LESS_OR_MORE(100, 10000);
srand(1);
scoped_refptr<ColumnDelta> delta(new ColumnDelta());
std::map<uint32_t, uint32_t> updates;

17
be/test/olap/memory/column_test.cpp
View File

@ -21,16 +21,17 @@
#include "olap/memory/column_reader.h"
#include "olap/memory/column_writer.h"
#include "test_util/test_util.h"
namespace doris {
namespace memory {
static const size_t InsertCount = LOOP_LESS_OR_MORE(1000, 1000000);
static const size_t UpdateTime = 70;
static const size_t UpdateCount = LOOP_LESS_OR_MORE(10, 10000);
template <class CppType, ColumnType CT>
struct ColumnTest {
static const size_t InsertCount = 1000000;
static const size_t UpdateTime = 70;
static const size_t UpdateCount = 10000;
static bool is_null(CppType v) { return ((int64_t)v) % 10 == 0; }
static void test_not_null() {
@ -46,7 +47,9 @@ struct ColumnTest {
scoped_refptr<Column> newc;
ASSERT_TRUE(writer->finalize(2).ok());
ASSERT_TRUE(writer->get_new_column(&newc).ok());
EXPECT_TRUE(c.get() != newc.get());
if (!AllowSlowTests()) {
EXPECT_TRUE(c.get() != newc.get());
}
std::unique_ptr<ColumnReader> readc;
ASSERT_TRUE(newc->create_reader(2, &readc).ok());
for (uint32_t i = 0; i < values.size(); i++) {
@ -73,7 +76,9 @@ struct ColumnTest {
scoped_refptr<Column> newc;
ASSERT_TRUE(writer->finalize(2).ok());
ASSERT_TRUE(writer->get_new_column(&newc).ok());
EXPECT_TRUE(c.get() != newc.get());
if (!AllowSlowTests()) {
EXPECT_TRUE(c.get() != newc.get());
}
std::unique_ptr<ColumnReader> readc;
ASSERT_TRUE(newc->create_reader(2, &readc).ok());
for (uint32_t i = 0; i < values.size(); i++) {

7
be/test/olap/memory/mem_tablet_test.cpp
View File

@ -22,6 +22,7 @@
#include "olap/memory/mem_tablet_scan.h"
#include "olap/memory/write_txn.h"
#include "olap/tablet_meta.h"
#include "test_util/test_util.h"
namespace doris {
namespace memory {
@ -34,9 +35,9 @@ struct TData {
};
TEST(MemTablet, writescan) {
const int num_insert = 2000000;
const int insert_per_write = 500000;
const int num_update = 10000;
const int num_insert = LOOP_LESS_OR_MORE(2000, 2000000);
const int insert_per_write = LOOP_LESS_OR_MORE(500, 500000);
const int num_update = LOOP_LESS_OR_MORE(10, 10000);
const int update_time = 3;
scoped_refptr<Schema> sc;
ASSERT_TRUE(Schema::create("id int,uv int,pv int,city tinyint null", &sc).ok());

3
be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp
View File

@ -31,6 +31,7 @@
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "util/debug_util.h"
#include "test_util/test_util.h"
namespace doris {
namespace segment_v2 {
@ -221,7 +222,7 @@ TEST_F(BinaryDictPageTest, TestEncodingRatio) {
while (getline(infile, line)) {
src_strings.emplace_back(line);
}
for (int i = 0; i < 10000; ++i) {
for (int i = 0; i < LOOP_LESS_OR_MORE(100, 10000); ++i) {
for (const auto& src_string : src_strings) {
slices.push_back(src_string);
}

6
be/test/olap/rowset/segment_v2/bitmap_index_test.cpp
View File

@ -31,6 +31,7 @@
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "util/file_utils.h"
#include "test_util/test_util.h"
namespace doris {
namespace segment_v2 {
@ -176,12 +177,13 @@ TEST_F(BitmapIndexTest, test_invert_2) {
}
TEST_F(BitmapIndexTest, test_multi_pages) {
size_t num_uint8_rows = 1024 * 1024;
size_t times = LOOP_LESS_OR_MORE(1, 1024);
size_t num_uint8_rows = times * 1024;
int64_t* val = new int64_t[num_uint8_rows];
for (int i = 0; i < num_uint8_rows; ++i) {
val[i] = random() + 10000;
}
val[1024 * 510] = 2019;
val[times * 510] = 2019;
std::string file_name = kTestDir + "/mul";
ColumnIndexMetaPB meta;

7
be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
View File

@ -32,6 +32,7 @@
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "util/file_utils.h"
#include "test_util/test_util.h"
using std::string;
@ -368,7 +369,7 @@ void test_array_nullable_data(Collection* src_data, uint8_t* src_is_null, int nu
}
TEST_F(ColumnReaderWriterTest, test_array_type) {
size_t num_list = 24 * 1024;
size_t num_list = LOOP_LESS_OR_MORE(1024, 24 * 1024);
size_t num_item = num_list * 3;
uint8_t* array_is_null = new uint8_t[BitmapSize(num_list)];
@ -513,7 +514,7 @@ void test_read_default_value(string value, void* result) {
}
TEST_F(ColumnReaderWriterTest, test_nullable) {
size_t num_uint8_rows = 1024 * 1024;
size_t num_uint8_rows = LOOP_LESS_OR_MORE(1024, 1024 * 1024);
uint8_t* is_null = new uint8_t[num_uint8_rows];
uint8_t* val = new uint8_t[num_uint8_rows];
for (int i = 0; i < num_uint8_rows; ++i) {
@ -571,7 +572,7 @@ TEST_F(ColumnReaderWriterTest, test_nullable) {
}
TEST_F(ColumnReaderWriterTest, test_types) {
size_t num_uint8_rows = 1024 * 1024;
size_t num_uint8_rows = LOOP_LESS_OR_MORE(1024, 1024 * 1024);
uint8_t* is_null = new uint8_t[num_uint8_rows];
bool* bool_vals = new bool[num_uint8_rows];

3
be/test/olap/rowset/segment_v2/segment_test.cpp
View File

@ -41,6 +41,7 @@
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "util/file_utils.h"
#include "test_util/test_util.h"
namespace doris {
namespace segment_v2 {
@ -626,7 +627,7 @@ TEST_F(SegmentReaderWriterTest, estimate_segment_size) {
// 0, 1, 2, 3
// 10, 11, 12, 13
// 20, 21, 22, 23
for (int i = 0; i < 1048576; ++i) {
for (int i = 0; i < LOOP_LESS_OR_MORE(1024, 1048576); ++i) {
for (int j = 0; j < 4; ++j) {
auto cell = row.cell(j);
cell.set_not_null();

5
be/test/olap/rowset/unique_rowset_id_generator_test.cpp
View File

@ -24,6 +24,7 @@
#include "util/pretty_printer.h"
#include "util/runtime_profile.h"
#include "util/threadpool.h"
#include "test_util/test_util.h"
namespace doris {
class UniqueRowsetIdGeneratorTest : public testing::Test {};
@ -98,7 +99,7 @@ TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdTest) {
TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdBenchmark) {
const int kNumThreads = 8;
const int kIdPerThread = 1000000;
const int kIdPerThread = LOOP_LESS_OR_MORE(1000, 1000000);
UniqueId backend_uid = UniqueId::gen_uid();
UniqueRowsetIdGenerator id_generator(backend_uid);
@ -113,7 +114,7 @@ TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdBenchmark) {
{
SCOPED_RAW_TIMER(&cost_ns);
for (int i = 0; i < kNumThreads; i++) {
ASSERT_TRUE(pool->submit_func([&id_generator]() {
ASSERT_TRUE(pool->submit_func([&id_generator, kIdPerThread]() {
for (int i = 0; i < kIdPerThread; ++i) {
id_generator.next_id();
}

21
be/test/olap/skiplist_test.cpp
View File

@ -30,6 +30,7 @@
#include "util/mutex.h"
#include "util/priority_thread_pool.hpp"
#include "util/random.h"
#include "test_util/test_util.h"
namespace doris {
@ -395,7 +396,7 @@ static void concurrent_reader(void* arg) {
static void run_concurrent(int run) {
const int seed = random_seed + (run * 100);
Random rnd(seed);
const int N = 1000;
const int N = LOOP_LESS_OR_MORE(10, 1000);
const int kSize = 1000;
PriorityThreadPool thread_pool(10, 100);
for (int i = 0; i < N; i++) {
@ -413,20 +414,10 @@ static void run_concurrent(int run) {
}
}
TEST_F(SkipTest, Concurrent1) {
run_concurrent(1);
}
TEST_F(SkipTest, Concurrent2) {
run_concurrent(2);
}
TEST_F(SkipTest, Concurrent3) {
run_concurrent(3);
}
TEST_F(SkipTest, Concurrent4) {
run_concurrent(4);
}
TEST_F(SkipTest, Concurrent5) {
run_concurrent(5);
TEST_F(SkipTest, Concurrent) {
for (int i = 1; i < LOOP_LESS_OR_MORE(2, 6); ++i) {
run_concurrent(i);
}
}
} // namespace doris

4
be/test/plugin/plugin_zip_test.cpp
View File

@ -142,9 +142,9 @@ TEST_F(PluginZipTest, http_normal) {
}
TEST_F(PluginZipTest, already_install) {
// This test case will finish very soon, sleep 1 second to ensure that EvHttpServer worker has started
// This test case will finish very soon, sleep 100 us to ensure that EvHttpServer worker has started
// before this unit test case finished, or there may cause an ASAN error.
sleep(1);
usleep(100);
FileUtils::remove_all(_path + "/plugin_test/target");
PluginZip zip("http://127.0.0.1:29191/test.zip");

3
be/test/runtime/cache/partition_cache_test.cpp vendored
View File

@ -25,6 +25,7 @@
#include "runtime/cache/result_cache.h"
#include "util/cpu_info.h"
#include "util/logging.h"
#include "test_util/test_util.h"
namespace doris {
@ -275,7 +276,7 @@ TEST_F(PartitionCacheTest, fetch_data_overdue) {
TEST_F(PartitionCacheTest, prune_data) {
init(1, 1);
init_batch_data(129, 1, 1024); // 16*1024*128=2M
init_batch_data(LOOP_LESS_OR_MORE(10, 129), 1, 1024); // 16*1024*128=2M
ASSERT_LE(_cache->get_cache_size(), 1 * 1024 * 1024); //cache_size <= 1M
clear();
}

8
be/test/runtime/routine_load_task_executor_test.cpp
View File

@ -97,25 +97,23 @@ TEST_F(RoutineLoadTaskExecutorTest, exec_task) {
st = executor.submit_task(task);
ASSERT_TRUE(st.ok());
sleep(2);
usleep(200);
k_info.brokers = "127.0.0.1:9092";
task.__set_kafka_load_info(k_info);
st = executor.submit_task(task);
ASSERT_TRUE(st.ok());
sleep(2);
usleep(200);
k_info.brokers = "192.0.0.2:9092";
task.__set_kafka_load_info(k_info);
st = executor.submit_task(task);
ASSERT_TRUE(st.ok());
sleep(2);
usleep(200);
k_info.brokers = "192.0.0.2:9092";
task.__set_kafka_load_info(k_info);
st = executor.submit_task(task);
ASSERT_TRUE(st.ok());
sleep(2);
}
} // namespace doris

29
be/test/test_util/CMakeLists.txt Normal file
View File

@ -0,0 +1,29 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# where to put generated libraries
set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/test/test_util")
set(TEST_UTIL_FILES
test_util.cpp
)
add_library(Test_util STATIC
${TEST_UTIL_FILES}
)
target_link_libraries(Test_util Gutil glog gflags)

51
be/test/test_util/test_util.cpp Normal file
View File

@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "test_util/test_util.h"
#include <strings.h>
#include "gutil/strings/substitute.h"
using strings::Substitute;
namespace doris {
static const char* const kSlowTestsEnvVar = "DORIS_ALLOW_SLOW_TESTS";
bool AllowSlowTests() { return GetBooleanEnvironmentVariable(kSlowTestsEnvVar); }
bool GetBooleanEnvironmentVariable(const char* env_var_name) {
const char* const e = getenv(env_var_name);
if ((e == nullptr) ||
(strlen(e) == 0) ||
(strcasecmp(e, "false") == 0) ||
(strcasecmp(e, "0") == 0) ||
(strcasecmp(e, "no") == 0)) {
return false;
}
if ((strcasecmp(e, "true") == 0) ||
(strcasecmp(e, "1") == 0) ||
(strcasecmp(e, "yes") == 0)) {
return true;
}
LOG(FATAL) << Substitute("$0: invalid value for environment variable $0",
e, env_var_name);
return false; // unreachable
}
} // namespace doris

30
be/test/test_util/test_util.h Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
namespace doris {
#define LOOP_LESS_OR_MORE(less, more) (AllowSlowTests() ? less : more)
// Get the value of an environment variable that has boolean semantics.
bool GetBooleanEnvironmentVariable(const char* env_var_name);
// Returns true if slow tests are runtime-enabled.
bool AllowSlowTests();
} // namespace doris

12
be/test/util/core_local_test.cpp
View File

@ -25,6 +25,7 @@
#include "common/logging.h"
#include "time.h"
#include "util/stopwatch.hpp"
#include "test_util/test_util.h"
namespace doris {
@ -35,23 +36,24 @@ protected:
~CoreLocalTest() {}
};
void updater(CoreLocalValue<int64_t>* value, int64_t* used_ns) {
sleep(1);
void updater(int64_t loop, CoreLocalValue<int64_t>* value, int64_t* used_ns) {
usleep(100);
MonotonicStopWatch stopwatch;
stopwatch.start();
for (int i = 0; i < 1000000L; ++i) {
for (int i = 0; i < loop; ++i) {
__sync_fetch_and_add(value->access(), 1);
}
*used_ns = stopwatch.elapsed_time();
}
TEST_F(CoreLocalTest, CoreLocalValue) {
int64_t loop = LOOP_LESS_OR_MORE(1000, 1000000L);
CoreLocalValue<int64_t> value;
std::vector<int64_t> used_ns;
used_ns.resize(8);
std::vector<std::thread> workers;
for (int i = 0; i < 8; ++i) {
workers.emplace_back(updater, &value, &used_ns[i]);
workers.emplace_back(updater, loop, &value, &used_ns[i]);
}
int64_t sum_ns = 0;
for (int i = 0; i < 8; ++i) {
@ -62,7 +64,7 @@ TEST_F(CoreLocalTest, CoreLocalValue) {
for (int i = 0; i < value.size(); ++i) {
sum += __sync_fetch_and_add(value.access_at_core(i), 0);
}
ASSERT_EQ(8 * 1000000L, sum);
ASSERT_EQ(8 * loop, sum);
LOG(INFO) << "time:" << sum_ns / sum << "ns/op";
}

2
be/test/util/counter_cond_variable_test.cpp
View File

@ -46,7 +46,7 @@ void worker() {
std::unique_lock<std::mutex> lock(g_io_mu);
std::cout << "worker " << i << std::endl;
}
sleep(1);
usleep(100);
g_cond.dec();
}
}

5
be/test/util/internal_queue_test.cpp
View File

@ -25,6 +25,7 @@
#include "common/configbase.h"
#include "util/logging.h"
#include "test_util/test_util.h"
using std::vector;
using boost::thread;
@ -222,7 +223,7 @@ TEST(InternalQueue, TestClear) {
TEST(InternalQueue, TestSingleProducerSingleConsumer) {
std::vector<IntNode> nodes;
AtomicInt<int32_t> counter;
nodes.resize(1000000);
nodes.resize(LOOP_LESS_OR_MORE(100, 1000000));
std::vector<int> results;
InternalQueue<IntNode> queue;
@ -246,7 +247,7 @@ TEST(InternalQueue, TestSingleProducerSingleConsumer) {
TEST(InternalQueue, TestMultiProducerMultiConsumer) {
std::vector<IntNode> nodes;
nodes.resize(1000000);
nodes.resize(LOOP_LESS_OR_MORE(100, 1000000));
bool failed = false;
for (int num_producers = 1; num_producers < 5; num_producers += 3) {

20
be/test/util/new_metrics_test.cpp
View File

@ -24,6 +24,7 @@
#include "util/logging.h"
#include "util/metrics.h"
#include "util/stopwatch.hpp"
#include "test_util/test_util.h"
namespace doris {
@ -69,11 +70,11 @@ TEST_F(MetricsTest, Counter) {
}
template <typename T>
void mt_updater(T* counter, std::atomic<uint64_t>* used_time) {
void mt_updater(int32_t loop, T* counter, std::atomic<uint64_t>* used_time) {
sleep(1);
MonotonicStopWatch watch;
watch.start();
for (int i = 0; i < 1000000L; ++i) {
for (int i = 0; i < loop; ++i) {
counter->increment(1);
}
uint64_t elapsed = watch.elapsed_time();
@ -81,7 +82,8 @@ void mt_updater(T* counter, std::atomic<uint64_t>* used_time) {
}
TEST_F(MetricsTest, CounterPerf) {
static const int kLoopCount = 100000000;
static const int kLoopCount = LOOP_LESS_OR_MORE(10, 100000000);
static const int kThreadLoopCount = LOOP_LESS_OR_MORE(1000, 1000000);
// volatile int64_t
{
volatile int64_t sum = 0;
@ -126,14 +128,14 @@ TEST_F(MetricsTest, CounterPerf) {
std::vector<std::thread> updaters;
std::atomic<uint64_t> used_time(0);
for (int i = 0; i < 8; ++i) {
updaters.emplace_back(&mt_updater<IntCounter>, &mt_counter, &used_time);
updaters.emplace_back(&mt_updater<IntCounter>, kThreadLoopCount, &mt_counter, &used_time);
}
for (int i = 0; i < 8; ++i) {
updaters[i].join();
}
LOG(INFO) << "IntCounter multi-thread elapsed: " << used_time.load()
<< "ns, ns/iter:" << used_time.load() / (8 * 1000000L);
ASSERT_EQ(8 * 1000000L, mt_counter.value());
<< "ns, ns/iter:" << used_time.load() / (8 * kThreadLoopCount);
ASSERT_EQ(8 * kThreadLoopCount, mt_counter.value());
}
// multi-thread for IntAtomicCounter
{
@ -141,14 +143,14 @@ TEST_F(MetricsTest, CounterPerf) {
std::vector<std::thread> updaters;
std::atomic<uint64_t> used_time(0);
for (int i = 0; i < 8; ++i) {
updaters.emplace_back(&mt_updater<IntAtomicCounter>, &mt_counter, &used_time);
updaters.emplace_back(&mt_updater<IntAtomicCounter>, kThreadLoopCount, &mt_counter, &used_time);
}
for (int i = 0; i < 8; ++i) {
updaters[i].join();
}
LOG(INFO) << "IntAtomicCounter multi-thread elapsed: " << used_time.load()
<< "ns, ns/iter:" << used_time.load() / (8 * 1000000L);
ASSERT_EQ(8 * 1000000L, mt_counter.value());
<< "ns, ns/iter:" << used_time.load() / (8 * kThreadLoopCount);
ASSERT_EQ(8 * kThreadLoopCount, mt_counter.value());
}
}

7
be/test/util/rle_encoding_test.cpp
View File

@ -36,6 +36,7 @@
#include "util/debug_util.h"
#include "util/faststring.h"
#include "util/rle_encoding.h"
#include "test_util/test_util.h"
using std::string;
using std::vector;
@ -186,7 +187,7 @@ TEST_F(BitRle, Flush) {
// Test some random bool sequences.
TEST_F(BitRle, RandomBools) {
int iters = 0;
const int n_iters = 20;
const int n_iters = LOOP_LESS_OR_MORE(5, 20);
while (iters < n_iters) {
srand(iters++);
if (iters % 10000 == 0) LOG(ERROR) << "Seed: " << iters;
@ -209,12 +210,12 @@ TEST_F(BitRle, RandomBools) {
// Test some random 64-bit sequences.
TEST_F(BitRle, Random64Bit) {
int iters = 0;
const int n_iters = 20;
const int n_iters = LOOP_LESS_OR_MORE(5, 20);
while (iters < n_iters) {
srand(iters++);
if (iters % 10000 == 0) LOG(ERROR) << "Seed: " << iters;
std::vector<uint64_t> values;
for (int i = 0; i < 1000; ++i) {
for (int i = 0; i < LOOP_LESS_OR_MORE(10, 1000); ++i) {
int group_size = rand() % 20 + 1; // NOLINT(*)
uint64_t cur_value =
(static_cast<uint64_t>(rand()) << 32) + static_cast<uint64_t>(rand());

8
be/test/util/tdigest_test.cpp
View File

@ -21,6 +21,8 @@
#include <random>
#include "test_util/test_util.h"
namespace doris {
class TDigestTest : public ::testing::Test {
@ -87,7 +89,7 @@ TEST_F(TDigestTest, CrashAfterMerge) {
TDigest digest(1000);
std::uniform_real_distribution<> reals(0.0, 1.0);
std::random_device gen;
for (int i = 0; i < 100000; i++) {
for (int i = 0; i < LOOP_LESS_OR_MORE(100, 100000); i++) {
digest.add(reals(gen));
}
digest.compress();
@ -227,13 +229,13 @@ TEST_F(TDigestTest, Montonicity) {
TDigest digest(1000);
std::uniform_real_distribution<> reals(0.0, 1.0);
std::random_device gen;
for (int i = 0; i < 100000; i++) {
for (int i = 0; i < LOOP_LESS_OR_MORE(10, 100000); i++) {
digest.add(reals(gen));
}
double lastQuantile = -1;
double lastX = -1;
for (double z = 0; z <= 1; z += 1e-5) {
for (double z = 0; z <= 1; z += LOOP_LESS_OR_MORE(0.1, 1e-5)) {
double x = digest.quantile(z);
EXPECT_GE(x, lastX);
lastX = x;