From 47a3d5000b11d69a6de89c8e99332a2aca08ebdd Mon Sep 17 00:00:00 2001 From: lichaoyong Date: Fri, 20 Mar 2020 20:37:50 +0800 Subject: [PATCH] [UnitTest] Fix unit test bug in BetaRowset and PageCacheTest (#3157) 1. BlockManager has been added into StorageEngine. So StorageEngine should be initialized when starting BetaRowset unit test. 2. Cache should not use the same buf to store value, otherwise the address will be freed twice and crash. --- be/test/olap/page_cache_test.cpp | 44 +++++++++---------- be/test/olap/rowset/beta_rowset_test.cpp | 35 +++++++++++---- be/test/olap/rowset/rowset_converter_test.cpp | 10 +++++ 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/be/test/olap/page_cache_test.cpp b/be/test/olap/page_cache_test.cpp index 381a997e79..137dbd6ae4 100644 --- a/be/test/olap/page_cache_test.cpp +++ b/be/test/olap/page_cache_test.cpp @@ -34,37 +34,33 @@ TEST(StoragePageCacheTest, normal) { StoragePageCache::CacheKey key("abc", 0); StoragePageCache::CacheKey memory_key("mem", 0); - char* buf = new char[1024]; - // insert normal page { + // insert normal page + char* buf = new char[1024]; PageCacheHandle handle; Slice data(buf, 1024); cache.insert(key, data, &handle, false); ASSERT_EQ(handle.data().data, buf); + + auto found = cache.lookup(key, &handle); + ASSERT_TRUE(found); + ASSERT_EQ(buf, handle.data().data); } - // insert in_memory page + { + // insert in_memory page + char* buf = new char[1024]; PageCacheHandle handle; Slice data(buf, 1024); cache.insert(memory_key, data, &handle, true); ASSERT_EQ(handle.data().data, buf); - } - // cache hit - { - PageCacheHandle handle; - auto found = cache.lookup(key, &handle); + + auto found = cache.lookup(memory_key, &handle); ASSERT_TRUE(found); - ASSERT_EQ(buf, handle.data().data); - } - // cache miss - { - PageCacheHandle handle; - StoragePageCache::CacheKey miss_key("abc", 1); - auto found = cache.lookup(miss_key, &handle); - ASSERT_FALSE(found); } + // put too many page to eliminate first page for (int i = 0; i < 10 * kNumShards; ++i) { StoragePageCache::CacheKey key("bcd", i); @@ -72,18 +68,22 @@ TEST(StoragePageCacheTest, normal) { Slice data(new char[1024], 1024); cache.insert(key, data, &handle, false); } + + // cache miss + { + PageCacheHandle handle; + StoragePageCache::CacheKey miss_key("abc", 1); + auto found = cache.lookup(miss_key, &handle); + ASSERT_FALSE(found); + } + // cache miss for eliminated key { PageCacheHandle handle; auto found = cache.lookup(key, &handle); ASSERT_FALSE(found); } - // cache hit for in memory key - { - PageCacheHandle handle; - auto found = cache.lookup(memory_key, &handle); - ASSERT_TRUE(found); - } + } } // namespace doris diff --git a/be/test/olap/rowset/beta_rowset_test.cpp b/be/test/olap/rowset/beta_rowset_test.cpp index e88f638b0f..026eccc219 100644 --- a/be/test/olap/rowset/beta_rowset_test.cpp +++ b/be/test/olap/rowset/beta_rowset_test.cpp @@ -31,6 +31,7 @@ #include "olap/tablet_schema.h" #include "olap/utils.h" #include "olap/comparison_predicate.h" +#include "runtime/exec_env.h" #include "runtime/mem_tracker.h" #include "runtime/mem_pool.h" #include "util/slice.h" @@ -40,20 +41,38 @@ using std::string; namespace doris { +static const uint32_t MAX_PATH_LEN = 1024; +StorageEngine* k_engine = nullptr; + class BetaRowsetTest : public testing::Test { protected: - const string kRowsetDir = "./ut_dir/beta_rowset_test"; OlapReaderStatistics _stats; void SetUp() override { - if (FileUtils::check_exist(kRowsetDir)) { - ASSERT_TRUE(FileUtils::remove_all(kRowsetDir).ok()); - } - ASSERT_TRUE(FileUtils::create_dir(kRowsetDir).ok()); + char buffer[MAX_PATH_LEN]; + getcwd(buffer, MAX_PATH_LEN); + config::storage_root_path = std::string(buffer) + "/data_test"; + + ASSERT_TRUE(FileUtils::remove_all(config::storage_root_path).ok()); + ASSERT_TRUE(FileUtils::create_dir(config::storage_root_path).ok()); + + std::vector paths; + paths.emplace_back(config::storage_root_path, -1); + + doris::EngineOptions options; + options.store_paths = paths; + doris::StorageEngine::open(options, &k_engine); + + ExecEnv* exec_env = doris::ExecEnv::GetInstance(); + exec_env->set_storage_engine(k_engine); + + const string rowset_dir = "./data_test/data/beta_rowset_test"; + ASSERT_TRUE(FileUtils::create_dir(rowset_dir).ok()); } + void TearDown() override { - if (FileUtils::check_exist(kRowsetDir)) { - ASSERT_TRUE(FileUtils::remove_all(kRowsetDir).ok()); + if (FileUtils::check_exist(config::storage_root_path)) { + ASSERT_TRUE(FileUtils::remove_all(config::storage_root_path).ok()); } } @@ -109,7 +128,7 @@ protected: rowset_writer_context->tablet_schema_hash = 1111; rowset_writer_context->partition_id = 10; rowset_writer_context->rowset_type = BETA_ROWSET; - rowset_writer_context->rowset_path_prefix = kRowsetDir; + rowset_writer_context->rowset_path_prefix = "./data_test/data/beta_rowset_test"; rowset_writer_context->rowset_state = VISIBLE; rowset_writer_context->tablet_schema = tablet_schema; rowset_writer_context->version.first = 10; diff --git a/be/test/olap/rowset/rowset_converter_test.cpp b/be/test/olap/rowset/rowset_converter_test.cpp index 10877d675c..40360a0d7c 100644 --- a/be/test/olap/rowset/rowset_converter_test.cpp +++ b/be/test/olap/rowset/rowset_converter_test.cpp @@ -36,6 +36,7 @@ #include "olap/data_dir.h" #include "olap/storage_engine.h" #include "olap/olap_cond.h" +#include "runtime/exec_env.h" #ifndef BE_TEST #define BE_TEST @@ -49,6 +50,7 @@ using std::string; namespace doris { static const uint32_t MAX_PATH_LEN = 1024; +StorageEngine* k_engine = nullptr; void create_rowset_writer_context(TabletSchema* tablet_schema, RowsetTypePB dst_type, RowsetWriterContext* rowset_writer_context) { @@ -156,6 +158,14 @@ public: ASSERT_TRUE(FileUtils::create_dir(config::storage_root_path).ok()); std::vector paths; paths.emplace_back(config::storage_root_path, -1); + + doris::EngineOptions options; + options.store_paths = paths; + doris::StorageEngine::open(options, &k_engine); + + ExecEnv* exec_env = doris::ExecEnv::GetInstance(); + exec_env->set_storage_engine(k_engine); + std::string data_path = config::storage_root_path + "/data"; ASSERT_TRUE(FileUtils::create_dir(data_path).ok()); std::string shard_path = data_path + "/0";