hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

289
unittest/sql/engine/set/test_hash_set_dump.cpp
View File

@ -31,15 +31,20 @@
#include "share/ob_cluster_version.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include "observer/ob_server.h"
namespace oceanbase {
namespace sql {
#include "share/ob_simple_mem_limit_getter.h"
namespace oceanbase
{
namespace sql
{
using namespace common;
using namespace share;
using namespace omt;
static ObSimpleMemLimitGetter getter;
class MockSqlExpression : public ObSqlExpression {
class MockSqlExpression : public ObSqlExpression
{
public:
MockSqlExpression(ObIAllocator& alloc) : ObSqlExpression(alloc)
MockSqlExpression(ObIAllocator &alloc): ObSqlExpression(alloc)
{
set_item_count(10);
}
@ -48,40 +53,42 @@ public:
#define TEST_SET_DUMP_GET_HASH_AREA_SIZE() (get_hash_area_size())
#define TEST_SET_DUMP_SET_HASH_AREA_SIZE(size) (set_hash_area_size(size))
class ObHashSetDumpTest : public blocksstable::TestDataFilePrepare, public ::testing::WithParamInterface<ObJoinType> {
class ObHashSetDumpTest:
public blocksstable::TestDataFilePrepare, public ::testing::WithParamInterface<ObJoinType>
{
public:
enum TestAlgo { UNION = 0, INTERSECT, EXCEPT };
enum TestAlgo
{
UNION = 0,
INTERSECT,
EXCEPT
};
protected:
struct SetPlan {
explicit SetPlan(ObIAllocator& alloc) : set_op_(nullptr), left_(alloc), right_(alloc), expr_(alloc)
{}
struct SetPlan
{
explicit SetPlan(ObIAllocator &alloc)
: exec_ctx_(alloc), set_op_(nullptr), left_(alloc), right_(alloc), expr_(alloc) {}
int setup_plan(ObSetOperator* set_op);
int setup_plan(ObSetOperator *set_op);
ObSQLSessionInfo session_;
ObPhysicalPlan plan_;
ObExecContext exec_ctx_;
ObSetOperator* set_op_;
ObSetOperator *set_op_;
SetDataGenerator left_;
SetDataGenerator right_;
MockSqlExpression expr_;
};
public:
ObHashSetDumpTest()
: blocksstable::TestDataFilePrepare("TestDiskIR", 8 << 20, 5000),
hash_union_(alloc_),
merge_union_(alloc_),
hash_intersect_(alloc_),
merge_intersect_(alloc_),
hash_except_(alloc_),
merge_except_(alloc_),
hash_set_op_(nullptr),
merge_set_op_(nullptr),
hash_plan_(alloc_),
merge_plan_(alloc_)
{}
: blocksstable::TestDataFilePrepare(&getter,"TestDiskIR", 8<<20, 5000),
hash_union_(alloc_), merge_union_(alloc_),
hash_intersect_(alloc_), merge_intersect_(alloc_),
hash_except_(alloc_), merge_except_(alloc_),
hash_set_op_(nullptr), merge_set_op_(nullptr),
hash_plan_(alloc_), merge_plan_(alloc_)
{
}
int init_tenant_mgr();
virtual void SetUp() override
@ -89,14 +96,13 @@ public:
ASSERT_EQ(OB_SUCCESS, init_tenant_mgr());
blocksstable::TestDataFilePrepare::SetUp();
ASSERT_EQ(OB_SUCCESS, blocksstable::ObTmpFileManager::get_instance().init());
ASSERT_EQ(OB_SUCCESS, blocksstable::ObTmpFileManager::get_instance().start());
CHUNK_MGR.set_limit(128L * 1024L * 1024L * 1024L);
GCONF.enable_sql_operator_dump.set_value("True");
uint64_t cluster_version = CLUSTER_VERSION_3000;
common::ObClusterVersion::get_instance().update_cluster_version(cluster_version);
EXPECT_EQ(cluster_version, common::ObClusterVersion::get_instance().get_cluster_version());
LOG_INFO(
"set cluster version", K(cluster_version), K(common::ObClusterVersion::get_instance().get_cluster_version()));
LOG_INFO("set cluster version", K(cluster_version),
K(common::ObClusterVersion::get_instance().get_cluster_version()));
OBSERVER.init_schema();
OBSERVER.init_tz_info_mgr();
}
@ -109,7 +115,6 @@ public:
void destroy_tenant_mgr()
{
ObTenantManager::get_instance().destroy();
}
int64_t get_hash_area_size()
@ -137,10 +142,10 @@ public:
// ASSERT_EQ(OB_SUCCESS, ret);
}
void setup_plan(SetPlan& plan, bool hash_algo, ObHashSetDumpTest::TestAlgo algo);
void setup_test(TestAlgo algo, int32_t string_size, int64_t left_row_count, bool left_reverse,
SetDataGenerator::IdxCntFunc left_func, int64_t right_row_count, bool right_reverse,
SetDataGenerator::IdxCntFunc right_func);
void setup_plan(SetPlan &plan, bool hash_algo, ObHashSetDumpTest::TestAlgo algo);
void setup_test(TestAlgo algo, int32_t string_size,
int64_t left_row_count, bool left_reverse, SetDataGenerator::IdxCntFunc left_func,
int64_t right_row_count, bool right_reverse, SetDataGenerator::IdxCntFunc right_func);
// iterate hash join result and verify result with merge join.
void run_test(int64_t print_row_cnt = 0);
@ -157,14 +162,14 @@ protected:
ObHashExcept hash_except_;
ObMergeExcept merge_except_;
ObSetOperator* hash_set_op_;
ObSetOperator* merge_set_op_;
ObSetOperator *hash_set_op_;
ObSetOperator *merge_set_op_;
SetPlan hash_plan_;
SetPlan merge_plan_;
};
int ObHashSetDumpTest::SetPlan::setup_plan(ObSetOperator* set_op)
int ObHashSetDumpTest::SetPlan::setup_plan(ObSetOperator *set_op)
{
int ret = OB_SUCCESS;
left_.set_id(0);
@ -198,47 +203,48 @@ int ObHashSetDumpTest::SetPlan::setup_plan(ObSetOperator* set_op)
return ret;
}
void ObHashSetDumpTest::setup_plan(SetPlan& plan, bool hash_algo, ObHashSetDumpTest::TestAlgo algo)
void ObHashSetDumpTest::setup_plan(SetPlan &plan, bool hash_algo, ObHashSetDumpTest::TestAlgo algo)
{
int ret = OB_SUCCESS;
ObMergeSetOperator* merge_op = nullptr;
ObHashSetOperator* hash_op = nullptr;
switch (algo) {
case UNION:
if (hash_algo) {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&hash_union_));
hash_set_op_ = &hash_union_;
hash_op = &hash_union_;
} else {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&merge_union_));
merge_op = &merge_union_;
merge_set_op_ = &merge_union_;
}
break;
case INTERSECT:
if (hash_algo) {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&hash_intersect_));
hash_set_op_ = &hash_intersect_;
hash_op = &hash_intersect_;
} else {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&merge_intersect_));
merge_op = &merge_intersect_;
merge_set_op_ = &merge_intersect_;
}
break;
case EXCEPT:
if (hash_algo) {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&hash_except_));
hash_set_op_ = &hash_except_;
hash_op = &hash_except_;
} else {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&merge_except_));
merge_op = &merge_except_;
merge_set_op_ = &merge_except_;
}
break;
default:
break;
ObMergeSetOperator *merge_op = nullptr;
ObHashSetOperator *hash_op = nullptr;
switch (algo)
{
case UNION:
if (hash_algo) {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&hash_union_));
hash_set_op_ = &hash_union_;
hash_op = &hash_union_;
} else {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&merge_union_));
merge_op = &merge_union_;
merge_set_op_ = &merge_union_;
}
break;
case INTERSECT:
if (hash_algo) {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&hash_intersect_));
hash_set_op_ = &hash_intersect_;
hash_op = &hash_intersect_;
} else {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&merge_intersect_));
merge_op = &merge_intersect_;
merge_set_op_ = &merge_intersect_;
}
break;
case EXCEPT:
if (hash_algo) {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&hash_except_));
hash_set_op_ = &hash_except_;
hash_op = &hash_except_;
} else {
ASSERT_EQ(OB_SUCCESS, plan.setup_plan(&merge_except_));
merge_op = &merge_except_;
merge_set_op_ = &merge_except_;
}
break;
default:
break;
}
if (nullptr != merge_op) {
ASSERT_EQ(OB_SUCCESS, merge_op->init(SetDataGenerator::CELL_CNT));
@ -262,13 +268,13 @@ void ObHashSetDumpTest::setup_plan(SetPlan& plan, bool hash_algo, ObHashSetDumpT
}
}
void ObHashSetDumpTest::setup_test(TestAlgo algo, int32_t string_size, int64_t left_row_count, bool left_reverse,
SetDataGenerator::IdxCntFunc left_func, int64_t right_row_count, bool right_reverse,
SetDataGenerator::IdxCntFunc right_func)
void ObHashSetDumpTest::setup_test(TestAlgo algo, int32_t string_size,
int64_t left_row_count, bool left_reverse, SetDataGenerator::IdxCntFunc left_func,
int64_t right_row_count, bool right_reverse, SetDataGenerator::IdxCntFunc right_func)
{
SetPlan* plans[] = {&hash_plan_, &merge_plan_};
SetPlan *plans[] = { &hash_plan_, &merge_plan_ };
for (int i = 0; i < 2; i++) {
auto& plan = *plans[i];
auto &plan = *plans[i];
setup_plan(plan, i == 0 ? true : false, algo);
plan.left_.row_cnt_ = left_row_count;
@ -290,12 +296,13 @@ void ObHashSetDumpTest::setup_test(TestAlgo algo, int32_t string_size, int64_t l
void ObHashSetDumpTest::run_test(int64_t print_row_cnt)
{
ObArenaAllocator alloc;
typedef ObArray<int64_t*> ResArray;
typedef ObArray<int64_t *> ResArray;
int64_t res_cell_cnt = SetDataGenerator::CELL_CNT * 2;
auto fun = [&](SetPlan& plan, ResArray& res) -> void {
auto fun = [&](SetPlan &plan, ResArray &res)->void
{
ASSERT_EQ(OB_SUCCESS, plan.set_op_->open(plan.exec_ctx_));
int ret = OB_SUCCESS;
const ObNewRow* row = NULL;
const ObNewRow *row = NULL;
int64_t cnt = 0;
while (OB_SUCC(ret)) {
if (OB_FAIL(plan.set_op_->get_next_row(plan.exec_ctx_, row))) {
@ -304,10 +311,10 @@ void ObHashSetDumpTest::run_test(int64_t print_row_cnt)
if (cnt < print_row_cnt) {
LOG_INFO("join res", K(*row));
}
auto r = static_cast<int64_t*>(alloc.alloc(sizeof(int64_t) * res_cell_cnt));
auto r = static_cast<int64_t *>(alloc.alloc(sizeof(int64_t) * res_cell_cnt));
ASSERT_TRUE(NULL != r);
for (int64_t i = 0; i < res_cell_cnt; i++) {
auto& c = row->cells_[i];
auto &c = row->cells_[i];
if (i < row->count_ && c.get_type() == ObIntType) {
r[i] = c.get_int();
} else {
@ -320,7 +327,8 @@ void ObHashSetDumpTest::run_test(int64_t print_row_cnt)
}
};
auto pfunc = [&](int64_t* r) {
auto pfunc = [&](int64_t *r)
{
ObSqlString s;
for (int64_t i = 0; i < res_cell_cnt; i++) {
s.append_fmt("%ld, ", r[i]);
@ -336,7 +344,8 @@ void ObHashSetDumpTest::run_test(int64_t print_row_cnt)
ASSERT_FALSE(HasFatalFailure());
ASSERT_EQ(hash_res.count(), merge_res.count());
auto sort_cmp = [&](int64_t* l, int64_t* r) {
auto sort_cmp = [&](int64_t *l, int64_t *r)
{
for (int64_t i = 0; i < res_cell_cnt; i++) {
if (l[i] != r[i]) {
return l[i] < r[i];
@ -347,7 +356,8 @@ void ObHashSetDumpTest::run_test(int64_t print_row_cnt)
std::sort(&hash_res.at(0), &hash_res.at(0) + hash_res.count(), sort_cmp);
std::sort(&merge_res.at(0), &merge_res.at(0) + merge_res.count(), sort_cmp);
for (int64_t i = 0; i < hash_res.count(); i++) {
if (sort_cmp(hash_res.at(i), merge_res.at(i)) || sort_cmp(merge_res.at(i), hash_res.at(i))) {
if (sort_cmp(hash_res.at(i), merge_res.at(i))
|| sort_cmp(merge_res.at(i), hash_res.at(i))) {
pfunc(hash_res.at(i));
pfunc(merge_res.at(i));
ASSERT_FALSE(true);
@ -364,7 +374,6 @@ void ObHashSetDumpTest::run_test(int64_t print_row_cnt)
int ObHashSetDumpTest::init_tenant_mgr()
{
int ret = OB_SUCCESS;
ObTenantManager& tm = ObTenantManager::get_instance();
ObAddr self;
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
@ -377,22 +386,19 @@ int ObHashSetDumpTest::init_tenant_mgr()
self.set_ip_addr("127.0.0.1", 8086);
ret = ObTenantConfigMgr::get_instance().add_tenant_config(tenant_id);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(tenant_id);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.set_tenant_mem_limit(tenant_id, 4L * 1024L * 1024L * 1024L, 8L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
ret = getter.add_tenant(tenant_id,
4L * 1024L * 1024L * 1024L,
8L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
const int64_t ulmt = 256LL << 30;
const int64_t llmt = 256LL << 30;
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
ret = getter.add_tenant(OB_SERVER_TENANT_ID,
ulmt,
llmt);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.set_tenant_mem_limit(OB_SERVER_TENANT_ID, ulmt, llmt);
EXPECT_EQ(OB_SUCCESS, ret);
lib::ObTenantCtxAllocator* ctx_allocator = lib::ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(
OB_SERVER_TENANT_ID, common::ObCtxIds::DEFAULT_CTX_ID);
lib::ObTenantCtxAllocator *ctx_allocator =
lib::ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(
OB_SERVER_TENANT_ID, common::ObCtxIds::DEFAULT_CTX_ID);
EXPECT_EQ(OB_SUCCESS, ret);
ret = ctx_allocator->set_limit(8L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
@ -402,15 +408,9 @@ int ObHashSetDumpTest::init_tenant_mgr()
TEST_F(ObHashSetDumpTest, test_single)
{
setup_test(
ObHashSetDumpTest::TestAlgo::UNION,
512,
1000,
false,
[](int64_t id, int64_t) { return id % 3 == 0 ? 1 : 0; },
1000,
false,
[](int64_t id, int64_t) { return id % 5 == 0 ? 1 : 0; });
setup_test(ObHashSetDumpTest::TestAlgo::UNION, 512,
1000, false, [](int64_t id, int64_t) { return id % 3 == 0 ? 1 : 0; },
1000, false, [](int64_t id, int64_t) { return id % 5 == 0 ? 1 : 0; });
ASSERT_FALSE(HasFatalFailure());
run_test();
ASSERT_FALSE(HasFatalFailure());
@ -418,15 +418,9 @@ TEST_F(ObHashSetDumpTest, test_single)
TEST_F(ObHashSetDumpTest, test_except)
{
setup_test(
ObHashSetDumpTest::TestAlgo::EXCEPT,
512,
1000,
false,
[](int64_t id, int64_t) { return id % 3 == 0 ? 1 : 0; },
1000,
false,
[](int64_t id, int64_t) { return id % 5 == 0 ? 1 : 0; });
setup_test(ObHashSetDumpTest::TestAlgo::EXCEPT, 512,
1000, false, [](int64_t id, int64_t) { return id % 3 == 0 ? 1 : 0; },
1000, false, [](int64_t id, int64_t) { return id % 5 == 0 ? 1 : 0; });
ASSERT_FALSE(HasFatalFailure());
run_test();
ASSERT_FALSE(HasFatalFailure());
@ -434,15 +428,9 @@ TEST_F(ObHashSetDumpTest, test_except)
TEST_F(ObHashSetDumpTest, test_intersect)
{
setup_test(
ObHashSetDumpTest::TestAlgo::INTERSECT,
512,
1000,
false,
[](int64_t id, int64_t) { return id % 3 == 0 ? 1 : 0; },
1000,
false,
[](int64_t id, int64_t) { return id % 5 == 0 ? 1 : 0; });
setup_test(ObHashSetDumpTest::TestAlgo::INTERSECT, 512,
1000, false, [](int64_t id, int64_t) { return id % 3 == 0 ? 1 : 0; },
1000, false, [](int64_t id, int64_t) { return id % 5 == 0 ? 1 : 0; });
ASSERT_FALSE(HasFatalFailure());
run_test();
ASSERT_FALSE(HasFatalFailure());
@ -450,15 +438,9 @@ TEST_F(ObHashSetDumpTest, test_intersect)
TEST_F(ObHashSetDumpTest, test_dump_union)
{
setup_test(
ObHashSetDumpTest::TestAlgo::UNION,
2000,
200000 * 3,
false,
[](int64_t id, int64_t) { return id % 3 == 0 ? 2 : 0; },
200000 * 5,
false,
[](int64_t id, int64_t) { return id % 5 == 0 ? 2 : 0; });
setup_test(ObHashSetDumpTest::TestAlgo::UNION, 2000,
200000 * 3, false, [](int64_t id, int64_t) { return id % 3 == 0 ? 2 : 0; },
200000 * 5, false, [](int64_t id, int64_t) { return id % 5 == 0 ? 2 : 0; });
ASSERT_FALSE(HasFatalFailure());
run_test();
ASSERT_FALSE(HasFatalFailure());
@ -466,15 +448,9 @@ TEST_F(ObHashSetDumpTest, test_dump_union)
TEST_F(ObHashSetDumpTest, test_dump_intersect)
{
setup_test(
ObHashSetDumpTest::TestAlgo::INTERSECT,
2000,
200000 * 3,
false,
[](int64_t id, int64_t) { return id % 3 == 0 ? 2 : 0; },
200000 * 5,
false,
[](int64_t id, int64_t) { return id % 5 == 0 ? 2 : 0; });
setup_test(ObHashSetDumpTest::TestAlgo::INTERSECT, 2000,
200000 * 3, false, [](int64_t id, int64_t) { return id % 3 == 0 ? 2 : 0; },
200000 * 5, false, [](int64_t id, int64_t) { return id % 5 == 0 ? 2 : 0; });
ASSERT_FALSE(HasFatalFailure());
run_test();
ASSERT_FALSE(HasFatalFailure());
@ -482,20 +458,15 @@ TEST_F(ObHashSetDumpTest, test_dump_intersect)
TEST_F(ObHashSetDumpTest, test_dump_except)
{
setup_test(
ObHashSetDumpTest::TestAlgo::EXCEPT,
2000,
200000 * 3,
false,
[](int64_t id, int64_t) { return id % 3 == 0 ? 2 : 0; },
200000 * 5,
false,
[](int64_t id, int64_t) { return id % 5 == 0 ? 2 : 0; });
setup_test(ObHashSetDumpTest::TestAlgo::EXCEPT, 2000,
200000 * 3, false, [](int64_t id, int64_t) { return id % 3 == 0 ? 2 : 0; },
200000 * 5, false, [](int64_t id, int64_t) { return id % 5 == 0 ? 2 : 0; });
ASSERT_FALSE(HasFatalFailure());
run_test();
ASSERT_FALSE(HasFatalFailure());
}
// farm时间比较长，暂时skip掉
// TEST_F(ObHashSetDumpTest, Size20M_union)
// {
// int64_t hash_mem = 0;
@ -748,14 +719,14 @@ TEST_F(ObHashSetDumpTest, test_dump_except)
// ASSERT_EQ((100 * 1024 * 1024), hash_mem);
// }
} // namespace sql
} // namespace oceanbase
} // end sql
} // end oceanbase
int main(int argc, char** argv)
int main(int argc, char **argv)
{
oceanbase::sql::init_sql_factories();
OB_LOGGER.set_log_level("INFO");
::testing::InitGoogleTest(&argc, argv);
::testing::InitGoogleTest(&argc,argv);
int ret = RUN_ALL_TESTS();
OB_LOGGER.disable();
return ret;