hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
unittest/sql/engine/sort/CMakeLists.txt
View File

@ -1,10 +1,10 @@
function(sort_unittest case)
ob_unittest(${ARGV})
target_sources(${case} PRIVATE ob_fake_table.h ob_fake_table.cpp)
endfunction()
sort_unittest(ob_specific_columns_sort_test)
sort_unittest(ob_in_memory_sort_test)
sort_unittest(ob_sort_test)
sort_unittest(ob_merge_sort_test)
sort_unittest(test_sort_impl)
#function(sort_unittest case)
# ob_unittest(${ARGV})
# target_sources(${case} PRIVATE ob_fake_table.h ob_fake_table.cpp)
#endfunction()
#
#sort_unittest(ob_specific_columns_sort_test)
#sort_unittest(ob_in_memory_sort_test)
#sort_unittest(ob_sort_test)
#sort_unittest(ob_merge_sort_test)
#sort_unittest(test_sort_impl)

76
unittest/sql/engine/sort/ob_fake_table.cpp
View File

@ -14,28 +14,33 @@
#include "lib/utility/utility.h"
#include "lib/allocator/ob_malloc.h"
#include "sql/engine/ob_exec_context.h"
// using namespace oceanbase::sql;
// using namespace oceanbase::sql::test;
// using namespace oceanbase::common;
//using namespace oceanbase::sql;
//using namespace oceanbase::sql::test;
//using namespace oceanbase::common;
namespace oceanbase {
namespace oceanbase
{
using namespace common;
namespace sql {
namespace test {
ObFakeTable::ObFakeTable() : ObPhyOperator(alloc_), row_count_(0), n_segments_local_merge_sort_(0)
namespace sql
{
namespace test
{
ObFakeTable::ObFakeTable()
: ObPhyOperator(alloc_), row_count_(0), n_segments_local_merge_sort_(0)
{
set_column_count(COLUMN_COUNT);
}
ObFakeTable::~ObFakeTable()
{}
{
}
void ObFakeTable::set_row_count(const int64_t count)
{
row_count_ = count;
}
ObPhyOperator* ObFakeTable::get_child(int32_t child_idx) const
ObPhyOperator *ObFakeTable::get_child(int32_t child_idx) const
{
UNUSED(child_idx);
return NULL;
@ -46,14 +51,14 @@ int32_t ObFakeTable::get_child_num() const
return 0;
}
int ObFakeTable::set_child(int32_t child_idx, ObPhyOperator& child_operator)
int ObFakeTable::set_child(int32_t child_idx, ObPhyOperator &child_operator)
{
UNUSED(child_idx);
UNUSED(child_operator);
return OB_SUCCESS;
}
int ObFakeTable::inner_open(ObExecContext& exec_ctx) const
int ObFakeTable::inner_open(ObExecContext &exec_ctx) const
{
int ret = OB_SUCCESS;
if (OB_SUCCESS != (ret = init_op_ctx(exec_ctx))) {
@ -62,10 +67,10 @@ int ObFakeTable::inner_open(ObExecContext& exec_ctx) const
return ret;
}
int ObFakeTable::inner_create_operator_ctx(ObExecContext& exec_ctx, ObPhyOperatorCtx*& op_ctx) const
int ObFakeTable::inner_create_operator_ctx(ObExecContext &exec_ctx, ObPhyOperatorCtx *&op_ctx) const
{
int ret = OB_SUCCESS;
ObFakeTableCtx* table_ctx = NULL;
ObFakeTableCtx *table_ctx = NULL;
if (OB_FAIL(CREATE_PHY_OPERATOR_CTX(ObFakeTableCtx, exec_ctx, get_id(), get_type(), table_ctx))) {
_OB_LOG(WARN, "failed to create fake table ctx, ret=%d", ret);
} else {
@ -74,10 +79,10 @@ int ObFakeTable::inner_create_operator_ctx(ObExecContext& exec_ctx, ObPhyOperato
return ret;
}
int ObFakeTable::init_op_ctx(ObExecContext& ctx) const
int ObFakeTable::init_op_ctx(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObPhyOperatorCtx* op_ctx = NULL;
ObPhyOperatorCtx *op_ctx = NULL;
if (OB_SUCCESS != (ret = inner_create_operator_ctx(ctx, op_ctx))) {
_OB_LOG(WARN, "create operator context failed, ret=%d", ret);
} else if (OB_SUCCESS != (ret = op_ctx->create_cur_row(get_column_count(), projector_, projector_size_))) {
@ -86,10 +91,11 @@ int ObFakeTable::init_op_ctx(ObExecContext& ctx) const
return ret;
}
int ObFakeTable::inner_get_next_row(ObExecContext& exec_ctx, const ObNewRow*& row) const
int ObFakeTable::inner_get_next_row(ObExecContext &exec_ctx, const ObNewRow *&row) const
{
int ret = OB_SUCCESS;
ObFakeTableCtx* table_ctx = NULL;
ObFakeTableCtx *table_ctx = NULL;
if (NULL == (table_ctx = GET_PHY_OPERATOR_CTX(ObFakeTableCtx, exec_ctx, get_id()))) {
_OB_LOG(WARN, "failed to get table ctx, ret=%d", ret);
} else if (table_ctx->get_count_ < row_count_) {
@ -124,7 +130,7 @@ int ObFakeTable::inner_get_next_row(ObExecContext& exec_ctx, const ObNewRow*& ro
-----------------------------------------------------------------------------------------------------
rand str | row_idx | row_idx%2 | row_idx%3 | row_idx/2 | row_idx/3 | c1+c2 |c3+c4 |
***************************************************************************************************/
int ObFakeTable::cons_cur_row(ObFakeTableCtx* table_ctx, const int64_t row_idx) const
int ObFakeTable::cons_cur_row(ObFakeTableCtx *table_ctx, const int64_t row_idx) const
{
int ret = OB_SUCCESS;
int64_t c2_val = 0, c3_val = 0, c4_val = 0, c5_val = 0, c6_val = 0;
@ -179,9 +185,9 @@ int ObFakeTable::cons_cur_row(ObFakeTableCtx* table_ctx, const int64_t row_idx)
return ret;
}
#define A_Z_CNT (26)
#define GER_CNT (4) // ä, ö, ü, ß.
int ObFakeTable::cons_varchar_cell(ObFakeTableCtx* table_ctx, ObObj& cell) const
#define A_Z_CNT (26)
#define GER_CNT (4) // ä, ö, ü, ß.
int ObFakeTable::cons_varchar_cell(ObFakeTableCtx *table_ctx, ObObj &cell) const
{
int ret = OB_SUCCESS;
int charnum = rand_int(ObFakeTableCtx::VARCHAR_CELL_BUF_SIZE - 5) + 4;
@ -194,23 +200,19 @@ int ObFakeTable::cons_varchar_cell(ObFakeTableCtx* table_ctx, ObObj& cell) const
switch (rand) {
case 26:
case 27:
table_ctx->buf_[i] = (char)(0xA4);
break; // ä
table_ctx->buf_[i] = (char)(0xA4); break; // ä
case 28:
case 29:
table_ctx->buf_[i] = (char)(0xB6);
break; // ö
table_ctx->buf_[i] = (char)(0xB6); break; // ö
case 30:
case 31:
table_ctx->buf_[i] = (char)(0xBC);
break; // ü
table_ctx->buf_[i] = (char)(0xBC); break; // ü
case 32:
case 33:
table_ctx->buf_[i] = (char)(0x9F);
break; // ß
default:
break;
table_ctx->buf_[i] = (char)(0x9F); break; // ß
default: break;
}
}
}
table_ctx->buf_[charnum] = 0;
@ -218,11 +220,11 @@ int ObFakeTable::cons_varchar_cell(ObFakeTableCtx* table_ctx, ObObj& cell) const
varchar.assign_ptr(table_ctx->buf_, charnum);
cell.set_varchar(varchar);
cell.set_collation_type(common::CS_TYPE_UTF8MB4_BIN);
// printf("**** rand varchar: %s\n", table_ctx->buf_);
// printf("**** rand varchar: %s\n", table_ctx->buf_);
return ret;
}
int ObFakeTable::cons_random_int_cell(ObObj& cell) const
int ObFakeTable::cons_random_int_cell(ObObj &cell) const
{
int ret = OB_SUCCESS;
cell.set_int(rand());
@ -232,7 +234,7 @@ int ObFakeTable::cons_random_int_cell(ObObj& cell) const
inline int ObFakeTable::rand_int(int max) const
{
double fmax = max;
int j = (int)(fmax * (rand() / (RAND_MAX + 1.0)));
int j = (int) (fmax * (rand() / (RAND_MAX + 1.0)));
return j;
}
@ -242,6 +244,6 @@ int64_t ObFakeTable::to_string(char* buf, const int64_t buf_len) const
databuff_printf(buf, buf_len, pos, "FakeTableForTesting\n");
return pos;
}
} // end namespace test
} // end namespace sql
} // end namespace oceanbase
} // end namespace test
} // end namespace sql
} // end namespace oceanbase

80
unittest/sql/engine/sort/ob_fake_table.h
View File

@ -15,12 +15,16 @@
#include "sql/engine/ob_phy_operator.h"
#include "lib/container/ob_array.h"
#include "lib/allocator/page_arena.h"
namespace oceanbase {
namespace sql {
namespace test {
namespace oceanbase
{
namespace sql
{
namespace test
{
static common::ObArenaAllocator alloc_;
// fake table to feed input for testing
class ObFakeTable : public ObPhyOperator {
class ObFakeTable: public ObPhyOperator
{
public:
const static int64_t COL0_RAND_STR = 0;
const static int64_t COL1_ROW_ID = 1;
@ -38,26 +42,21 @@ public:
const static int64_t COL13_RAND_INT = 13;
const static int64_t COL14_RAND_INT = 14;
const static int64_t COL15_RAND_INT = 15;
protected:
class ObFakeTableCtx : public ObPhyOperatorCtx {
class ObFakeTableCtx: public ObPhyOperatorCtx
{
friend class ObFakeTable;
public:
ObFakeTableCtx(ObExecContext& ctx) : ObPhyOperatorCtx(ctx), get_count_(0)
ObFakeTableCtx(ObExecContext &ctx)
: ObPhyOperatorCtx(ctx),
get_count_(0)
{
buf_[0] = '\0';
}
virtual ~ObFakeTableCtx()
{}
virtual void destroy()
{
return ObPhyOperatorCtx::destroy_base();
}
virtual ~ObFakeTableCtx() {}
virtual void destroy() { return ObPhyOperatorCtx::destroy_base(); }
private:
static const int VARCHAR_CELL_BUF_SIZE = 32;
private:
int64_t get_count_;
char buf_[VARCHAR_CELL_BUF_SIZE];
@ -66,52 +65,41 @@ protected:
public:
ObFakeTable();
virtual ~ObFakeTable();
virtual ObPhyOperatorType get_type() const
{
return PHY_INVALID;
}
virtual ObPhyOperatorType get_type() const { return PHY_INVALID; }
void set_row_count(const int64_t count);
void reset()
{}
void reuse()
{}
void reset() {}
void reuse() {}
virtual ObPhyOperator* get_child(int32_t child_idx) const;
virtual ObPhyOperator *get_child(int32_t child_idx) const;
virtual int32_t get_child_num() const;
virtual int set_child(int32_t child_idx, ObPhyOperator& child_operator);
virtual int inner_open(ObExecContext& exec_ctx) const;
virtual int set_child(int32_t child_idx, ObPhyOperator &child_operator);
virtual int inner_open(ObExecContext &exec_ctx) const;
virtual int64_t to_string(char* buf, const int64_t buf_len) const;
virtual void set_segments_local_merge_sort(int64_t n_segments_local_merge_sort)
{
n_segments_local_merge_sort_ = n_segments_local_merge_sort;
}
virtual void set_segments_local_merge_sort(int64_t n_segments_local_merge_sort) { n_segments_local_merge_sort_ = n_segments_local_merge_sort; }
protected:
virtual int inner_get_next_row(ObExecContext& exec_ctx, const common::ObNewRow*& row) const;
virtual int init_op_ctx(ObExecContext& ctx) const;
virtual int inner_get_next_row(ObExecContext &exec_ctx, const common::ObNewRow *&row) const;
virtual int init_op_ctx(ObExecContext &ctx) const;
private:
// types and constants
// static const int COLUMN_COUNT = 16;
//static const int COLUMN_COUNT = 16;
static const int COLUMN_COUNT = 32;
private:
// disallow copy
ObFakeTable(const ObFakeTable& other);
ObFakeTable& operator=(const ObFakeTable& other);
ObFakeTable(const ObFakeTable &other);
ObFakeTable& operator=(const ObFakeTable &other);
// function members
int cons_cur_row(ObFakeTableCtx* table_ctx, const int64_t row_idx) const;
int cons_varchar_cell(ObFakeTableCtx* table_ctx, common::ObObj& cell) const;
int cons_random_int_cell(common::ObObj& cell) const;
int cons_cur_row(ObFakeTableCtx *table_ctx, const int64_t row_idx) const;
int cons_varchar_cell(ObFakeTableCtx *table_ctx, common::ObObj &cell) const;
int cons_random_int_cell(common::ObObj &cell) const;
int rand_int(int max) const;
virtual int inner_create_operator_ctx(ObExecContext& exec_ctx, ObPhyOperatorCtx*& op_ctx) const;
virtual int inner_create_operator_ctx(ObExecContext &exec_ctx, ObPhyOperatorCtx *&op_ctx) const;
protected:
int64_t row_count_;
int64_t n_segments_local_merge_sort_;
};
} // end namespace test
} // end namespace sql
} // end namespace oceanbase
} // end namespace test
} // end namespace sql
} // end namespace oceanbase
#endif /* _OB_FAKE_TABLE_H */

271
unittest/sql/engine/sort/ob_in_memory_sort_test.cpp
View File

@ -24,59 +24,71 @@ using namespace oceanbase::sql;
using namespace oceanbase::common;
using oceanbase::sql::test::ObFakeTable;
class TestInMemorySortTest : public ::testing::Test {
class TestInMemorySortTest: public ::testing::Test
{
public:
struct ColumnOpt {
struct ColumnOpt
{
int64_t col;
ObCollationType cs_type;
TO_STRING_KV("col", col);
};
TestInMemorySortTest();
virtual ~TestInMemorySortTest();
private:
// disallow copy
TestInMemorySortTest(const TestInMemorySortTest& other);
TestInMemorySortTest& operator=(const TestInMemorySortTest& other);
TestInMemorySortTest(const TestInMemorySortTest &other);
TestInMemorySortTest& operator=(const TestInMemorySortTest &other);
protected:
typedef ObSArray<ObSortColumn> ObSortColumns;
void sort_test(
int64_t row_count, ObArray<const ColumnOpt*>& columns, ObBaseSort* base_sort, int64_t column_keys_pos = -1);
void prefix_sort_test(int64_t row_count, ObArray<const ColumnOpt*>& columns,
ObArray<const ColumnOpt*>& prefix_columns, int64_t column_keys_pos);
void sort_test(int64_t row_count,
ObArray<const ColumnOpt *> &columns,
ObBaseSort *base_sort,
int64_t column_keys_pos = -1);
void prefix_sort_test(int64_t row_count,
ObArray<const ColumnOpt *> &columns,
ObArray<const ColumnOpt *> &prefix_columns,
int64_t column_keys_pos);
void serialize_test(int expect_ret);
void sort_exception_test(int expect_ret);
private:
int init(ObBaseSort* base_sort, ObFakeTable& input_table, int64_t row_count, ObArray<const ColumnOpt*>& columns,
ObSortColumns& sort_columns);
void cons_sort_columns(ObArray<const ColumnOpt*>& columns, ObSortColumns& sort_columns);
void cons_op_schema_objs(
const ObIArray<ObSortColumn>& sort_columns, const ObNewRow* row, ObIArray<ObOpSchemaObj>& op_schema_objs);
void copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size);
void cons_new_row(ObNewRow& row, int64_t column_count);
int init(ObBaseSort *base_sort,
ObFakeTable &input_table,
int64_t row_count,
ObArray<const ColumnOpt *> &columns,
ObSortColumns &sort_columns);
void cons_sort_columns(ObArray<const ColumnOpt *> &columns,
ObSortColumns &sort_columns);
void cons_op_schema_objs(const ObIArray<ObSortColumn> &sort_columns,
const ObNewRow *row,
ObIArray<ObOpSchemaObj> &op_schema_objs);
void copy_cell_varchar(ObObj &cell, char *buf, int64_t buf_size);
void cons_new_row(ObNewRow &row, int64_t column_count);
};
TestInMemorySortTest::TestInMemorySortTest()
{}
{
}
TestInMemorySortTest::~TestInMemorySortTest()
{}
{
}
void TestInMemorySortTest::prefix_sort_test(int64_t row_count, ObArray<const ColumnOpt*>& columns,
ObArray<const ColumnOpt*>& prefix_columns, int64_t column_keys_pos)
void TestInMemorySortTest::prefix_sort_test(int64_t row_count,
ObArray<const ColumnOpt *> &columns,
ObArray<const ColumnOpt *> &prefix_columns,
int64_t column_keys_pos)
{
int ret = OB_SUCCESS;
ObArenaAllocator alloc;
ObFakeTable input_table;
// in_memory_sort
ObSortColumns sort_columns; // 9,10, prefix_sort column -> 9,10,11,12, pos = 2
ObSortColumns sort_columns; // 9,10, prefix_sort column -> 9,10,11,12, pos = 2
ObBaseSort mem_sort;
ObExecContext exec_ctx;
ObPhysicalPlan physical_plan;
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
my_session.test_init(0,0,0,NULL);
exec_ctx.set_my_session(&my_session);
cons_sort_columns(columns, sort_columns);
@ -90,10 +102,10 @@ void TestInMemorySortTest::prefix_sort_test(int64_t row_count, ObArray<const Col
ASSERT_EQ(OB_SUCCESS, mem_sort.set_sort_columns(sort_columns, 0));
ObSEArray<ObOpSchemaObj, 16> op_schema_objs;
const ObNewRow* row = NULL;
const ObNewRow *row = NULL;
bool need_sort = false;
for (int i = 0; OB_SUCC(ret) && i < row_count; ++i) {
for (int i = 0 ; OB_SUCC(ret) && i < row_count; ++i) {
ASSERT_EQ(OB_SUCCESS, input_table.get_next_row(exec_ctx, row));
ASSERT_EQ(OB_SUCCESS, mem_sort.add_row(*row, need_sort));
}
@ -106,6 +118,7 @@ void TestInMemorySortTest::prefix_sort_test(int64_t row_count, ObArray<const Col
cons_sort_columns(prefix_columns, prefix_sort_columns);
ASSERT_EQ(OB_SUCCESS, prefix_sort.set_sort_columns(prefix_sort_columns, column_keys_pos));
ObNewRow prefix_row;
cons_new_row(prefix_row, input_table.get_column_count());
ObNewRow mem_row;
@ -115,20 +128,20 @@ void TestInMemorySortTest::prefix_sort_test(int64_t row_count, ObArray<const Col
ObObj first_cell[20];
need_sort = false;
// const ColumnOpt *cpt = NULL;
//const ColumnOpt *cpt = NULL;
ObSortColumn sort_col;
ASSERT_EQ(prefix_columns.count(), prefix_columns.count());
for (int i = 0; i < row_count; ++i) {
for (int i = 0; i < row_count ; ++i) {
ASSERT_EQ(OB_SUCCESS, mem_sort.get_next_row(mem_row));
ASSERT_EQ(OB_SUCCESS, prefix_sort.add_row(mem_row, need_sort));
if (need_sort) {
op_schema_objs.reset();
cons_op_schema_objs(prefix_sort_columns, &mem_row, op_schema_objs);
prefix_sort.sort_rows(); // do sort
prefix_sort.sort_rows(); // do sort
ret = prefix_sort.get_next_row(prefix_row);
// fisrt row , set last row
for (int j = 0; OB_SUCC(ret) && j < prefix_sort_columns.count(); ++j) {
//fisrt row , set last row
for (int j = 0 ; OB_SUCC(ret) && j < prefix_sort_columns.count(); ++j) {
sort_col = prefix_sort_columns.at(j);
last_cell[j] = prefix_row.cells_[sort_col.index_];
first_cell[j] = prefix_row.cells_[sort_col.index_];
@ -137,11 +150,11 @@ void TestInMemorySortTest::prefix_sort_test(int64_t row_count, ObArray<const Col
while (OB_SUCC(ret)) {
ret = prefix_sort.get_next_row(prefix_row);
int64_t cmp = 0;
for (int j = 0; OB_SUCC(ret) && 0 == cmp && j < prefix_sort_columns.count(); ++j) {
for (int j = 0 ; OB_SUCC(ret) && 0 == cmp && j < prefix_sort_columns.count(); ++j) {
// cpt = prefix_columns.at(j);
sort_col = prefix_sort_columns.at(j);
cmp = last_cell[j].compare(prefix_row.cells_[sort_col.index_], sort_col.cs_type_);
// SQL_ENG_LOG(WARN, "row_test", K(last_cell[j]), K(prefix_row.cells_[sort_col.index_]), K(sort_col));
//SQL_ENG_LOG(WARN, "row_test", K(last_cell[j]), K(prefix_row.cells_[sort_col.index_]), K(sort_col));
if (sort_col.is_ascending()) {
ASSERT_TRUE(cmp <= 0);
} else {
@ -149,15 +162,15 @@ void TestInMemorySortTest::prefix_sort_test(int64_t row_count, ObArray<const Col
}
}
// set last cell
for (int j = 0; j < prefix_sort_columns.count(); ++j) {
for (int j = 0 ; j < prefix_sort_columns.count(); ++j) {
sort_col = prefix_sort_columns.at(j);
last_cell[j] = prefix_row.cells_[sort_col.index_];
}
}
// compare prefix cols
for (int j = 0; j < column_keys_pos; ++j) {
for (int j = 0 ; j < column_keys_pos; ++j) {
sort_col = prefix_sort_columns.at(j);
ASSERT_EQ(0, first_cell[j].compare(last_cell[j]));
ASSERT_EQ(0 , first_cell[j].compare(last_cell[j]));
}
ASSERT_EQ(OB_ITER_END, ret);
ret = OB_SUCCESS;
@ -166,8 +179,10 @@ void TestInMemorySortTest::prefix_sort_test(int64_t row_count, ObArray<const Col
}
}
void TestInMemorySortTest::sort_test(
int64_t row_count, ObArray<const ColumnOpt*>& columns, ObBaseSort* base_sort, int64_t column_keys_pos)
void TestInMemorySortTest::sort_test(int64_t row_count,
ObArray<const ColumnOpt *> &columns,
ObBaseSort *base_sort,
int64_t column_keys_pos)
{
ObArenaAllocator alloc;
ObFakeTable input_table;
@ -175,9 +190,9 @@ void TestInMemorySortTest::sort_test(
UNUSED(column_keys_pos);
ASSERT_EQ(OB_SUCCESS, init(base_sort, input_table, row_count, columns, sort_columns));
// read and check
ObObj* cell[10];
ObObj *cell[10];
ObObj last_cell[10];
for (int i = 0; i < 10; i++) {
for (int i = 0 ;i < 10 ; i++) {
cell[i] = NULL;
}
@ -187,8 +202,8 @@ void TestInMemorySortTest::sort_test(
cons_new_row(row, input_table.get_column_count());
ASSERT_EQ(OB_SUCCESS, base_sort->get_next_row(row));
const ColumnOpt* opt = NULL;
for (int64_t j = 0; j < columns.count(); ++j) {
const ColumnOpt *opt = NULL;
for (int64_t j = 0 ; j < columns.count(); ++j) {
opt = columns.at(j);
ASSERT_TRUE(opt != NULL);
last_cell[j] = row.cells_[opt->col];
@ -197,11 +212,11 @@ void TestInMemorySortTest::sort_test(
ObSortColumn cn;
for (int64_t i = 1; i < row_count; ++i) {
ASSERT_EQ(OB_SUCCESS, base_sort->get_next_row(row));
// SQL_ENG_LOG(WARN, "yeti_test", K(row));
//SQL_ENG_LOG(WARN, "yeti_test", K(row));
// check order
int64_t cmp = 0;
bool cmp_next = true;
for (int64_t j = 0; cmp_next && j < columns.count(); ++j) {
for (int64_t j = 0 ; cmp_next && j < columns.count(); ++j) {
opt = columns.at(j);
cn = base_sort->get_sort_columns()->at(j);
ASSERT_TRUE(opt != NULL);
@ -217,8 +232,8 @@ void TestInMemorySortTest::sort_test(
cmp_next = false;
}
}
// set last_cell
for (int64_t j = 0; j < columns.count(); ++j) {
//set last_cell
for (int64_t j = 0 ; j < columns.count(); ++j) {
opt = columns.at(j);
last_cell[j] = row.cells_[opt->col];
if (ObVarcharType == last_cell[j].get_type()) {
@ -227,15 +242,15 @@ void TestInMemorySortTest::sort_test(
copy_cell_varchar(last_cell[j], varchar_buf, 1024);
}
}
} // end for
} // end for
ASSERT_EQ(OB_ITER_END, base_sort->get_next_row(row));
}
void TestInMemorySortTest::serialize_test(int expect_ret)
{
int ret = OB_SUCCESS;
ObArray<const ColumnOpt*> columns;
ColumnOpt cpt1, cpt2;
ObArray<const ColumnOpt *> columns;
ColumnOpt cpt1,cpt2;
ObSortColumns sort_columns_1;
ObSortColumns sort_columns_2;
cpt1.col = 0;
@ -250,13 +265,13 @@ void TestInMemorySortTest::serialize_test(int expect_ret)
int64_t pos = 0;
cons_sort_columns(columns, sort_columns_1);
if (OB_FAIL(sort_columns_1.serialize(buf, MAX_SERIALIZE_BUF_LEN, pos))) {
} else {
if (OB_FAIL(sort_columns_1.serialize(buf, MAX_SERIALIZE_BUF_LEN, pos))) {}
else {
ASSERT_EQ(pos, sort_columns_1.get_serialize_size());
int64_t data_len = pos;
pos = 0;
if (OB_FAIL(sort_columns_2.deserialize(buf, data_len, pos))) {
} else {
if (OB_FAIL(sort_columns_2.deserialize(buf, data_len, pos))){}
else {
ASSERT_EQ(0, strcmp(to_cstring(sort_columns_1), to_cstring(sort_columns_2)));
}
}
@ -272,8 +287,8 @@ void TestInMemorySortTest::sort_exception_test(int expect_ret)
ObSortColumns sort_columns;
ObFakeTable input_table;
int64_t row_count = 1024;
ObArray<const ColumnOpt*> columns;
ColumnOpt cpt1, cpt2;
ObArray<const ColumnOpt *> columns;
ColumnOpt cpt1,cpt2;
ObSortColumns sort_columns_1;
ObSortColumns sort_columns_2;
cpt1.col = 0;
@ -300,70 +315,72 @@ void TestInMemorySortTest::sort_exception_test(int expect_ret)
}
}
int TestInMemorySortTest::init(ObBaseSort* base_sort, ObFakeTable& input_table, int64_t row_count,
ObArray<const ColumnOpt*>& columns, ObSortColumns& sort_columns)
int TestInMemorySortTest::init(ObBaseSort *base_sort,
ObFakeTable &input_table,
int64_t row_count,
ObArray<const ColumnOpt *> &columns,
ObSortColumns &sort_columns)
{
int ret = OB_SUCCESS;
ObExecContext exec_ctx;
ObPhysicalPlan physical_plan;
const ObNewRow* row = NULL;
const ObNewRow *row = NULL;
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
my_session.test_init(0,0,0,NULL);
exec_ctx.set_my_session(&my_session);
cons_sort_columns(columns, sort_columns);
input_table.set_id(0);
input_table.set_row_count(row_count);
input_table.set_phy_plan(&physical_plan);
if (OB_FAIL(exec_ctx.init_phy_op(1))) {
} else if (OB_FAIL(exec_ctx.create_physical_plan_ctx())) {
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(base_sort->set_sort_columns(sort_columns, 0))) {
} else if (OB_FAIL(input_table.open(exec_ctx))) {
} else {
if (OB_FAIL(exec_ctx.init_phy_op(1))) {}
else if (OB_FAIL(exec_ctx.create_physical_plan_ctx())) {}
if (OB_FAIL(ret)) {}
else if (OB_FAIL(base_sort->set_sort_columns(sort_columns, 0))) {}
else if (OB_FAIL(input_table.open(exec_ctx))) {}
else {
ObSEArray<ObOpSchemaObj, 16> op_schema_objs;
bool need_sort = false;
for (int i = 0; OB_SUCC(ret) && i < row_count; ++i) {
if (OB_FAIL(input_table.get_next_row(exec_ctx, row))) {
} else if (OB_FAIL(base_sort->add_row(*row, need_sort))) {
}
if (OB_FAIL(input_table.get_next_row(exec_ctx, row))) {}
else if (OB_FAIL(base_sort->add_row(*row, need_sort))) {}
}
if (OB_FAIL(ret)) {
// do nothing
} else {
cons_op_schema_objs(sort_columns, row, op_schema_objs);
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(base_sort->sort_rows())) {
} else if (OB_FAIL(input_table.close(exec_ctx))) {
}
if (OB_FAIL(ret)) {}
else if (OB_FAIL(base_sort->sort_rows())) {}
else if (OB_FAIL(input_table.close(exec_ctx))) {}
}
return ret;
}
void TestInMemorySortTest::cons_sort_columns(ObArray<const ColumnOpt*>& columns, ObSortColumns& sort_columns)
void TestInMemorySortTest::cons_sort_columns(ObArray<const ColumnOpt *> &columns,
ObSortColumns &sort_columns)
{
ObSortColumn sort_column;
const ColumnOpt* cpt = NULL;
for (int64_t i = 0; i < columns.count(); ++i) {
const ColumnOpt *cpt = NULL;
for (int64_t i = 0 ; i < columns.count(); ++i) {
cpt = columns.at(i);
ASSERT_TRUE(cpt != NULL);
sort_column.index_ = cpt->col;
sort_column.cs_type_ = cpt->cs_type;
sort_column.set_is_ascending((i % 2) ? true : false);
sort_column.set_is_ascending((i%2) ? true : false);
ASSERT_EQ(OB_SUCCESS, sort_columns.push_back(sort_column));
}
return;
}
void TestInMemorySortTest::cons_op_schema_objs(
const ObIArray<ObSortColumn>& sort_columns, const ObNewRow* row, ObIArray<ObOpSchemaObj>& op_schema_objs)
void TestInMemorySortTest::cons_op_schema_objs(const ObIArray<ObSortColumn> &sort_columns,
const ObNewRow *row,
ObIArray<ObOpSchemaObj> &op_schema_objs)
{
if (NULL == row) {
// do nothing
} else {
for (int64_t i = 0; i < sort_columns.count(); i++) {
for (int64_t i =0; i < sort_columns.count(); i++) {
ObOpSchemaObj op_schema_obj(row->get_cell(sort_columns.at(i).index_).get_type());
op_schema_objs.push_back(op_schema_obj);
}
@ -371,7 +388,7 @@ void TestInMemorySortTest::cons_op_schema_objs(
return;
}
void TestInMemorySortTest::copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size)
void TestInMemorySortTest::copy_cell_varchar(ObObj &cell, char *buf, int64_t buf_size)
{
ObString str;
ASSERT_EQ(OB_SUCCESS, cell.get_varchar(str));
@ -382,15 +399,15 @@ void TestInMemorySortTest::copy_cell_varchar(ObObj& cell, char* buf, int64_t buf
return;
}
void TestInMemorySortTest::cons_new_row(ObNewRow& row, int64_t column_count)
void TestInMemorySortTest::cons_new_row(ObNewRow &row, int64_t column_count)
{
row.cells_ = static_cast<ObObj*>(malloc(column_count * sizeof(ObObj)));
row.cells_ = static_cast<ObObj *>(malloc(column_count * sizeof(ObObj)));
row.count_ = column_count;
}
TEST_F(TestInMemorySortTest, varchar_int_item)
{
ObArray<const ColumnOpt*> columns;
ObArray<const ColumnOpt *> columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
cpt1.col = 0;
@ -400,32 +417,32 @@ TEST_F(TestInMemorySortTest, varchar_int_item)
ASSERT_EQ(OB_SUCCESS, columns.push_back(&cpt1));
ASSERT_EQ(OB_SUCCESS, columns.push_back(&cpt2));
ObBaseSort in_mem_sort;
sort_test(1024, columns, &in_mem_sort);
sort_test( 1024, columns, &in_mem_sort);
ObBaseSort in_mem_sort1;
sort_test(16 * 1024, columns, &in_mem_sort1);
sort_test( 16 * 1024, columns, &in_mem_sort1);
ObBaseSort in_mem_sort2;
sort_test(256 * 1024, columns, &in_mem_sort2);
ObBaseSort in_mem_sort3;
cpt1.cs_type = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(1024, columns, &in_mem_sort3);
sort_test( 1024, columns, &in_mem_sort3);
ObBaseSort in_mem_sort4;
sort_test(16 * 1024, columns, &in_mem_sort4);
sort_test( 16 * 1024, columns, &in_mem_sort4);
ObBaseSort in_mem_sort5;
sort_test(256 * 1024, columns, &in_mem_sort5);
// topn sort
for (int64_t i = 1; i < 2048; i += 100) {
//topn sort
for (int64_t i = 1 ; i < 2048 ; i += 100) {
ObBaseSort mem_topn_sort;
sort_test(10, columns, &mem_topn_sort, i);
sort_test( 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(16 * 10, columns, &mem_topn_sort, i);
sort_test( 16 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(256 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
cpt1.cs_type = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(10, columns, &mem_topn_sort, i);
sort_test( 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(16 * 10, columns, &mem_topn_sort, i);
sort_test( 16 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(256 * 10, columns, &mem_topn_sort, i);
}
@ -433,7 +450,7 @@ TEST_F(TestInMemorySortTest, varchar_int_item)
TEST_F(TestInMemorySortTest, int_int_equal_item)
{
ObArray<const ColumnOpt*> columns;
ObArray<const ColumnOpt *> columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
cpt1.col = 2;
@ -443,18 +460,18 @@ TEST_F(TestInMemorySortTest, int_int_equal_item)
columns.push_back(&cpt1);
columns.push_back(&cpt2);
ObBaseSort in_mem_sort;
sort_test(1024, columns, &in_mem_sort);
sort_test( 1024, columns, &in_mem_sort);
ObBaseSort in_mem_sort1;
sort_test(16 * 1024, columns, &in_mem_sort1);
sort_test( 16 * 1024, columns, &in_mem_sort1);
ObBaseSort in_mem_sort2;
sort_test(256 * 1024, columns, &in_mem_sort2);
// topn sort
for (int64_t i = 1; i < 2048; i += 100) {
//topn sort
for (int64_t i = 1 ; i < 2048 ; i += 100) {
ObBaseSort mem_topn_sort;
sort_test(10, columns, &mem_topn_sort, i);
sort_test( 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(16 * 10, columns, &mem_topn_sort, i);
sort_test( 16 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(256 * 10, columns, &mem_topn_sort, i);
}
@ -462,7 +479,7 @@ TEST_F(TestInMemorySortTest, int_int_equal_item)
TEST_F(TestInMemorySortTest, int_int_item)
{
ObArray<const ColumnOpt*> columns;
ObArray<const ColumnOpt *> columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
cpt1.col = 11;
@ -472,18 +489,18 @@ TEST_F(TestInMemorySortTest, int_int_item)
columns.push_back(&cpt1);
columns.push_back(&cpt2);
ObBaseSort in_mem_sort;
sort_test(1024, columns, &in_mem_sort);
sort_test( 1024, columns, &in_mem_sort);
ObBaseSort in_mem_sort1;
sort_test(16 * 1024, columns, &in_mem_sort1);
sort_test( 16 * 1024, columns, &in_mem_sort1);
ObBaseSort in_mem_sort2;
sort_test(256 * 1024, columns, &in_mem_sort2);
// topn sort
for (int64_t i = 1; i < 2048; i += 100) {
//topn sort
for (int64_t i = 1 ; i < 2048 ; i += 100) {
ObBaseSort mem_topn_sort;
sort_test(10, columns, &mem_topn_sort, i);
sort_test( 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(16 * 10, columns, &mem_topn_sort, i);
sort_test( 16 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(256 * 10, columns, &mem_topn_sort, i);
}
@ -491,8 +508,8 @@ TEST_F(TestInMemorySortTest, int_int_item)
TEST_F(TestInMemorySortTest, prefix_int_test)
{
ObArray<const ColumnOpt*> columns;
ObArray<const ColumnOpt*> prefix_columns;
ObArray<const ColumnOpt *> columns;
ObArray<const ColumnOpt *> prefix_columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
ColumnOpt cpt3;
@ -516,17 +533,17 @@ TEST_F(TestInMemorySortTest, prefix_int_test)
prefix_columns.push_back(&cpt4);
ObBaseSort in_mem_sort;
prefix_sort_test(1024, columns, prefix_columns, 2);
prefix_sort_test( 1024, columns, prefix_columns, 2);
ObBaseSort in_mem_sort1;
prefix_sort_test(16 * 1024, columns, prefix_columns, 2);
prefix_sort_test( 16 * 1024, columns, prefix_columns, 2);
ObBaseSort in_mem_sort2;
prefix_sort_test(256 * 1024, columns, prefix_columns, 2);
}
TEST_F(TestInMemorySortTest, prefix_var_int_test)
{
ObArray<const ColumnOpt*> columns;
ObArray<const ColumnOpt*> prefix_columns;
ObArray<const ColumnOpt *> columns;
ObArray<const ColumnOpt *> prefix_columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
ColumnOpt cpt3;
@ -546,9 +563,9 @@ TEST_F(TestInMemorySortTest, prefix_var_int_test)
prefix_columns.push_back(&cpt3);
ObBaseSort in_mem_sort;
prefix_sort_test(1024, columns, prefix_columns, 2);
prefix_sort_test( 1024, columns, prefix_columns, 2);
ObBaseSort in_mem_sort1;
prefix_sort_test(16 * 1024, columns, prefix_columns, 2);
prefix_sort_test( 16 * 1024, columns, prefix_columns, 2);
ObBaseSort in_mem_sort2;
prefix_sort_test(256 * 1024, columns, prefix_columns, 2);
}
@ -559,10 +576,10 @@ TEST_F(TestInMemorySortTest, serialize)
}
#define SORT_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
} while (0)
TEST_F(TestInMemorySortTest, sort_exception)
@ -574,10 +591,10 @@ TEST_F(TestInMemorySortTest, sort_exception)
}
#define SERIALIZE_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
serialize_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
serialize_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
} while (0)
TEST_F(TestInMemorySortTest, serialize_exception)
@ -588,9 +605,9 @@ TEST_F(TestInMemorySortTest, serialize_exception)
SERIALIZE_EXCEPTION_TEST("ob_base_sort.cpp", "deserialize", "t3", OB_ERROR, OB_ERROR);
}
int main(int argc, char** argv)
int main(int argc, char **argv)
{
::testing::InitGoogleTest(&argc, argv);
::testing::InitGoogleTest(&argc,argv);
oceanbase::common::ObLogger::get_logger().set_log_level("WARN");
return RUN_ALL_TESTS();
}

298
unittest/sql/engine/sort/ob_in_memory_topn_sort_test.cpp Normal file
View File

@ -0,0 +1,298 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "sql/engine/sort/ob_in_memory_topn_sort.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/container/ob_se_array.h"
#include "sql/session/ob_sql_session_info.h"
#include "sql/engine/ob_physical_plan.h"
#include <gtest/gtest.h>
#include "ob_fake_table.h"
#include "sql/engine/ob_exec_context.h"
using namespace oceanbase::sql;
using namespace oceanbase::common;
using oceanbase::sql::test::ObFakeTable;
class InMemoryTopnSortTest: public ::testing::Test
{
public:
InMemoryTopnSortTest();
virtual ~InMemoryTopnSortTest();
private:
// disallow copy
InMemoryTopnSortTest(const InMemoryTopnSortTest &other);
InMemoryTopnSortTest& operator=(const InMemoryTopnSortTest &other);
protected:
typedef ObSArray<ObSortColumn> ObSortColumns;
void sort_test(int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2,
int64_t topn);
void sort_exception_test(int expect_ret);
private:
int init(ObInMemoryTopnSort &in_mem_topn_sort,
ObFakeTable &input_table,
int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2,
int64_t topn);
void cons_sort_columns(int64_t col1, ObCollationType cs_type1,
int64_t col2, ObCollationType cs_type2,
ObSortColumns &sort_columns);
void cons_op_schema_objs(const ObIArray<ObSortColumn> &sort_columns,
const ObNewRow *row,
ObIArray<ObOpSchemaObj> &op_schema_objs);
void copy_cell_varchar(ObObj &cell, char *buf, int64_t buf_size);
void cons_new_row(ObNewRow &row, int64_t column_count);
};
InMemoryTopnSortTest::InMemoryTopnSortTest()
{
}
InMemoryTopnSortTest::~InMemoryTopnSortTest()
{
}
void InMemoryTopnSortTest::sort_test(int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2,
int64_t topn)
{
ObArenaAllocator alloc;
ObInMemoryTopnSort in_mem_topn_sort;
ObFakeTable input_table;
ASSERT_EQ(OB_SUCCESS, init(in_mem_topn_sort, input_table, row_count,
sort_col1, cs_type1, sort_col2, cs_type2, topn));
// read and check
const ObObj *cell1 = NULL;
const ObObj *cell2 = NULL;
ObObj last_cell1;
ObObj last_cell2;
char varchar_buf[1024];
ObMalloc allocator;
ObNewRow row;
cons_new_row(row, input_table.get_column_count());
if (0 != topn) {
ASSERT_EQ(OB_SUCCESS, in_mem_topn_sort.get_next_row(row));
}
last_cell1 = row.cells_[sort_col1];
last_cell2 = row.cells_[sort_col2];
for (int64_t i = 1; i < min(row_count, topn); ++i) {
ASSERT_EQ(OB_SUCCESS, in_mem_topn_sort.get_next_row(row));
cell1 = &row.cells_[sort_col1];
cell2 = &row.cells_[sort_col2];
// check order, cell1: descending, cell2: ascending.
if (0 == last_cell1.compare(*cell1, cs_type1)) {
ASSERT_TRUE(last_cell2.compare(*cell2, cs_type2) <= 0);
} else {
ASSERT_TRUE(last_cell1.compare(*cell1, cs_type1) > 0);
}
last_cell1 = *cell1;
last_cell2 = *cell2;
if (ObVarcharType == last_cell1.get_type()) {
copy_cell_varchar(last_cell1, varchar_buf, 1024);
} else if (ObVarcharType == last_cell2.get_type()) {
copy_cell_varchar(last_cell2, varchar_buf, 1024);
}
} // end for
ASSERT_EQ(OB_ITER_END, in_mem_topn_sort.get_next_row(row));
}
void InMemoryTopnSortTest::sort_exception_test(int expect_ret)
{
int ret = OB_SUCCESS;
ObInMemoryTopnSort in_mem_sort;
ObFakeTable input_table;
int64_t row_count = 1024;
int64_t sort_col1 = 0;
int64_t sort_col2 = 1;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
ObNewRow row;
int64_t topn = 100;
if (OB_FAIL(init(in_mem_sort, input_table, row_count,
sort_col1, cs_type1, sort_col2, cs_type2, topn))) {
} else {
cons_new_row(row, input_table.get_column_count());
for (int64_t i = 0; OB_SUCC(ret) && i < min(topn, row_count); ++i) {
ret = in_mem_sort.get_next_row(row);
}
if (OB_SUCC(ret)) {
ASSERT_EQ(OB_ITER_END, in_mem_sort.get_next_row(row));
}
}
in_mem_sort.reuse();
if (OB_FAIL(ret)) {
ASSERT_EQ(expect_ret, ret);
}
}
int InMemoryTopnSortTest::init(ObInMemoryTopnSort &in_mem_topn_sort,
ObFakeTable &input_table,
int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2,
int64_t topn)
{
int ret = OB_SUCCESS;
ObSortColumns sort_columns;
ObExecContext exec_ctx;
ObPhysicalPlan physical_plan;
const ObNewRow *row = NULL;
ObSQLSessionInfo my_session;
my_session.test_init(0,0,0,NULL);
exec_ctx.set_my_session(&my_session);
cons_sort_columns(sort_col1, cs_type1, sort_col2, cs_type2, sort_columns);
input_table.set_id(0);
input_table.set_row_count(row_count);
input_table.set_phy_plan(&physical_plan);
if (OB_FAIL(exec_ctx.init_phy_op(1))) {}
else if (OB_FAIL(exec_ctx.create_physical_plan_ctx())) {}
else {
exec_ctx.get_physical_plan_ctx()->set_phy_plan(&physical_plan);
}
in_mem_topn_sort.set_topn(topn);
if (OB_FAIL(ret)) {}
else if (OB_FAIL(in_mem_topn_sort.set_sort_columns(sort_columns))) {}
else if (OB_FAIL(input_table.open(exec_ctx))) {}
else {
ObSEArray<ObOpSchemaObj, 8> op_schema_objs;
for (int i = 0; OB_SUCC(ret) && i < row_count; ++i) {
if (OB_FAIL(input_table.get_next_row(exec_ctx, row))) {}
else if (OB_FAIL(in_mem_topn_sort.add_row(*row))) {}
}
cons_op_schema_objs(sort_columns, row, op_schema_objs)
if (OB_FAIL(ret)) {}
else if (OB_FAIL(in_mem_topn_sort.sort_rows(op_schema_objs))) {}
else if (OB_FAIL(input_table.close(exec_ctx))) {}
}
return ret;
}
void InMemoryTopnSortTest::cons_sort_columns(int64_t col1, ObCollationType cs_type1,
int64_t col2, ObCollationType cs_type2,
ObSortColumns &sort_columns)
{
ObSortColumn sort_column;
sort_column.index_ = col1;
sort_column.cs_type_ = cs_type1;
sort_column.set_is_ascending(false);
ASSERT_EQ(OB_SUCCESS, sort_columns.push_back(sort_column));
sort_column.index_ = col2;
sort_column.cs_type_ = cs_type2;
sort_column.set_is_ascending(true);
ASSERT_EQ(OB_SUCCESS, sort_columns.push_back(sort_column));
return;
}
void InMemoryTopnSortTest::cons_op_schema_objs(const ObIArray<ObSortColumn> &sort_columns,
const ObNewRow *row,
ObIArray<ObOpSchemaObj> &op_schema_objs)
{
if (NULL == row) {
// do nothing
} else {
for (int64_t i =0; i < sort_columns.count(); i++) {
ObOpSchemaObj op_schema_obj(row->get_cell(sort_columns.at(i).index_).get_type());
op_schema_objs.push_back(op_schema_obj);
}
}
return;
}
void InMemoryTopnSortTest::copy_cell_varchar(ObObj &cell, char *buf, int64_t buf_size)
{
ObString str;
ASSERT_EQ(OB_SUCCESS, cell.get_varchar(str));
ASSERT_TRUE(str.length() < buf_size);
memcpy(buf, str.ptr(), str.length());
str.assign_ptr(buf, str.length());
cell.set_varchar(str);
return;
}
void InMemoryTopnSortTest::cons_new_row(ObNewRow &row, int64_t column_count)
{
row.cells_ = static_cast<ObObj *>(malloc(column_count * sizeof(ObObj)));
row.count_ = column_count;
}
TEST_F(InMemoryTopnSortTest, varchar_int_item)
{
int64_t sort_col1 = 0;
int64_t sort_col2 = 1;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
for (int64_t i = 1 ; i < 2048 ; i += 100) {
sort_test( 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
sort_test( 16 * 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
sort_test(256 * 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
cs_type1 = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test( 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
sort_test( 16 * 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
sort_test(256 * 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
}
}
TEST_F(InMemoryTopnSortTest, int_int_equal_item)
{
int64_t sort_col1 = 2;
int64_t sort_col2 = 3;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
for (int64_t i = 1 ; i < 2048 ; i += 100) {
sort_test( 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
sort_test( 16 * 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
sort_test(256 * 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
}
}
TEST_F(InMemoryTopnSortTest, int_int_item)
{
int64_t sort_col1 = 11;
int64_t sort_col2 = 12;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
for (int64_t i = 1 ; i < 2048 ; i += 100) {
sort_test( 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
sort_test( 16 * 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
sort_test(256 * 10, sort_col1, cs_type1, sort_col2, cs_type2, i);
}
}
#define SORT_TOPN_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
} while (0)
TEST_F(InMemoryTopnSortTest, sort_exception)
{
SORT_TOPN_EXCEPTION_TEST("ob_in_memory_sort.cpp", "set_sort_columns", "t1", OB_ERROR, OB_ERROR);
SORT_TOPN_EXCEPTION_TEST("ob_in_memory_sort.cpp", "add_row", "t1", OB_ERROR, OB_ERROR);
SORT_TOPN_EXCEPTION_TEST("ob_in_memory_sort.cpp", "add_row", "t3", OB_ERROR, OB_ERROR);
SORT_TOPN_EXCEPTION_TEST("ob_in_memory_sort.cpp", "get_next_row", "t1", OB_ERROR, OB_ERROR);
}
int main(int argc, char **argv)
{
::testing::InitGoogleTest(&argc,argv);
oceanbase::common::ObLogger::get_logger().set_log_level("WARN");
return RUN_ALL_TESTS();
}

212
unittest/sql/engine/sort/ob_merge_sort_test.cpp
View File

@ -24,27 +24,28 @@
#include "sql/engine/ob_physical_plan.h"
#include "sql/session/ob_sql_session_info.h"
#include "ob_fake_table.h"
#include "share/ob_simple_mem_limit_getter.h"
using namespace oceanbase::sql;
using namespace oceanbase::common;
using namespace oceanbase::storage;
using namespace oceanbase::blocksstable;
using oceanbase::sql::test::ObFakeTable;
class TestMergeSort : public ObMergeSort {
static ObSimpleMemLimitGetter getter;
class TestMergeSort : public ObMergeSort
{
public:
TestMergeSort(ObIAllocator& alloc) : ObMergeSort(alloc)
{}
~TestMergeSort()
{}
TestMergeSort(ObIAllocator &alloc) : ObMergeSort(alloc) {}
~TestMergeSort() {}
};
class TestMergeSortTest : public oceanbase::blocksstable::TestDataFilePrepare {
class TestMergeSortTest: public oceanbase::blocksstable::TestDataFilePrepare
{
static const int64_t SORT_BUF_SIZE = 2 * 1024 * 1024;
static const int64_t EXPIRE_TIMESTAMP = 0;
static const int64_t MACRO_BLOCK_SIZE = 2 * 1024 * 1024;
static const int64_t MACRO_BLOCK_COUNT = 50 * 1024;
static const uint64_t TENATN_ID = 1;
public:
TestMergeSortTest();
virtual ~TestMergeSortTest();
@ -52,44 +53,52 @@ public:
virtual void TearDown();
int init_tenant_mgr();
void destroy_tenant_mgr();
private:
// disallow copy
TestMergeSortTest(const TestMergeSortTest& other);
TestMergeSortTest& operator=(const TestMergeSortTest& other);
TestMergeSortTest(const TestMergeSortTest &other);
TestMergeSortTest& operator=(const TestMergeSortTest &other);
protected:
typedef ObSArray<ObSortColumn> ObSortColumns;
void sort_test(int64_t run_count, int64_t row_count, int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2,
ObCollationType cs_type2);
void sort_test(int64_t run_count, int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2);
void sort_exception_test(int expect_ret);
private:
int init_op(TestMergeSort& merge_sort, ObFakeTable& input_table, ObSortColumns& sort_cols, int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2, ObCollationType cs_type2);
int init_data(TestMergeSort& merge_sort, ObBaseSort& in_mem_sort, ObFakeTable& input_table, ObSortColumns& sort_cols,
int64_t run_count, int64_t row_count);
void cons_sort_cols(
int64_t col1, ObCollationType cs_type1, int64_t col2, ObCollationType cs_type2, ObSortColumns& sort_cols);
void copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size);
void cons_new_row(ObNewRow& row, int64_t column_count);
int init_op(TestMergeSort &merge_sort,
ObFakeTable &input_table,
ObSortColumns &sort_cols,
int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2);
int init_data(TestMergeSort &merge_sort,
ObBaseSort &in_mem_sort,
ObFakeTable &input_table,
ObSortColumns &sort_cols,
int64_t run_count,
int64_t row_count);
void cons_sort_cols(int64_t col1, ObCollationType cs_type1,
int64_t col2, ObCollationType cs_type2,
ObSortColumns &sort_cols);
void copy_cell_varchar(ObObj &cell, char *buf, int64_t buf_size);
void cons_new_row(ObNewRow &row, int64_t column_count);
private:
ObPhysicalPlan physical_plan_;
ObArenaAllocator alloc_;
};
TestMergeSortTest::TestMergeSortTest() : TestDataFilePrepare("TestMergeSort", MACRO_BLOCK_SIZE, MACRO_BLOCK_COUNT)
{}
TestMergeSortTest::TestMergeSortTest()
: TestDataFilePrepare(&getter, "TestMergeSort", MACRO_BLOCK_SIZE, MACRO_BLOCK_COUNT)
{
}
TestMergeSortTest::~TestMergeSortTest()
{}
{
}
void TestMergeSortTest::SetUp()
{
TestDataFilePrepare::SetUp();
FILE_MANAGER_INSTANCE_V2.init();
FILE_MANAGER_INSTANCE_V2.start();
ASSERT_EQ(OB_SUCCESS, init_tenant_mgr());
}
@ -104,7 +113,6 @@ void TestMergeSortTest::TearDown()
int TestMergeSortTest::init_tenant_mgr()
{
int ret = OB_SUCCESS;
ObTenantManager& tm = ObTenantManager::get_instance();
ObAddr self;
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
@ -112,19 +120,12 @@ int TestMergeSortTest::init_tenant_mgr()
oceanbase::share::ObRsMgr rs_mgr;
int64_t tenant_id = 1;
self.set_ip_addr("127.0.0.1", 8086);
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(tenant_id);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.set_tenant_mem_limit(tenant_id, 2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SYS_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
ret = getter.add_tenant(tenant_id,
2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
const int64_t ulmt = 128LL << 30;
const int64_t llmt = 128LL << 30;
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
ret = getter.add_tenant(OB_SERVER_TENANT_ID, ulmt, llmt);
EXPECT_EQ(OB_SUCCESS, ret);
oceanbase::lib::set_memory_limit(128LL << 32);
return ret;
@ -132,27 +133,26 @@ int TestMergeSortTest::init_tenant_mgr()
void TestMergeSortTest::destroy_tenant_mgr()
{
ObTenantManager& tm = ObTenantManager::get_instance();
tm.destroy();
}
void TestMergeSortTest::sort_test(int64_t run_count, int64_t row_count, int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
void TestMergeSortTest::sort_test(int64_t run_count, int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
{
ObArenaAllocator alloc;
TestMergeSort merge_sort(alloc_);
ObBaseSort in_mem_sort;
ObFakeTable input_table;
ObSortColumns sort_cols;
const ObObj* cell1 = NULL;
const ObObj* cell2 = NULL;
const ObObj *cell1 = NULL;
const ObObj *cell2 = NULL;
ObObj last_cell1;
ObObj last_cell2;
char varchar_buf[1024];
ObNewRow row;
ASSERT_EQ(OB_SUCCESS, SLOGGER.begin(OB_LOG_CS_DAILY_MERGE));
ASSERT_EQ(
OB_SUCCESS, init_op(merge_sort, input_table, sort_cols, row_count, sort_col1, cs_type1, sort_col2, cs_type2));
// ASSERT_EQ(OB_SUCCESS, SLOGGER.begin(OB_LOG_CS_DAILY_MERGE));
ASSERT_EQ(OB_SUCCESS, init_op(merge_sort, input_table, sort_cols, row_count,
sort_col1, cs_type1, sort_col2, cs_type2));
ASSERT_EQ(OB_SUCCESS, init_data(merge_sort, in_mem_sort, input_table, sort_cols, run_count, row_count));
ASSERT_EQ(OB_SUCCESS, merge_sort.do_merge_sort(input_table.get_column_count()));
cons_new_row(row, input_table.get_column_count());
@ -160,7 +160,7 @@ void TestMergeSortTest::sort_test(int64_t run_count, int64_t row_count, int64_t
last_cell1 = row.cells_[sort_col1];
last_cell2 = row.cells_[sort_col2];
int64_t total_row_count = run_count * row_count;
// int ret = OB_SUCCESS;
//int ret = OB_SUCCESS;
for (int64_t i = 1; i < total_row_count; ++i) {
merge_sort.get_next_row(row);
cell1 = &row.cells_[sort_col1];
@ -180,7 +180,7 @@ void TestMergeSortTest::sort_test(int64_t run_count, int64_t row_count, int64_t
}
}
ASSERT_EQ(OB_ITER_END, merge_sort.get_next_row(row));
ASSERT_EQ(OB_SUCCESS, SLOGGER.abort());
// ASSERT_EQ(OB_SUCCESS, SLOGGER.abort());
return;
}
@ -200,10 +200,11 @@ void TestMergeSortTest::sort_exception_test(int expect_ret)
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
if (OB_FAIL(init_op(merge_sort, input_table, sort_cols, row_count, sort_col1, cs_type1, sort_col2, cs_type2))) {
} else if (OB_FAIL(init_data(merge_sort, in_mem_sort, input_table, sort_cols, run_count, row_count))) {
} else if (OB_FAIL(merge_sort.do_merge_sort(input_table.get_column_count()))) {
} else {
if (OB_FAIL(init_op(merge_sort, input_table, sort_cols, row_count,
sort_col1, cs_type1, sort_col2, cs_type2))) {}
else if (OB_FAIL(init_data(merge_sort, in_mem_sort, input_table, sort_cols, run_count, row_count))) {}
else if (OB_FAIL(merge_sort.do_merge_sort(input_table.get_column_count()))) {}
else {
ObNewRow row;
cons_new_row(row, input_table.get_column_count());
int64_t total_row_count = run_count * row_count;
@ -220,8 +221,12 @@ void TestMergeSortTest::sort_exception_test(int expect_ret)
}
}
int TestMergeSortTest::init_op(TestMergeSort& merge_sort, ObFakeTable& input_table, ObSortColumns& sort_cols,
int64_t row_count, int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2, ObCollationType cs_type2)
int TestMergeSortTest::init_op(TestMergeSort &merge_sort,
ObFakeTable &input_table,
ObSortColumns &sort_cols,
int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
{
int ret = OB_SUCCESS;
ObString filename;
@ -238,29 +243,32 @@ int TestMergeSortTest::init_op(TestMergeSort& merge_sort, ObFakeTable& input_tab
return ret;
}
int TestMergeSortTest::init_data(TestMergeSort& merge_sort, ObBaseSort& in_mem_sort, ObFakeTable& input_table,
ObSortColumns& sort_cols, int64_t run_count, int64_t row_count)
int TestMergeSortTest::init_data(TestMergeSort &merge_sort,
ObBaseSort &in_mem_sort,
ObFakeTable &input_table,
ObSortColumns &sort_cols,
int64_t run_count,
int64_t row_count)
{
int ret = OB_SUCCESS;
const ObNewRow* row = NULL;
const ObNewRow *row = NULL;
bool need_false = false;
// loop run_count to generate dumped runs.
for (int i = 0; OB_SUCC(ret) && i < run_count; ++i) {
ObExecContext exec_ctx;
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
my_session.test_init(0,0,0,NULL);
exec_ctx.set_my_session(&my_session);
my_session.test_init(0, 0, 0, NULL);
my_session.test_init(0,0,0,NULL);
in_mem_sort.reuse();
if (OB_FAIL(exec_ctx.init_phy_op(1))) {
} else if (OB_FAIL(exec_ctx.create_physical_plan_ctx())) {
} else if (OB_FAIL(in_mem_sort.set_sort_columns(sort_cols, 0))) {
} else if (OB_FAIL(input_table.open(exec_ctx))) {
} else {
if (OB_FAIL(exec_ctx.init_phy_op(1))) {}
else if (OB_FAIL(exec_ctx.create_physical_plan_ctx())) {}
else if (OB_FAIL(in_mem_sort.set_sort_columns(sort_cols, 0))) {}
else if (OB_FAIL(input_table.open(exec_ctx))) {}
else {
for (int j = 0; OB_SUCC(ret) && j < row_count; ++j) {
if (OB_FAIL(input_table.get_next_row(exec_ctx, row))) {
} else if (OB_FAIL(in_mem_sort.add_row(*row, need_false))) {
}
if (OB_FAIL(input_table.get_next_row(exec_ctx, row))) {}
else if (OB_FAIL(in_mem_sort.add_row(*row, need_false))) {}
}
}
ObSEArray<ObOpSchemaObj, 8> op_schema_objs;
@ -268,17 +276,17 @@ int TestMergeSortTest::init_data(TestMergeSort& merge_sort, ObBaseSort& in_mem_s
// do nothing
} else {
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(in_mem_sort.sort_rows())) {
} else if (OB_FAIL(input_table.close(exec_ctx))) {
} else if (OB_FAIL(merge_sort.dump_base_run(in_mem_sort))) {
}
} // end for i
if (OB_FAIL(ret)) {}
else if (OB_FAIL(in_mem_sort.sort_rows())) {}
else if (OB_FAIL(input_table.close(exec_ctx))) {}
else if (OB_FAIL(merge_sort.dump_base_run(in_mem_sort))) {}
} // end for i
return ret;
}
void TestMergeSortTest::cons_sort_cols(
int64_t col1, ObCollationType cs_type1, int64_t col2, ObCollationType cs_type2, ObSortColumns& sort_cols)
void TestMergeSortTest::cons_sort_cols(int64_t col1, ObCollationType cs_type1,
int64_t col2, ObCollationType cs_type2,
ObSortColumns &sort_cols)
{
ObSortColumn sort_col;
sort_col.index_ = col1;
@ -292,7 +300,7 @@ void TestMergeSortTest::cons_sort_cols(
return;
}
void TestMergeSortTest::copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size)
void TestMergeSortTest::copy_cell_varchar(ObObj &cell, char *buf, int64_t buf_size)
{
ObString str;
ASSERT_EQ(OB_SUCCESS, cell.get_varchar(str));
@ -303,9 +311,9 @@ void TestMergeSortTest::copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_si
return;
}
void TestMergeSortTest::cons_new_row(ObNewRow& row, int64_t column_count)
void TestMergeSortTest::cons_new_row(ObNewRow &row, int64_t column_count)
{
row.cells_ = static_cast<ObObj*>(malloc(column_count * sizeof(ObObj)));
row.cells_ = static_cast<ObObj *>(malloc(column_count * sizeof(ObObj)));
row.count_ = column_count;
}
@ -315,12 +323,12 @@ TEST_F(TestMergeSortTest, sorted_int_int_test)
int64_t sort_col2 = 2;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(4, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 65536, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256, 4, sort_col1, cs_type1, sort_col2, cs_type2);
}
@ -330,11 +338,11 @@ TEST_F(TestMergeSortTest, random_int_int_test)
int64_t sort_col2 = 13;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 65536, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256, 4, sort_col1, cs_type1, sort_col2, cs_type2);
}
@ -344,27 +352,27 @@ TEST_F(TestMergeSortTest, random_varchar_int_test)
int64_t sort_col2 = 1;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 65536, sort_col1, cs_type1, sort_col2, cs_type2);
cs_type1 = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 65536, sort_col1, cs_type1, sort_col2, cs_type2);
}
#define SORT_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
} while (0)
TEST_F(TestMergeSortTest, sort_exception)
@ -382,12 +390,12 @@ TEST_F(TestMergeSortTest, sort_exception)
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "get_next_row", "t7", OB_ERROR, OB_ERROR);
}
int main(int argc, char** argv)
int main(int argc, char **argv)
{
system("rm -f test_merge_sort.log*");
OB_LOGGER.set_file_name("test_merge_sort.log", true, true);
// OB_LOGGER.set_log_level("INFO");
::testing::InitGoogleTest(&argc, argv);
//OB_LOGGER.set_log_level("INFO");
::testing::InitGoogleTest(&argc,argv);
oceanbase::common::ObLogger::get_logger().set_log_level("WARN");
return RUN_ALL_TESTS();
}

433
unittest/sql/engine/sort/ob_sort_test.cpp
View File

@ -19,10 +19,11 @@
#include <gtest/gtest.h>
#include "ob_fake_table.h"
#include "sql/engine/ob_exec_context.h"
#include "share/ob_worker.h"
#include "lib/worker.h"
#include "observer/ob_signal_handle.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include "share/ob_simple_mem_limit_getter.h"
#include <thread>
#include <vector>
@ -34,19 +35,20 @@ using namespace oceanbase::omt;
using namespace oceanbase::common;
using oceanbase::sql::test::ObFakeTable;
static ObSimpleMemLimitGetter getter;
#define TEST_SORT_DUMP_GET_HASH_AREA_SIZE() (get_sort_area_size())
#define TEST_SORT_DUMP_SET_HASH_AREA_SIZE(size) (set_sort_area_size(size))
class ObSortTest : public blocksstable::TestDataFilePrepare {
class ObSortTest: public blocksstable::TestDataFilePrepare
{
public:
ObSortTest();
virtual ~ObSortTest();
private:
// disallow copy
ObSortTest(const ObSortTest& other);
ObSortTest& operator=(const ObSortTest& other);
ObSortTest(const ObSortTest &other);
ObSortTest& operator=(const ObSortTest &other);
protected:
virtual void SetUp() override
{
@ -87,21 +89,20 @@ protected:
}
void destroy_tenant_mgr()
{
ObTenantManager::get_instance().destroy();
}
template <typename SortInit>
void sort_test(int64_t row_count, int64_t mem_limit, SortInit sort_init, int64_t verify_row_cnt = -1,
bool local_merge_sort = false);
void sort_test(int64_t row_count, int64_t mem_limit, int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2,
ObCollationType cs_type2);
void local_merge_sort_test(int64_t row_count, int64_t mem_limit, int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2);
template<typename SortInit>
void sort_test(int64_t row_count, int64_t mem_limit, SortInit sort_init, int64_t verify_row_cnt = -1, bool local_merge_sort = false);
void sort_test(int64_t row_count, int64_t mem_limit,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2);
void local_merge_sort_test(int64_t row_count, int64_t mem_limit,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2);
void serialize_test();
void sort_exception_test(int expect_ret);
private:
static void copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size)
static void copy_cell_varchar(ObObj &cell, char *buf, int64_t buf_size)
{
ObString str;
ASSERT_EQ(OB_SUCCESS, cell.get_varchar(str));
@ -111,9 +112,10 @@ private:
cell.set_varchar(str);
return;
}
void cons_op_schema_objs(const ObIArray<ObSortColumn>& sort_columns, ObIArray<ObOpSchemaObj>& op_schema_objs)
void cons_op_schema_objs(const ObIArray<ObSortColumn> &sort_columns,
ObIArray<ObOpSchemaObj> &op_schema_objs)
{
for (int64_t i = 0; i < sort_columns.count(); i++) {
for (int64_t i =0; i < sort_columns.count(); i++) {
ObOpSchemaObj op_schema_obj;
if (0 == sort_columns.at(i).index_) {
op_schema_obj.obj_type_ = common::ObVarcharType;
@ -126,13 +128,14 @@ private:
}
};
class ObSortPlan {
class ObSortPlan
{
public:
static ObSort& get_instance()
static ObSort &get_instance()
{
return *sort_;
}
template <typename SortInit>
template<typename SortInit>
static int init(int64_t row_count, int64_t mem_limit, SortInit sort_init)
{
if (mem_limit <= 0) {
@ -156,9 +159,8 @@ public:
input_table_->set_phy_plan(physical_plan_);
sort_->set_phy_plan(physical_plan_);
row_count_ = row_count;
if (OB_FAIL(sort_->set_child(0, *input_table_))) {
} else if (OB_FAIL(sort_init(*sort_))) {
}
if (OB_FAIL(sort_->set_child(0, *input_table_))) {}
else if (OB_FAIL(sort_init(*sort_))) {}
return ret;
}
@ -167,17 +169,22 @@ public:
sort_->set_local_merge_sort(true);
}
static int init(int64_t row_count, int64_t mem_limit, int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2,
ObCollationType cs_type2)
static int init(int64_t row_count, int64_t mem_limit,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
{
return init(row_count, mem_limit, [&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, false, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true, ObMaxType, default_asc_direction()))) {
}
return ret;
});
return init(row_count, mem_limit,
[&](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, false,
ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true,
ObMaxType, default_asc_direction()))) {
}
return ret;
}
);
}
static void reset()
{
@ -191,96 +198,83 @@ public:
input_table_->reuse();
row_count_ = -1;
}
private:
ObSortPlan();
static void cons_run_filename(ObString& filename)
static void cons_run_filename(ObString &filename)
{
char* filename_buf = (char*)"ob_sort_test.run";
char *filename_buf = (char *)"ob_sort_test.run";
filename.assign_ptr(filename_buf, (int32_t)strlen(filename_buf));
return;
}
public:
private:
static ObPhysicalPlan* physical_plan_;
static ObFakeTable* input_table_;
static ObSort* sort_;
static ObPhysicalPlan *physical_plan_;
static ObFakeTable *input_table_;
static ObSort *sort_;
static int64_t row_count_;
static ObString* filename_;
static ObString *filename_;
};
ObSortTest::ObSortTest() : blocksstable::TestDataFilePrepare("TestDiskIR", 2 << 20, 2000)
{}
ObSortTest::ObSortTest()
: blocksstable::TestDataFilePrepare(&getter, "TestDiskIR", 2<<20, 2000)
{
}
ObSortTest::~ObSortTest()
{}
{
}
int ObSortTest::init_tenant_mgr()
{
int ret = OB_SUCCESS;
ObTenantManager& tm = ObTenantManager::get_instance();
ObAddr self;
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
oceanbase::obrpc::ObCommonRpcProxy rs_rpc_proxy;
oceanbase::share::ObRsMgr rs_mgr;
int64_t tenant_id = 1;
int64_t tenant_id = OB_SYS_TENANT_ID;
self.set_ip_addr("127.0.0.1", 8086);
ret = ObTenantConfigMgr::get_instance().add_tenant_config(tenant_id);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(tenant_id);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.set_tenant_mem_limit(tenant_id, 2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SYS_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
ret = getter.add_tenant(tenant_id,
2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
const int64_t ulmt = 128LL << 30;
const int64_t llmt = 128LL << 30;
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
ret = getter.add_tenant(OB_SERVER_TENANT_ID,
ulmt,
llmt);
EXPECT_EQ(OB_SUCCESS, ret);
oceanbase::lib::set_memory_limit(128LL << 32);
return ret;
}
#define BEGIN_THREAD_CODE_V2(num) \
{ \
std::vector<std::thread*> _threads; \
for (int _i = 0; _i < (num); _i++) _threads.push_back(new std::thread([&]
#define END_THREAD_CODE_V2() )); \
for (auto t : _threads) \
t->join(); \
}
#define BEGIN_THREAD_CODE_V2(num) {std::vector<std::thread *> _threads; for (int _i = 0; _i < (num); _i++) _threads.push_back(new std::thread([&]
#define END_THREAD_CODE_V2() )); for (auto t : _threads) t->join();}
template <typename SortInit>
void ObSortTest::sort_test(
int64_t row_count, int64_t mem_limit, SortInit sort_init, int64_t verify_row_cnt, bool local_merge_sort)
template<typename SortInit>
void ObSortTest::sort_test(int64_t row_count, int64_t mem_limit, SortInit sort_init, int64_t verify_row_cnt, bool local_merge_sort)
{
ASSERT_EQ(OB_SUCCESS, ObSortPlan::init(row_count, mem_limit, sort_init));
if (local_merge_sort) {
ObSortPlan::set_local_merge_sort();
}
BEGIN_THREAD_CODE_V2(2)
{
BEGIN_THREAD_CODE_V2(2) {
ObExecContext exec_ctx;
ASSERT_EQ(OB_SUCCESS, exec_ctx.init_phy_op(2));
ASSERT_EQ(OB_SUCCESS, exec_ctx.create_physical_plan_ctx());
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
my_session.test_init(0,0,0,NULL);
my_session.init_tenant("sys", 1);
exec_ctx.set_my_session(&my_session);
THIS_WORKER.set_timeout_ts(ObTimeUtility::current_time() + 600000000);
// do sort.
ObSort& sort = ObSortPlan::get_instance();
ObSort &sort = ObSortPlan::get_instance();
ASSERT_EQ(OB_SUCCESS, sort.open(exec_ctx));
auto& sort_columns = sort.get_sort_columns();
auto &sort_columns = sort.get_sort_columns();
ObSEArray<ObOpSchemaObj, 8> op_schema_objs;
cons_op_schema_objs(sort_columns, op_schema_objs);
@ -288,13 +282,13 @@ void ObSortTest::sort_test(
ObObj pre[sort_columns.count()];
char varchar_buf[1024];
const ObNewRow* row = NULL;
const ObNewRow *row = NULL;
int64_t cnt = verify_row_cnt > 0 ? verify_row_cnt : row_count;
for (int64_t i = 0; i < cnt; ++i) {
ASSERT_EQ(OB_SUCCESS, sort.get_next_row(exec_ctx, row)) << i;
// check order
for (int64_t j = 0; i > 0 && j < sort_columns.count(); j++) {
auto& col = sort_columns.at(j);
auto &col = sort_columns.at(j);
int cmp = pre[j].compare(row->cells_[col.index_], col.cs_type_);
if (cmp != 0) {
ASSERT_TRUE(col.is_ascending() ? cmp < 0 : cmp > 0);
@ -305,7 +299,7 @@ void ObSortTest::sort_test(
int64_t pos = 0;
for (int64_t j = 0; j < sort_columns.count(); j++) {
pre[j] = row->cells_[sort_columns.at(j).index_];
auto& c = pre[j];
auto &c = pre[j];
if (c.is_string_type()) {
auto len = std::min((uint64_t)c.val_len_, sizeof(varchar_buf) - pos);
MEMCPY(varchar_buf + pos, c.v_.string_, len);
@ -313,7 +307,7 @@ void ObSortTest::sort_test(
pos += len;
}
}
} // end for
} // end for
ASSERT_EQ(OB_ITER_END, sort.get_next_row(exec_ctx, row));
@ -323,46 +317,51 @@ void ObSortTest::sort_test(
ASSERT_EQ(row_count, sort_row_count);
ob_print_mod_memory_usage();
ObTenantManager::get_instance().print_tenant_usage();
// TODO: not supported now.
// ObTenantManager::get_instance().print_tenant_usage();
}
END_THREAD_CODE_V2();
ObSortPlan::reset();
}
void ObSortTest::sort_test(int64_t row_count, int64_t mem_limit, int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
void ObSortTest::sort_test(int64_t row_count, int64_t mem_limit,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
{
sort_test(row_count, mem_limit, [&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, false, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true, ObMaxType, default_asc_direction()))) {
}
return ret;
});
}
void ObSortTest::local_merge_sort_test(int64_t row_count, int64_t mem_limit, int64_t sort_col1,
ObCollationType cs_type1, int64_t sort_col2, ObCollationType cs_type2)
{
sort_test(
row_count,
mem_limit,
[&](ObSort& sort) {
sort_test(row_count, mem_limit,
[&](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, false,
ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true,
ObMaxType, default_asc_direction()))) {
}
return ret;
},
-1,
true);
}
);
}
void ObSortTest::local_merge_sort_test(int64_t row_count, int64_t mem_limit,
int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
{
sort_test(row_count, mem_limit,
[&](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, true,
ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true,
ObMaxType, default_asc_direction()))) {
}
return ret;
}, -1, true);
}
void ObSortTest::serialize_test()
{
ObSort& sort_1 = ObSortPlan::get_instance();
ObSort &sort_1 = ObSortPlan::get_instance();
ObArenaAllocator alloc;
ObSort sort_2(alloc);
const int64_t MAX_SERIALIZE_BUF_LEN = 1024;
@ -374,14 +373,14 @@ void ObSortTest::serialize_test()
ASSERT_EQ(pos, sort_1.get_serialize_size());
int64_t data_len = pos;
sort_2.set_phy_plan(const_cast<ObPhysicalPlan*>(sort_1.get_phy_plan()));
sort_2.set_phy_plan(const_cast<ObPhysicalPlan *>(sort_1.get_phy_plan()));
pos = 0;
ASSERT_EQ(OB_SUCCESS, sort_2.deserialize(buf, data_len, pos));
ASSERT_EQ(pos, data_len);
const char* str_1 = to_cstring(sort_1);
const char* str_2 = to_cstring(sort_2);
ASSERT_EQ(0, strcmp(str_1, str_2)) << "sort_1: " << to_cstring(sort_1) << std::endl
<< "sort_2: " << to_cstring(sort_2);
const char *str_1 = to_cstring(sort_1);
const char *str_2 = to_cstring(sort_2);
ASSERT_EQ(0, strcmp(str_1, str_2)) << "sort_1: " << to_cstring(sort_1)
<< std::endl << "sort_2: " << to_cstring(sort_2);
ObSortPlan::reuse();
}
@ -390,21 +389,21 @@ void ObSortTest::sort_exception_test(int expect_ret)
{
int ret = OB_SUCCESS;
ObExecContext exec_ctx;
const ObNewRow* row = NULL;
const ObNewRow *row = NULL;
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
my_session.test_init(0,0,0,NULL);
my_session.init_tenant("sys", 1);
exec_ctx.set_my_session(&my_session);
ASSERT_EQ(OB_SUCCESS, exec_ctx.init_phy_op(2));
ASSERT_EQ(OB_SUCCESS, exec_ctx.create_physical_plan_ctx());
ObSort& sort = ObSortPlan::get_instance();
ObSort &sort = ObSortPlan::get_instance();
ObSortPlan::reset();
int64_t row_count = 16 * 1024;
if (OB_FAIL(ObSortPlan::init(row_count, 0, 0, CS_TYPE_UTF8MB4_BIN, 1, CS_TYPE_UTF8MB4_BIN))) {
} else if (OB_FAIL(sort.open(exec_ctx))) {
} else {
if (OB_FAIL(ObSortPlan::init(row_count, 0, 0, CS_TYPE_UTF8MB4_BIN, 1, CS_TYPE_UTF8MB4_BIN))) {}
else if (OB_FAIL(sort.open(exec_ctx))) {}
else {
ObSEArray<ObOpSchemaObj, 8> op_schema_objs;
cons_op_schema_objs(sort.get_sort_columns(), op_schema_objs);
sort.get_op_schema_objs_for_update().assign(op_schema_objs);
@ -413,9 +412,9 @@ void ObSortTest::sort_exception_test(int expect_ret)
}
if (OB_ITER_END == ret) {
int64_t sort_row_count = 0;
if (OB_FAIL(sort.close(exec_ctx))) {
} else if (OB_FAIL(sort.get_sort_row_count(exec_ctx, sort_row_count))) {
} else {
if (OB_FAIL(sort.close(exec_ctx))) {}
else if (OB_FAIL(sort.get_sort_row_count(exec_ctx, sort_row_count))) {}
else {
ASSERT_EQ(row_count, sort_row_count);
}
}
@ -448,12 +447,12 @@ TEST_F(ObSortTest, varchar_int_item_merge_sort_test)
int64_t sort_col2 = 1;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(128 * 1024, 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(128* 1024, 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256 * 1024, 4 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
// recursive merge needed.
sort_test(256 * 1024, 512 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
cs_type2 = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(128 * 1024, 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(128* 1024, 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256 * 1024, 4 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
}
@ -480,77 +479,72 @@ TEST_F(ObSortTest, int_int_item_merge_sort_test)
TEST_F(ObSortTest, test_conf)
{
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(128 * 1024 * 1024);
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(128* 1024 * 1024);
int64_t sort_mem = 0;
sort_mem = TEST_SORT_DUMP_GET_HASH_AREA_SIZE();
ASSERT_EQ((128 * 1024 * 1024), sort_mem);
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(1 * 1024 * 1024 * 1024);
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(1 * 1024* 1024 * 1024);
sort_mem = TEST_SORT_DUMP_GET_HASH_AREA_SIZE();
ASSERT_EQ((1024 * 1024 * 1024), sort_mem);
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(128 * 1024 * 1024);
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(128* 1024 * 1024);
}
TEST_F(ObSortTest, prefix_sort_test1)
{
sort_test(1024 * 1024, 1024 * 1024, [](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_BINARY, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
sort_test(1024 * 1024, 1024 * 1024, [](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL1_ROW_ID, CS_TYPE_BINARY, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
}
TEST_F(ObSortTest, prefix_sort_test3)
{
sort_test(256 * 1024, 1024 * 1024, [](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL5_ROW_ID_DIV_3, CS_TYPE_BINARY, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
sort_test(256 * 1024, 1024 * 1024, [](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL5_ROW_ID_DIV_3, CS_TYPE_BINARY, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
}
TEST_F(ObSortTest, prefix_merge_sort_test)
{
sort_test(256 * 1024, 1024 * 1024, [](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL11_ROW_ID_MULTIPLY_3_DIV_COUNT,
CS_TYPE_BINARY,
true,
ObMaxType,
default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
sort_test(256 * 1024, 1024 * 1024, [](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL11_ROW_ID_MULTIPLY_3_DIV_COUNT,
CS_TYPE_BINARY,
true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL0_RAND_STR,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL1_ROW_ID,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
}
class MockExpr : public ObSqlExpression {
class MockExpr : public ObSqlExpression
{
public:
MockExpr(ObIAllocator& alloc, int64_t cnt) : ObSqlExpression(alloc)
MockExpr(ObIAllocator &alloc, int64_t cnt) : ObSqlExpression(alloc)
{
set_item_count(1);
start_gen_infix_exr();
@ -560,7 +554,7 @@ public:
add_expr_item(item);
}
int calc(common::ObExprCtx&, const common::ObNewRow&, common::ObObj& result) const
int calc(common::ObExprCtx &, const common::ObNewRow &, common::ObObj &result) const
{
result.set_int(get_expr_items().at(0).get_obj().get_int());
return OB_SUCCESS;
@ -572,16 +566,15 @@ TEST_F(ObSortTest, topn_sort_test)
ObArenaAllocator alloc;
MockExpr expr(alloc, 4);
sort_test(
20,
10 << 20,
[&](ObSort& sort) {
sort_test(20, 10 << 20, [&](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL0_RAND_STR,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL1_ROW_ID,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_topn_expr(&expr);
}
@ -595,16 +588,15 @@ TEST_F(ObSortTest, topn_disk_sort_test)
ObArenaAllocator alloc;
MockExpr expr(alloc, 4);
sort_test(
20,
0,
[&](ObSort& sort) {
sort_test(20, 0, [&](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL0_RAND_STR,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL1_ROW_ID,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_topn_expr(&expr);
}
@ -634,47 +626,44 @@ TEST_F(ObSortTest, local_merge_sort_disk_test)
ObArenaAllocator alloc;
MockExpr expr(alloc, 4);
sort_test(
256 * 1024,
1 << 20,
[&](ObSort& sort) {
sort_test(256 * 1024, 1 << 20, [&](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL0_RAND_STR,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL1_ROW_ID,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_topn_expr(&expr);
}
return ret;
},
4,
true);
4, true);
}
TEST_F(ObSortTest, local_merge_sort_topn_test)
{
ObArenaAllocator alloc;
MockExpr expr(alloc, 4);
sort_test(
1024,
10 << 20,
[&](ObSort& sort) {
sort_test(1024, 10 << 20, [&](ObSort &sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL0_RAND_STR,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL1_ROW_ID,
CS_TYPE_UTF8MB4_BIN,
true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_topn_expr(&expr);
}
return ret;
},
4,
true);
4, true);
}
TEST_F(ObSortTest, ser)
@ -683,11 +672,11 @@ TEST_F(ObSortTest, ser)
}
#define SORT_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
ASSERT_FALSE(HasFatalFailure()); \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
ASSERT_FALSE(HasFatalFailure()); \
} while (0)
TEST_F(ObSortTest, sort_exception)
@ -717,13 +706,13 @@ TEST_F(ObSortTest, sort_exception)
SORT_EXCEPTION_TEST("ob_sort.cpp", "get_sort_row_count", "t1", 1, OB_ERR_UNEXPECTED);
}
ObPhysicalPlan* ObSortPlan::physical_plan_ = nullptr;
ObSort* ObSortPlan::sort_ = nullptr;
ObFakeTable* ObSortPlan::input_table_ = nullptr;
ObPhysicalPlan *ObSortPlan::physical_plan_ = nullptr;
ObSort *ObSortPlan::sort_ = nullptr;
ObFakeTable *ObSortPlan::input_table_ = nullptr;
int64_t ObSortPlan::row_count_ = -1;
ObString* ObSortPlan::filename_ = nullptr;
ObString *ObSortPlan::filename_ = nullptr;
int main(int argc, char** argv)
int main(int argc, char **argv)
{
ObClockGenerator::init();
@ -735,19 +724,19 @@ int main(int argc, char** argv)
oceanbase::observer::ObSignalHandle::change_signal_mask();
signal_handle.start();
void* buf = nullptr;
void *buf = nullptr;
ObArenaAllocator allocator;
buf = allocator.alloc(sizeof(ObPhysicalPlan));
ObSortPlan::physical_plan_ = new (buf) ObPhysicalPlan();
ObSortPlan::physical_plan_ = new(buf)ObPhysicalPlan();
buf = allocator.alloc(sizeof(ObSort));
ObSortPlan::sort_ = new (buf) ObSort(ObSortPlan::physical_plan_->get_allocator());
ObSortPlan::sort_ = new (buf)ObSort(ObSortPlan::physical_plan_->get_allocator());
buf = allocator.alloc(sizeof(ObFakeTable));
ObSortPlan::input_table_ = new (buf) ObFakeTable();
ObSortPlan::input_table_ = new(buf)ObFakeTable();
ObSortPlan::row_count_ = -1;
buf = allocator.alloc(sizeof(ObString));
ObSortPlan::filename_ = new (buf) ObString();
ObSortPlan::filename_ = new (buf)ObString();
::testing::InitGoogleTest(&argc, argv);
::testing::InitGoogleTest(&argc,argv);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
return RUN_ALL_TESTS();
}

1353
unittest/sql/engine/sort/ob_specific_columns_sort_test.cpp
View File

File diff suppressed because it is too large Load Diff

154
unittest/sql/engine/sort/test_sort_impl.cpp
View File

@ -27,20 +27,26 @@
#include "storage/blocksstable/ob_tmp_file.h"
#include "sql/engine/join/join_data_generator.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include "share/ob_simple_mem_limit_getter.h"
#include <thread>
#include <vector>
#include <gtest/gtest.h>
using namespace oceanbase;
using namespace oceanbase::sql;
using namespace oceanbase::common;
using namespace oceanbase::omt;
static ObSimpleMemLimitGetter getter;
class TestSortImpl : public blocksstable::TestDataFilePrepare {
class TestSortImpl : public blocksstable::TestDataFilePrepare
{
public:
TestSortImpl() : blocksstable::TestDataFilePrepare("TestDiskIR", 2 << 20, 1000), data_(alloc_)
{}
TestSortImpl() : blocksstable::TestDataFilePrepare(&getter,
"TestDiskIR", 2<<20, 1000), data_(alloc_)
{
}
void set_sort_area_size(int64_t size)
{
int ret = OB_SUCCESS;
@ -60,9 +66,8 @@ public:
ASSERT_EQ(OB_SUCCESS, init_tenant_mgr());
blocksstable::TestDataFilePrepare::SetUp();
ASSERT_EQ(OB_SUCCESS, blocksstable::ObTmpFileManager::get_instance().init());
ASSERT_EQ(OB_SUCCESS, blocksstable::ObTmpFileManager::get_instance().start());
for (int64_t i = 0; i < ARRAYSIZEOF(cols_); i++) {
ObSortColumn& sc = cols_[i];
ObSortColumn &sc = cols_[i];
sc.cs_type_ = CS_TYPE_UTF8MB4_BIN;
sc.set_is_ascending(true);
}
@ -84,7 +89,6 @@ public:
int init_tenant_mgr()
{
int ret = OB_SUCCESS;
ObTenantManager& tm = ObTenantManager::get_instance();
ObAddr self;
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
@ -93,19 +97,14 @@ public:
self.set_ip_addr("127.0.0.1", 8086);
ret = ObTenantConfigMgr::get_instance().add_tenant_config(tenant_id_);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(tenant_id_);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.set_tenant_mem_limit(tenant_id_, 2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SYS_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
ret = getter.add_tenant(tenant_id_,
2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
const int64_t ulmt = 128LL << 30;
const int64_t llmt = 128LL << 30;
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
ret = getter.add_tenant(OB_SERVER_TENANT_ID,
ulmt,
llmt);
EXPECT_EQ(OB_SUCCESS, ret);
oceanbase::lib::set_memory_limit(128LL << 32);
return ret;
@ -113,14 +112,13 @@ public:
void destroy_tenant_mgr()
{
ObTenantManager::get_instance().destroy();
}
template <typename T>
void verify_sort(T& sort, int64_t cols, bool unique)
void verify_sort(T &sort, int64_t cols, bool unique)
{
ObExecContext ctx;
const ObNewRow* r = NULL;
const ObNewRow *r = NULL;
int64_t rows = data_.row_cnt_;
for (int64_t i = 0; i < rows; i++) {
ASSERT_EQ(OB_SUCCESS, data_.get_next_row(ctx, r));
@ -162,7 +160,7 @@ public:
if (i > 0) {
int cmp = 0;
for (int64_t i = 0; cmp == 0 && i < cols; i++) {
auto& col = cols_[i];
auto &col = cols_[i];
cmp = prev.get_cell(col.index_).compare(r->get_cell(col.index_), col.cs_type_);
if (cmp != 0) {
if (!col.is_ascending()) {
@ -188,86 +186,86 @@ public:
ObArenaAllocator alloc_;
JoinDataGenerator data_;
ObSortColumn cols_[3];
uint64_t tenant_id_ = 1;
uint64_t tenant_id_ = OB_SYS_TENANT_ID;
};
TEST_F(TestSortImpl, sort_impl)
{
ObSortImpl sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 20; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.idx_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObSortImpl sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 20; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.idx_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObArrayHelper<ObSortColumn> helper;
helper.init(2, cols_, 2);
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, false, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
ObArrayHelper<ObSortColumn> helper;
helper.init(2, cols_, 2);
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, false, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.test_init();
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.test_init();
sort.reuse();
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
sort.reuse();
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
}
TEST_F(TestSortImpl, local_order)
{
ObSortImpl sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 20; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObSortImpl sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 20; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObArrayHelper<ObSortColumn> helper;
helper.init(2, cols_, 2);
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, true, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
ObArrayHelper<ObSortColumn> helper;
helper.init(2, cols_, 2);
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, true, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 10000; };
data_.test_init();
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 10000; };
data_.test_init();
sort.reset();
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, true, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
sort.reset();
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, true, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
}
TEST_F(TestSortImpl, unique)
{
ObUniqueSort sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 3; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.idx_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObUniqueSort sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 3; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.idx_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObArrayHelper<ObSortColumn> helper;
helper.init(1, cols_, 1);
ASSERT_EQ(0, sort.init(tenant_id_, helper, true));
verify_sort(sort, 1, true);
ASSERT_FALSE(HasFatalFailure());
ObArrayHelper<ObSortColumn> helper;
helper.init(1, cols_, 1);
ASSERT_EQ(0, sort.init(tenant_id_, helper, true));
verify_sort(sort, 1, true);
ASSERT_FALSE(HasFatalFailure());
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 50000; };
data_.test_init();
cols_[0].index_ = JoinDataGenerator::IDX_CELL;
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 50000; };
data_.test_init();
cols_[0].index_ = JoinDataGenerator::IDX_CELL;
sort.reset();
ASSERT_EQ(0, sort.init(tenant_id_, helper, true));
verify_sort(sort, 1, true);
ASSERT_FALSE(HasFatalFailure());
sort.reset();
ASSERT_EQ(0, sort.init(tenant_id_, helper, true));
verify_sort(sort, 1, true);
ASSERT_FALSE(HasFatalFailure());
}
int main(int argc, char** argv)
int main(int argc, char **argv)
{
ObClockGenerator::init();
@ -279,7 +277,7 @@ int main(int argc, char** argv)
oceanbase::observer::ObSignalHandle::change_signal_mask();
signal_handle.start();
::testing::InitGoogleTest(&argc, argv);
::testing::InitGoogleTest(&argc,argv);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
return RUN_ALL_TESTS();
}