hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

init push

Browse Source
This commit is contained in:
oceanbase-admin
2021-05-31 22:56:52 +08:00
commit cea7de1475
7020 changed files with 5689869 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
unittest/sql/engine/sort/CMakeLists.txt Normal file
View File

@ -0,0 +1,10 @@
function(sort_unittest case)
ob_unittest(${ARGV})
target_sources(${case} PRIVATE ob_fake_table.h ob_fake_table.cpp)
endfunction()
sort_unittest(ob_specific_columns_sort_test)
sort_unittest(ob_in_memory_sort_test)
sort_unittest(ob_sort_test)
sort_unittest(ob_merge_sort_test)
sort_unittest(test_sort_impl)

247
unittest/sql/engine/sort/ob_fake_table.cpp Normal file
View File

@ -0,0 +1,247 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ob_fake_table.h"
#include "lib/utility/utility.h"
#include "lib/allocator/ob_malloc.h"
#include "sql/engine/ob_exec_context.h"
// using namespace oceanbase::sql;
// using namespace oceanbase::sql::test;
// using namespace oceanbase::common;
namespace oceanbase {
using namespace common;
namespace sql {
namespace test {
ObFakeTable::ObFakeTable() : ObPhyOperator(alloc_), row_count_(0), n_segments_local_merge_sort_(0)
{
set_column_count(COLUMN_COUNT);
}
ObFakeTable::~ObFakeTable()
{}
void ObFakeTable::set_row_count(const int64_t count)
{
row_count_ = count;
}
ObPhyOperator* ObFakeTable::get_child(int32_t child_idx) const
{
UNUSED(child_idx);
return NULL;
}
int32_t ObFakeTable::get_child_num() const
{
return 0;
}
int ObFakeTable::set_child(int32_t child_idx, ObPhyOperator& child_operator)
{
UNUSED(child_idx);
UNUSED(child_operator);
return OB_SUCCESS;
}
int ObFakeTable::inner_open(ObExecContext& exec_ctx) const
{
int ret = OB_SUCCESS;
if (OB_SUCCESS != (ret = init_op_ctx(exec_ctx))) {
_OB_LOG(WARN, "failed to create fake table ctx, ret=%d", ret);
}
return ret;
}
int ObFakeTable::inner_create_operator_ctx(ObExecContext& exec_ctx, ObPhyOperatorCtx*& op_ctx) const
{
int ret = OB_SUCCESS;
ObFakeTableCtx* table_ctx = NULL;
if (OB_FAIL(CREATE_PHY_OPERATOR_CTX(ObFakeTableCtx, exec_ctx, get_id(), get_type(), table_ctx))) {
_OB_LOG(WARN, "failed to create fake table ctx, ret=%d", ret);
} else {
op_ctx = table_ctx;
}
return ret;
}
int ObFakeTable::init_op_ctx(ObExecContext& ctx) const
{
int ret = OB_SUCCESS;
ObPhyOperatorCtx* op_ctx = NULL;
if (OB_SUCCESS != (ret = inner_create_operator_ctx(ctx, op_ctx))) {
_OB_LOG(WARN, "create operator context failed, ret=%d", ret);
} else if (OB_SUCCESS != (ret = op_ctx->create_cur_row(get_column_count(), projector_, projector_size_))) {
_OB_LOG(WARN, "create cur_row failed, ret=%d", ret);
}
return ret;
}
int ObFakeTable::inner_get_next_row(ObExecContext& exec_ctx, const ObNewRow*& row) const
{
int ret = OB_SUCCESS;
ObFakeTableCtx* table_ctx = NULL;
if (NULL == (table_ctx = GET_PHY_OPERATOR_CTX(ObFakeTableCtx, exec_ctx, get_id()))) {
_OB_LOG(WARN, "failed to get table ctx, ret=%d", ret);
} else if (table_ctx->get_count_ < row_count_) {
if (n_segments_local_merge_sort_ <= 0) {
if (OB_SUCCESS != (ret = cons_cur_row(table_ctx, table_ctx->get_count_))) {
_OB_LOG(WARN, "failed to cons current row, err=%d", ret);
} else {
row = &table_ctx->cur_row_;
++table_ctx->get_count_;
}
} else {
int64_t count_per = row_count_ / n_segments_local_merge_sort_;
int64_t get_count = table_ctx->get_count_ % count_per;
if (table_ctx->get_count_ >= (n_segments_local_merge_sort_ - 1) * count_per) {
get_count = table_ctx->get_count_ - (n_segments_local_merge_sort_ - 1) * count_per;
}
if (OB_SUCCESS != (ret = cons_cur_row(table_ctx, get_count))) {
_OB_LOG(WARN, "failed to cons current row, err=%d", ret);
} else {
row = &table_ctx->cur_row_;
++table_ctx->get_count_;
}
}
} else {
_OB_LOG(INFO, "end of table");
ret = OB_ITER_END;
}
return ret;
}
/***************************************************************************************************
c0 | c1 | c2 | c3 | c4 | c5 | c6 | c7 |
-----------------------------------------------------------------------------------------------------
rand str | row_idx | row_idx%2 | row_idx%3 | row_idx/2 | row_idx/3 | c1+c2 |c3+c4 |
***************************************************************************************************/
int ObFakeTable::cons_cur_row(ObFakeTableCtx* table_ctx, const int64_t row_idx) const
{
int ret = OB_SUCCESS;
int64_t c2_val = 0, c3_val = 0, c4_val = 0, c5_val = 0, c6_val = 0;
ObObj cell;
if (OB_SUCCESS != (ret = cons_varchar_cell(table_ctx, cell))) {
_OB_LOG(WARN, "failed to cons varchar cell");
} else {
// column 0: varchar
table_ctx->cur_row_.cells_[COL0_RAND_STR] = cell;
// column 1: int, row_idx
cell.set_int(row_idx);
table_ctx->cur_row_.cells_[COL1_ROW_ID] = cell;
// column 2: int, row_idx % 2
cell.set_int(c2_val = row_idx % 2);
table_ctx->cur_row_.cells_[COL2_ROW_ID_MOD_2] = cell;
// column 3: int, row_idx % 3
cell.set_int(c3_val = row_idx % 3);
table_ctx->cur_row_.cells_[COL3_ROW_ID_MOD_3] = cell;
// column 4 int, row_idx / 2, e.g. 0,0,1,1,2,2,3,3,...
cell.set_int(c4_val = row_idx / 2);
table_ctx->cur_row_.cells_[COL4_ROW_ID_DIV_2] = cell;
// column 5 int, row_idx / 3, e.g. 0,0,0,1,1,1,2,2,2,...
cell.set_int(c5_val = row_idx / 3);
table_ctx->cur_row_.cells_[COL5_ROW_ID_DIV_3] = cell;
// column 6 int, c2+c3
cell.set_int(c6_val = c2_val + c3_val);
table_ctx->cur_row_.cells_[COL6_SUM_COL2_COL3] = cell;
// column 7 int, c4+c5
cell.set_int(c6_val = c4_val + c5_val);
table_ctx->cur_row_.cells_[COL7_SUM_COL5_COL5] = cell;
// column 8: column with null values, value is null for each even row
(0 == row_idx % 2) ? cell.set_null() : cell.set_int(row_idx);
table_ctx->cur_row_.cells_[COL8_ROW_ID_OR_NULL] = cell;
// column 9 int, row_idx/2*2, e.g. 0,0,2,2,4,4,6,6,...
cell.set_int(row_idx / 2 * 2);
table_ctx->cur_row_.cells_[COL9_ROW_ID_DIV_2_MULTIPLY_2] = cell;
// column 10 int, row_idx/3*3, e.g. 0,0,0,3,3,3,6,6,6,...
cell.set_int(row_idx / 3 * 3);
table_ctx->cur_row_.cells_[COL10_ROW_ID_DIV_3_MULTIPLY_3] = cell;
// column 11 int, row_index_ * 3 / row_count_
cell.set_int(row_idx * 3 / row_count_);
table_ctx->cur_row_.cells_[COL11_ROW_ID_MULTIPLY_3_DIV_COUNT] = cell;
// column 12-15: int, random data
for (int64_t cell_idx = COL12_RAND_INT; cell_idx < COLUMN_COUNT; ++cell_idx) {
if (OB_SUCCESS != (ret = cons_random_int_cell(cell))) {
break;
} else {
table_ctx->cur_row_.cells_[cell_idx] = cell;
}
}
}
return ret;
}
#define A_Z_CNT (26)
#define GER_CNT (4) // ä, ö, ü, ß.
int ObFakeTable::cons_varchar_cell(ObFakeTableCtx* table_ctx, ObObj& cell) const
{
int ret = OB_SUCCESS;
int charnum = rand_int(ObFakeTableCtx::VARCHAR_CELL_BUF_SIZE - 5) + 4;
for (int i = 0; i < charnum; ++i) {
int rand = (i < charnum - 1) ? rand_int(A_Z_CNT + 2 * GER_CNT) : rand_int(A_Z_CNT);
if (rand < A_Z_CNT) {
table_ctx->buf_[i] = (char)('A' + rand);
} else {
table_ctx->buf_[i++] = (char)(0xC3);
switch (rand) {
case 26:
case 27:
table_ctx->buf_[i] = (char)(0xA4);
break; // ä
case 28:
case 29:
table_ctx->buf_[i] = (char)(0xB6);
break; // ö
case 30:
case 31:
table_ctx->buf_[i] = (char)(0xBC);
break; // ü
case 32:
case 33:
table_ctx->buf_[i] = (char)(0x9F);
break; // ß
default:
break;
}
}
}
table_ctx->buf_[charnum] = 0;
ObString varchar;
varchar.assign_ptr(table_ctx->buf_, charnum);
cell.set_varchar(varchar);
cell.set_collation_type(common::CS_TYPE_UTF8MB4_BIN);
// printf("**** rand varchar: %s\n", table_ctx->buf_);
return ret;
}
int ObFakeTable::cons_random_int_cell(ObObj& cell) const
{
int ret = OB_SUCCESS;
cell.set_int(rand());
return ret;
}
inline int ObFakeTable::rand_int(int max) const
{
double fmax = max;
int j = (int)(fmax * (rand() / (RAND_MAX + 1.0)));
return j;
}
int64_t ObFakeTable::to_string(char* buf, const int64_t buf_len) const
{
int64_t pos = 0;
databuff_printf(buf, buf_len, pos, "FakeTableForTesting\n");
return pos;
}
} // end namespace test
} // end namespace sql
} // end namespace oceanbase

117
unittest/sql/engine/sort/ob_fake_table.h Normal file
View File

@ -0,0 +1,117 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_FAKE_TABLE_H
#define _OB_FAKE_TABLE_H 1
#include "sql/engine/ob_phy_operator.h"
#include "lib/container/ob_array.h"
#include "lib/allocator/page_arena.h"
namespace oceanbase {
namespace sql {
namespace test {
static common::ObArenaAllocator alloc_;
// fake table to feed input for testing
class ObFakeTable : public ObPhyOperator {
public:
const static int64_t COL0_RAND_STR = 0;
const static int64_t COL1_ROW_ID = 1;
const static int64_t COL2_ROW_ID_MOD_2 = 2;
const static int64_t COL3_ROW_ID_MOD_3 = 3;
const static int64_t COL4_ROW_ID_DIV_2 = 4;
const static int64_t COL5_ROW_ID_DIV_3 = 5;
const static int64_t COL6_SUM_COL2_COL3 = 6;
const static int64_t COL7_SUM_COL5_COL5 = 7;
const static int64_t COL8_ROW_ID_OR_NULL = 8;
const static int64_t COL9_ROW_ID_DIV_2_MULTIPLY_2 = 9;
const static int64_t COL10_ROW_ID_DIV_3_MULTIPLY_3 = 10;
const static int64_t COL11_ROW_ID_MULTIPLY_3_DIV_COUNT = 11;
const static int64_t COL12_RAND_INT = 12;
const static int64_t COL13_RAND_INT = 13;
const static int64_t COL14_RAND_INT = 14;
const static int64_t COL15_RAND_INT = 15;
protected:
class ObFakeTableCtx : public ObPhyOperatorCtx {
friend class ObFakeTable;
public:
ObFakeTableCtx(ObExecContext& ctx) : ObPhyOperatorCtx(ctx), get_count_(0)
{
buf_[0] = '\0';
}
virtual ~ObFakeTableCtx()
{}
virtual void destroy()
{
return ObPhyOperatorCtx::destroy_base();
}
private:
static const int VARCHAR_CELL_BUF_SIZE = 32;
private:
int64_t get_count_;
char buf_[VARCHAR_CELL_BUF_SIZE];
};
public:
ObFakeTable();
virtual ~ObFakeTable();
virtual ObPhyOperatorType get_type() const
{
return PHY_INVALID;
}
void set_row_count(const int64_t count);
void reset()
{}
void reuse()
{}
virtual ObPhyOperator* get_child(int32_t child_idx) const;
virtual int32_t get_child_num() const;
virtual int set_child(int32_t child_idx, ObPhyOperator& child_operator);
virtual int inner_open(ObExecContext& exec_ctx) const;
virtual int64_t to_string(char* buf, const int64_t buf_len) const;
virtual void set_segments_local_merge_sort(int64_t n_segments_local_merge_sort)
{
n_segments_local_merge_sort_ = n_segments_local_merge_sort;
}
protected:
virtual int inner_get_next_row(ObExecContext& exec_ctx, const common::ObNewRow*& row) const;
virtual int init_op_ctx(ObExecContext& ctx) const;
private:
// types and constants
// static const int COLUMN_COUNT = 16;
static const int COLUMN_COUNT = 32;
private:
// disallow copy
ObFakeTable(const ObFakeTable& other);
ObFakeTable& operator=(const ObFakeTable& other);
// function members
int cons_cur_row(ObFakeTableCtx* table_ctx, const int64_t row_idx) const;
int cons_varchar_cell(ObFakeTableCtx* table_ctx, common::ObObj& cell) const;
int cons_random_int_cell(common::ObObj& cell) const;
int rand_int(int max) const;
virtual int inner_create_operator_ctx(ObExecContext& exec_ctx, ObPhyOperatorCtx*& op_ctx) const;
protected:
int64_t row_count_;
int64_t n_segments_local_merge_sort_;
};
} // end namespace test
} // end namespace sql
} // end namespace oceanbase
#endif /* _OB_FAKE_TABLE_H */

596
unittest/sql/engine/sort/ob_in_memory_sort_test.cpp Normal file
View File

@ -0,0 +1,596 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "sql/engine/sort/ob_base_sort.h"
#include "sql/engine/sort/ob_in_memory_topn_sort.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/container/ob_se_array.h"
#include "sql/session/ob_sql_session_info.h"
#include "sql/engine/ob_physical_plan.h"
#include <gtest/gtest.h>
#include "ob_fake_table.h"
#include "sql/engine/ob_exec_context.h"
#include "sql/ob_sql_init.h"
using namespace oceanbase::sql;
using namespace oceanbase::common;
using oceanbase::sql::test::ObFakeTable;
class TestInMemorySortTest : public ::testing::Test {
public:
struct ColumnOpt {
int64_t col;
ObCollationType cs_type;
TO_STRING_KV("col", col);
};
TestInMemorySortTest();
virtual ~TestInMemorySortTest();
private:
// disallow copy
TestInMemorySortTest(const TestInMemorySortTest& other);
TestInMemorySortTest& operator=(const TestInMemorySortTest& other);
protected:
typedef ObSArray<ObSortColumn> ObSortColumns;
void sort_test(
int64_t row_count, ObArray<const ColumnOpt*>& columns, ObBaseSort* base_sort, int64_t column_keys_pos = -1);
void prefix_sort_test(int64_t row_count, ObArray<const ColumnOpt*>& columns,
ObArray<const ColumnOpt*>& prefix_columns, int64_t column_keys_pos);
void serialize_test(int expect_ret);
void sort_exception_test(int expect_ret);
private:
int init(ObBaseSort* base_sort, ObFakeTable& input_table, int64_t row_count, ObArray<const ColumnOpt*>& columns,
ObSortColumns& sort_columns);
void cons_sort_columns(ObArray<const ColumnOpt*>& columns, ObSortColumns& sort_columns);
void cons_op_schema_objs(
const ObIArray<ObSortColumn>& sort_columns, const ObNewRow* row, ObIArray<ObOpSchemaObj>& op_schema_objs);
void copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size);
void cons_new_row(ObNewRow& row, int64_t column_count);
};
TestInMemorySortTest::TestInMemorySortTest()
{}
TestInMemorySortTest::~TestInMemorySortTest()
{}
void TestInMemorySortTest::prefix_sort_test(int64_t row_count, ObArray<const ColumnOpt*>& columns,
ObArray<const ColumnOpt*>& prefix_columns, int64_t column_keys_pos)
{
int ret = OB_SUCCESS;
ObArenaAllocator alloc;
ObFakeTable input_table;
// in_memory_sort
ObSortColumns sort_columns; // 9,10, prefix_sort column -> 9,10,11,12, pos = 2
ObBaseSort mem_sort;
ObExecContext exec_ctx;
ObPhysicalPlan physical_plan;
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
exec_ctx.set_my_session(&my_session);
cons_sort_columns(columns, sort_columns);
input_table.set_id(0);
input_table.set_row_count(row_count);
input_table.set_phy_plan(&physical_plan);
ASSERT_EQ(OB_SUCCESS, exec_ctx.init_phy_op(1));
ASSERT_EQ(OB_SUCCESS, exec_ctx.create_physical_plan_ctx());
ASSERT_EQ(OB_SUCCESS, input_table.open(exec_ctx));
ASSERT_EQ(OB_SUCCESS, mem_sort.set_sort_columns(sort_columns, 0));
ObSEArray<ObOpSchemaObj, 16> op_schema_objs;
const ObNewRow* row = NULL;
bool need_sort = false;
for (int i = 0; OB_SUCC(ret) && i < row_count; ++i) {
ASSERT_EQ(OB_SUCCESS, input_table.get_next_row(exec_ctx, row));
ASSERT_EQ(OB_SUCCESS, mem_sort.add_row(*row, need_sort));
}
cons_op_schema_objs(sort_columns, row, op_schema_objs);
ASSERT_EQ(OB_SUCCESS, mem_sort.sort_rows());
// prefix_sort
ObBaseSort prefix_sort;
ObSortColumns prefix_sort_columns;
cons_sort_columns(prefix_columns, prefix_sort_columns);
ASSERT_EQ(OB_SUCCESS, prefix_sort.set_sort_columns(prefix_sort_columns, column_keys_pos));
ObNewRow prefix_row;
cons_new_row(prefix_row, input_table.get_column_count());
ObNewRow mem_row;
cons_new_row(mem_row, input_table.get_column_count());
ObObj last_cell[20];
ObObj first_cell[20];
need_sort = false;
// const ColumnOpt *cpt = NULL;
ObSortColumn sort_col;
ASSERT_EQ(prefix_columns.count(), prefix_columns.count());
for (int i = 0; i < row_count; ++i) {
ASSERT_EQ(OB_SUCCESS, mem_sort.get_next_row(mem_row));
ASSERT_EQ(OB_SUCCESS, prefix_sort.add_row(mem_row, need_sort));
if (need_sort) {
op_schema_objs.reset();
cons_op_schema_objs(prefix_sort_columns, &mem_row, op_schema_objs);
prefix_sort.sort_rows(); // do sort
ret = prefix_sort.get_next_row(prefix_row);
// fisrt row , set last row
for (int j = 0; OB_SUCC(ret) && j < prefix_sort_columns.count(); ++j) {
sort_col = prefix_sort_columns.at(j);
last_cell[j] = prefix_row.cells_[sort_col.index_];
first_cell[j] = prefix_row.cells_[sort_col.index_];
}
// prefix sort get next row
while (OB_SUCC(ret)) {
ret = prefix_sort.get_next_row(prefix_row);
int64_t cmp = 0;
for (int j = 0; OB_SUCC(ret) && 0 == cmp && j < prefix_sort_columns.count(); ++j) {
// cpt = prefix_columns.at(j);
sort_col = prefix_sort_columns.at(j);
cmp = last_cell[j].compare(prefix_row.cells_[sort_col.index_], sort_col.cs_type_);
// SQL_ENG_LOG(WARN, "row_test", K(last_cell[j]), K(prefix_row.cells_[sort_col.index_]), K(sort_col));
if (sort_col.is_ascending()) {
ASSERT_TRUE(cmp <= 0);
} else {
ASSERT_TRUE(cmp >= 0);
}
}
// set last cell
for (int j = 0; j < prefix_sort_columns.count(); ++j) {
sort_col = prefix_sort_columns.at(j);
last_cell[j] = prefix_row.cells_[sort_col.index_];
}
}
// compare prefix cols
for (int j = 0; j < column_keys_pos; ++j) {
sort_col = prefix_sort_columns.at(j);
ASSERT_EQ(0, first_cell[j].compare(last_cell[j]));
}
ASSERT_EQ(OB_ITER_END, ret);
ret = OB_SUCCESS;
need_sort = false;
}
}
}
void TestInMemorySortTest::sort_test(
int64_t row_count, ObArray<const ColumnOpt*>& columns, ObBaseSort* base_sort, int64_t column_keys_pos)
{
ObArenaAllocator alloc;
ObFakeTable input_table;
ObSortColumns sort_columns;
UNUSED(column_keys_pos);
ASSERT_EQ(OB_SUCCESS, init(base_sort, input_table, row_count, columns, sort_columns));
// read and check
ObObj* cell[10];
ObObj last_cell[10];
for (int i = 0; i < 10; i++) {
cell[i] = NULL;
}
char varchar_buf[1024];
ObMalloc allocator;
ObNewRow row;
cons_new_row(row, input_table.get_column_count());
ASSERT_EQ(OB_SUCCESS, base_sort->get_next_row(row));
const ColumnOpt* opt = NULL;
for (int64_t j = 0; j < columns.count(); ++j) {
opt = columns.at(j);
ASSERT_TRUE(opt != NULL);
last_cell[j] = row.cells_[opt->col];
}
ASSERT_EQ(columns.count(), base_sort->get_sort_columns()->count());
ObSortColumn cn;
for (int64_t i = 1; i < row_count; ++i) {
ASSERT_EQ(OB_SUCCESS, base_sort->get_next_row(row));
// SQL_ENG_LOG(WARN, "yeti_test", K(row));
// check order
int64_t cmp = 0;
bool cmp_next = true;
for (int64_t j = 0; cmp_next && j < columns.count(); ++j) {
opt = columns.at(j);
cn = base_sort->get_sort_columns()->at(j);
ASSERT_TRUE(opt != NULL);
last_cell[j] = row.cells_[opt->col];
cell[j] = &row.cells_[opt->col];
cmp = last_cell[j].compare(*cell[j], opt->cs_type);
if (cn.is_ascending()) {
ASSERT_TRUE(cmp <= 0);
} else {
ASSERT_TRUE(cmp >= 0);
}
if (0 != cmp) {
cmp_next = false;
}
}
// set last_cell
for (int64_t j = 0; j < columns.count(); ++j) {
opt = columns.at(j);
last_cell[j] = row.cells_[opt->col];
if (ObVarcharType == last_cell[j].get_type()) {
copy_cell_varchar(last_cell[j], varchar_buf, 1024);
} else if (ObVarcharType == last_cell[j].get_type()) {
copy_cell_varchar(last_cell[j], varchar_buf, 1024);
}
}
} // end for
ASSERT_EQ(OB_ITER_END, base_sort->get_next_row(row));
}
void TestInMemorySortTest::serialize_test(int expect_ret)
{
int ret = OB_SUCCESS;
ObArray<const ColumnOpt*> columns;
ColumnOpt cpt1, cpt2;
ObSortColumns sort_columns_1;
ObSortColumns sort_columns_2;
cpt1.col = 0;
cpt2.col = 1;
cpt1.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt2.cs_type = CS_TYPE_UTF8MB4_BIN;
ASSERT_EQ(OB_SUCCESS, columns.push_back(&cpt1));
ASSERT_EQ(OB_SUCCESS, columns.push_back(&cpt2));
const int64_t MAX_SERIALIZE_BUF_LEN = 1024;
char buf[MAX_SERIALIZE_BUF_LEN] = {'\0'};
int64_t pos = 0;
cons_sort_columns(columns, sort_columns_1);
if (OB_FAIL(sort_columns_1.serialize(buf, MAX_SERIALIZE_BUF_LEN, pos))) {
} else {
ASSERT_EQ(pos, sort_columns_1.get_serialize_size());
int64_t data_len = pos;
pos = 0;
if (OB_FAIL(sort_columns_2.deserialize(buf, data_len, pos))) {
} else {
ASSERT_EQ(0, strcmp(to_cstring(sort_columns_1), to_cstring(sort_columns_2)));
}
}
if (OB_FAIL(ret)) {
ASSERT_EQ(expect_ret, ret);
}
}
void TestInMemorySortTest::sort_exception_test(int expect_ret)
{
int ret = OB_SUCCESS;
ObBaseSort in_mem_sort;
ObSortColumns sort_columns;
ObFakeTable input_table;
int64_t row_count = 1024;
ObArray<const ColumnOpt*> columns;
ColumnOpt cpt1, cpt2;
ObSortColumns sort_columns_1;
ObSortColumns sort_columns_2;
cpt1.col = 0;
cpt2.col = 1;
cpt1.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt2.cs_type = CS_TYPE_UTF8MB4_BIN;
ASSERT_EQ(OB_SUCCESS, columns.push_back(&cpt1));
ASSERT_EQ(OB_SUCCESS, columns.push_back(&cpt2));
ObNewRow row;
if (OB_FAIL(init(&in_mem_sort, input_table, row_count, columns, sort_columns))) {
} else {
cons_new_row(row, input_table.get_column_count());
for (int64_t i = 0; OB_SUCC(ret) && i < row_count; ++i) {
ret = in_mem_sort.get_next_row(row);
}
if (OB_SUCC(ret)) {
ASSERT_EQ(OB_ITER_END, in_mem_sort.get_next_row(row));
}
}
in_mem_sort.reuse();
if (OB_FAIL(ret)) {
ASSERT_EQ(expect_ret, ret);
}
}
int TestInMemorySortTest::init(ObBaseSort* base_sort, ObFakeTable& input_table, int64_t row_count,
ObArray<const ColumnOpt*>& columns, ObSortColumns& sort_columns)
{
int ret = OB_SUCCESS;
ObExecContext exec_ctx;
ObPhysicalPlan physical_plan;
const ObNewRow* row = NULL;
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
exec_ctx.set_my_session(&my_session);
cons_sort_columns(columns, sort_columns);
input_table.set_id(0);
input_table.set_row_count(row_count);
input_table.set_phy_plan(&physical_plan);
if (OB_FAIL(exec_ctx.init_phy_op(1))) {
} else if (OB_FAIL(exec_ctx.create_physical_plan_ctx())) {
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(base_sort->set_sort_columns(sort_columns, 0))) {
} else if (OB_FAIL(input_table.open(exec_ctx))) {
} else {
ObSEArray<ObOpSchemaObj, 16> op_schema_objs;
bool need_sort = false;
for (int i = 0; OB_SUCC(ret) && i < row_count; ++i) {
if (OB_FAIL(input_table.get_next_row(exec_ctx, row))) {
} else if (OB_FAIL(base_sort->add_row(*row, need_sort))) {
}
}
if (OB_FAIL(ret)) {
// do nothing
} else {
cons_op_schema_objs(sort_columns, row, op_schema_objs);
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(base_sort->sort_rows())) {
} else if (OB_FAIL(input_table.close(exec_ctx))) {
}
}
return ret;
}
void TestInMemorySortTest::cons_sort_columns(ObArray<const ColumnOpt*>& columns, ObSortColumns& sort_columns)
{
ObSortColumn sort_column;
const ColumnOpt* cpt = NULL;
for (int64_t i = 0; i < columns.count(); ++i) {
cpt = columns.at(i);
ASSERT_TRUE(cpt != NULL);
sort_column.index_ = cpt->col;
sort_column.cs_type_ = cpt->cs_type;
sort_column.set_is_ascending((i % 2) ? true : false);
ASSERT_EQ(OB_SUCCESS, sort_columns.push_back(sort_column));
}
return;
}
void TestInMemorySortTest::cons_op_schema_objs(
const ObIArray<ObSortColumn>& sort_columns, const ObNewRow* row, ObIArray<ObOpSchemaObj>& op_schema_objs)
{
if (NULL == row) {
// do nothing
} else {
for (int64_t i = 0; i < sort_columns.count(); i++) {
ObOpSchemaObj op_schema_obj(row->get_cell(sort_columns.at(i).index_).get_type());
op_schema_objs.push_back(op_schema_obj);
}
}
return;
}
void TestInMemorySortTest::copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size)
{
ObString str;
ASSERT_EQ(OB_SUCCESS, cell.get_varchar(str));
ASSERT_TRUE(str.length() < buf_size);
memcpy(buf, str.ptr(), str.length());
str.assign_ptr(buf, str.length());
cell.set_varchar(str);
return;
}
void TestInMemorySortTest::cons_new_row(ObNewRow& row, int64_t column_count)
{
row.cells_ = static_cast<ObObj*>(malloc(column_count * sizeof(ObObj)));
row.count_ = column_count;
}
TEST_F(TestInMemorySortTest, varchar_int_item)
{
ObArray<const ColumnOpt*> columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
cpt1.col = 0;
cpt2.col = 1;
cpt1.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt2.cs_type = CS_TYPE_UTF8MB4_BIN;
ASSERT_EQ(OB_SUCCESS, columns.push_back(&cpt1));
ASSERT_EQ(OB_SUCCESS, columns.push_back(&cpt2));
ObBaseSort in_mem_sort;
sort_test(1024, columns, &in_mem_sort);
ObBaseSort in_mem_sort1;
sort_test(16 * 1024, columns, &in_mem_sort1);
ObBaseSort in_mem_sort2;
sort_test(256 * 1024, columns, &in_mem_sort2);
ObBaseSort in_mem_sort3;
cpt1.cs_type = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(1024, columns, &in_mem_sort3);
ObBaseSort in_mem_sort4;
sort_test(16 * 1024, columns, &in_mem_sort4);
ObBaseSort in_mem_sort5;
sort_test(256 * 1024, columns, &in_mem_sort5);
// topn sort
for (int64_t i = 1; i < 2048; i += 100) {
ObBaseSort mem_topn_sort;
sort_test(10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(16 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(256 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
cpt1.cs_type = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(16 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(256 * 10, columns, &mem_topn_sort, i);
}
}
TEST_F(TestInMemorySortTest, int_int_equal_item)
{
ObArray<const ColumnOpt*> columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
cpt1.col = 2;
cpt2.col = 3;
cpt1.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt2.cs_type = CS_TYPE_UTF8MB4_BIN;
columns.push_back(&cpt1);
columns.push_back(&cpt2);
ObBaseSort in_mem_sort;
sort_test(1024, columns, &in_mem_sort);
ObBaseSort in_mem_sort1;
sort_test(16 * 1024, columns, &in_mem_sort1);
ObBaseSort in_mem_sort2;
sort_test(256 * 1024, columns, &in_mem_sort2);
// topn sort
for (int64_t i = 1; i < 2048; i += 100) {
ObBaseSort mem_topn_sort;
sort_test(10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(16 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(256 * 10, columns, &mem_topn_sort, i);
}
}
TEST_F(TestInMemorySortTest, int_int_item)
{
ObArray<const ColumnOpt*> columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
cpt1.col = 11;
cpt2.col = 12;
cpt1.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt2.cs_type = CS_TYPE_UTF8MB4_BIN;
columns.push_back(&cpt1);
columns.push_back(&cpt2);
ObBaseSort in_mem_sort;
sort_test(1024, columns, &in_mem_sort);
ObBaseSort in_mem_sort1;
sort_test(16 * 1024, columns, &in_mem_sort1);
ObBaseSort in_mem_sort2;
sort_test(256 * 1024, columns, &in_mem_sort2);
// topn sort
for (int64_t i = 1; i < 2048; i += 100) {
ObBaseSort mem_topn_sort;
sort_test(10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(16 * 10, columns, &mem_topn_sort, i);
mem_topn_sort.reset();
sort_test(256 * 10, columns, &mem_topn_sort, i);
}
}
TEST_F(TestInMemorySortTest, prefix_int_test)
{
ObArray<const ColumnOpt*> columns;
ObArray<const ColumnOpt*> prefix_columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
ColumnOpt cpt3;
ColumnOpt cpt4;
cpt1.col = 9;
cpt2.col = 10;
cpt3.col = 11;
cpt4.col = 12;
cpt1.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt2.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt3.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt4.cs_type = CS_TYPE_UTF8MB4_BIN;
columns.push_back(&cpt1);
columns.push_back(&cpt2);
prefix_columns.push_back(&cpt1);
prefix_columns.push_back(&cpt2);
prefix_columns.push_back(&cpt3);
prefix_columns.push_back(&cpt4);
ObBaseSort in_mem_sort;
prefix_sort_test(1024, columns, prefix_columns, 2);
ObBaseSort in_mem_sort1;
prefix_sort_test(16 * 1024, columns, prefix_columns, 2);
ObBaseSort in_mem_sort2;
prefix_sort_test(256 * 1024, columns, prefix_columns, 2);
}
TEST_F(TestInMemorySortTest, prefix_var_int_test)
{
ObArray<const ColumnOpt*> columns;
ObArray<const ColumnOpt*> prefix_columns;
ColumnOpt cpt1;
ColumnOpt cpt2;
ColumnOpt cpt3;
cpt1.col = 9;
cpt2.col = 10;
cpt3.col = 0;
cpt1.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt2.cs_type = CS_TYPE_UTF8MB4_BIN;
cpt3.cs_type = CS_TYPE_UTF8MB4_BIN;
columns.push_back(&cpt1);
columns.push_back(&cpt2);
prefix_columns.push_back(&cpt1);
prefix_columns.push_back(&cpt2);
prefix_columns.push_back(&cpt3);
ObBaseSort in_mem_sort;
prefix_sort_test(1024, columns, prefix_columns, 2);
ObBaseSort in_mem_sort1;
prefix_sort_test(16 * 1024, columns, prefix_columns, 2);
ObBaseSort in_mem_sort2;
prefix_sort_test(256 * 1024, columns, prefix_columns, 2);
}
TEST_F(TestInMemorySortTest, serialize)
{
serialize_test(OB_SUCCESS);
}
#define SORT_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
} while (0)
TEST_F(TestInMemorySortTest, sort_exception)
{
SORT_EXCEPTION_TEST("ob_base_sort.cpp", "set_sort_columns", "t1", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_base_sort.cpp", "add_row", "t1", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_base_sort.cpp", "add_row", "t3", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_base_sort.cpp", "get_next_row", "t1", OB_ERROR, OB_ERROR);
}
#define SERIALIZE_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
serialize_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
} while (0)
TEST_F(TestInMemorySortTest, serialize_exception)
{
SERIALIZE_EXCEPTION_TEST("ob_base_sort.cpp", "serialize", "t1", OB_ERROR, OB_ERROR);
SERIALIZE_EXCEPTION_TEST("ob_base_sort.cpp", "serialize", "t3", OB_ERROR, OB_ERROR);
SERIALIZE_EXCEPTION_TEST("ob_base_sort.cpp", "deserialize", "t1", OB_ERROR, OB_ERROR);
SERIALIZE_EXCEPTION_TEST("ob_base_sort.cpp", "deserialize", "t3", OB_ERROR, OB_ERROR);
}
int main(int argc, char** argv)
{
::testing::InitGoogleTest(&argc, argv);
oceanbase::common::ObLogger::get_logger().set_log_level("WARN");
return RUN_ALL_TESTS();
}

393
unittest/sql/engine/sort/ob_merge_sort_test.cpp Normal file
View File

@ -0,0 +1,393 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include "lib/allocator/ob_malloc.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/container/ob_se_array.h"
#include "share/ob_tenant_mgr.h"
#include "share/ob_srv_rpc_proxy.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "sql/engine/ob_exec_context.h"
#include "sql/ob_sql_init.h"
#include "sql/engine/sort/ob_merge_sort.h"
#include "sql/engine/ob_physical_plan.h"
#include "sql/session/ob_sql_session_info.h"
#include "ob_fake_table.h"
using namespace oceanbase::sql;
using namespace oceanbase::common;
using namespace oceanbase::storage;
using namespace oceanbase::blocksstable;
using oceanbase::sql::test::ObFakeTable;
class TestMergeSort : public ObMergeSort {
public:
TestMergeSort(ObIAllocator& alloc) : ObMergeSort(alloc)
{}
~TestMergeSort()
{}
};
class TestMergeSortTest : public oceanbase::blocksstable::TestDataFilePrepare {
static const int64_t SORT_BUF_SIZE = 2 * 1024 * 1024;
static const int64_t EXPIRE_TIMESTAMP = 0;
static const int64_t MACRO_BLOCK_SIZE = 2 * 1024 * 1024;
static const int64_t MACRO_BLOCK_COUNT = 50 * 1024;
static const uint64_t TENATN_ID = 1;
public:
TestMergeSortTest();
virtual ~TestMergeSortTest();
virtual void SetUp();
virtual void TearDown();
int init_tenant_mgr();
void destroy_tenant_mgr();
private:
// disallow copy
TestMergeSortTest(const TestMergeSortTest& other);
TestMergeSortTest& operator=(const TestMergeSortTest& other);
protected:
typedef ObSArray<ObSortColumn> ObSortColumns;
void sort_test(int64_t run_count, int64_t row_count, int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2,
ObCollationType cs_type2);
void sort_exception_test(int expect_ret);
private:
int init_op(TestMergeSort& merge_sort, ObFakeTable& input_table, ObSortColumns& sort_cols, int64_t row_count,
int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2, ObCollationType cs_type2);
int init_data(TestMergeSort& merge_sort, ObBaseSort& in_mem_sort, ObFakeTable& input_table, ObSortColumns& sort_cols,
int64_t run_count, int64_t row_count);
void cons_sort_cols(
int64_t col1, ObCollationType cs_type1, int64_t col2, ObCollationType cs_type2, ObSortColumns& sort_cols);
void copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size);
void cons_new_row(ObNewRow& row, int64_t column_count);
private:
ObPhysicalPlan physical_plan_;
ObArenaAllocator alloc_;
};
TestMergeSortTest::TestMergeSortTest() : TestDataFilePrepare("TestMergeSort", MACRO_BLOCK_SIZE, MACRO_BLOCK_COUNT)
{}
TestMergeSortTest::~TestMergeSortTest()
{}
void TestMergeSortTest::SetUp()
{
TestDataFilePrepare::SetUp();
FILE_MANAGER_INSTANCE_V2.init();
FILE_MANAGER_INSTANCE_V2.start();
ASSERT_EQ(OB_SUCCESS, init_tenant_mgr());
}
void TestMergeSortTest::TearDown()
{
alloc_.reset();
FILE_MANAGER_INSTANCE_V2.destroy();
TestDataFilePrepare::TearDown();
destroy_tenant_mgr();
}
int TestMergeSortTest::init_tenant_mgr()
{
int ret = OB_SUCCESS;
ObTenantManager& tm = ObTenantManager::get_instance();
ObAddr self;
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
oceanbase::obrpc::ObCommonRpcProxy rs_rpc_proxy;
oceanbase::share::ObRsMgr rs_mgr;
int64_t tenant_id = 1;
self.set_ip_addr("127.0.0.1", 8086);
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(tenant_id);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.set_tenant_mem_limit(tenant_id, 2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SYS_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
const int64_t ulmt = 128LL << 30;
const int64_t llmt = 128LL << 30;
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
EXPECT_EQ(OB_SUCCESS, ret);
oceanbase::lib::set_memory_limit(128LL << 32);
return ret;
}
void TestMergeSortTest::destroy_tenant_mgr()
{
ObTenantManager& tm = ObTenantManager::get_instance();
tm.destroy();
}
void TestMergeSortTest::sort_test(int64_t run_count, int64_t row_count, int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
{
ObArenaAllocator alloc;
TestMergeSort merge_sort(alloc_);
ObBaseSort in_mem_sort;
ObFakeTable input_table;
ObSortColumns sort_cols;
const ObObj* cell1 = NULL;
const ObObj* cell2 = NULL;
ObObj last_cell1;
ObObj last_cell2;
char varchar_buf[1024];
ObNewRow row;
ASSERT_EQ(OB_SUCCESS, SLOGGER.begin(OB_LOG_CS_DAILY_MERGE));
ASSERT_EQ(
OB_SUCCESS, init_op(merge_sort, input_table, sort_cols, row_count, sort_col1, cs_type1, sort_col2, cs_type2));
ASSERT_EQ(OB_SUCCESS, init_data(merge_sort, in_mem_sort, input_table, sort_cols, run_count, row_count));
ASSERT_EQ(OB_SUCCESS, merge_sort.do_merge_sort(input_table.get_column_count()));
cons_new_row(row, input_table.get_column_count());
ASSERT_EQ(OB_SUCCESS, merge_sort.get_next_row(row));
last_cell1 = row.cells_[sort_col1];
last_cell2 = row.cells_[sort_col2];
int64_t total_row_count = run_count * row_count;
// int ret = OB_SUCCESS;
for (int64_t i = 1; i < total_row_count; ++i) {
merge_sort.get_next_row(row);
cell1 = &row.cells_[sort_col1];
cell2 = &row.cells_[sort_col2];
// check order, cell1: descending, cell2: ascending.
if (0 == last_cell1.compare(*cell1, cs_type1)) {
ASSERT_TRUE(last_cell2.compare(*cell2, cs_type2) <= 0);
} else {
ASSERT_TRUE(last_cell1.compare(*cell1, cs_type1) > 0);
}
last_cell1 = *cell1;
last_cell2 = *cell2;
if (ObVarcharType == last_cell1.get_type()) {
copy_cell_varchar(last_cell1, varchar_buf, 1024);
} else if (ObVarcharType == last_cell2.get_type()) {
copy_cell_varchar(last_cell2, varchar_buf, 1024);
}
}
ASSERT_EQ(OB_ITER_END, merge_sort.get_next_row(row));
ASSERT_EQ(OB_SUCCESS, SLOGGER.abort());
return;
}
void TestMergeSortTest::sort_exception_test(int expect_ret)
{
int ret = OB_SUCCESS;
ObArenaAllocator alloc;
TestMergeSort merge_sort(alloc);
ObBaseSort in_mem_sort;
ObFakeTable input_table;
ObSortColumns sort_cols;
ObMalloc allocator;
int64_t run_count = 4;
int64_t row_count = 4096;
int64_t sort_col1 = 0;
int64_t sort_col2 = 1;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
if (OB_FAIL(init_op(merge_sort, input_table, sort_cols, row_count, sort_col1, cs_type1, sort_col2, cs_type2))) {
} else if (OB_FAIL(init_data(merge_sort, in_mem_sort, input_table, sort_cols, run_count, row_count))) {
} else if (OB_FAIL(merge_sort.do_merge_sort(input_table.get_column_count()))) {
} else {
ObNewRow row;
cons_new_row(row, input_table.get_column_count());
int64_t total_row_count = run_count * row_count;
for (int64_t i = 0; OB_SUCC(ret) && i < total_row_count; ++i) {
ret = merge_sort.get_next_row(row);
}
if (OB_SUCC(ret)) {
ASSERT_EQ(OB_ITER_END, merge_sort.get_next_row(row));
}
}
merge_sort.reuse();
if (OB_FAIL(ret)) {
ASSERT_EQ(expect_ret, ret);
}
}
int TestMergeSortTest::init_op(TestMergeSort& merge_sort, ObFakeTable& input_table, ObSortColumns& sort_cols,
int64_t row_count, int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2, ObCollationType cs_type2)
{
int ret = OB_SUCCESS;
ObString filename;
cons_sort_cols(sort_col1, cs_type1, sort_col2, cs_type2, sort_cols);
ObNewRow row;
cons_new_row(row, input_table.get_column_count());
if (OB_FAIL(merge_sort.init(sort_cols, row, TENATN_ID))) {
} else {
merge_sort.set_merge_run_count(2);
input_table.set_id(0);
input_table.set_row_count(row_count);
input_table.set_phy_plan(&physical_plan_);
}
return ret;
}
int TestMergeSortTest::init_data(TestMergeSort& merge_sort, ObBaseSort& in_mem_sort, ObFakeTable& input_table,
ObSortColumns& sort_cols, int64_t run_count, int64_t row_count)
{
int ret = OB_SUCCESS;
const ObNewRow* row = NULL;
bool need_false = false;
// loop run_count to generate dumped runs.
for (int i = 0; OB_SUCC(ret) && i < run_count; ++i) {
ObExecContext exec_ctx;
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
exec_ctx.set_my_session(&my_session);
my_session.test_init(0, 0, 0, NULL);
in_mem_sort.reuse();
if (OB_FAIL(exec_ctx.init_phy_op(1))) {
} else if (OB_FAIL(exec_ctx.create_physical_plan_ctx())) {
} else if (OB_FAIL(in_mem_sort.set_sort_columns(sort_cols, 0))) {
} else if (OB_FAIL(input_table.open(exec_ctx))) {
} else {
for (int j = 0; OB_SUCC(ret) && j < row_count; ++j) {
if (OB_FAIL(input_table.get_next_row(exec_ctx, row))) {
} else if (OB_FAIL(in_mem_sort.add_row(*row, need_false))) {
}
}
}
ObSEArray<ObOpSchemaObj, 8> op_schema_objs;
if (OB_FAIL(ret)) {
// do nothing
} else {
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(in_mem_sort.sort_rows())) {
} else if (OB_FAIL(input_table.close(exec_ctx))) {
} else if (OB_FAIL(merge_sort.dump_base_run(in_mem_sort))) {
}
} // end for i
return ret;
}
void TestMergeSortTest::cons_sort_cols(
int64_t col1, ObCollationType cs_type1, int64_t col2, ObCollationType cs_type2, ObSortColumns& sort_cols)
{
ObSortColumn sort_col;
sort_col.index_ = col1;
sort_col.cs_type_ = cs_type1;
sort_col.set_is_ascending(false);
ASSERT_EQ(OB_SUCCESS, sort_cols.push_back(sort_col));
sort_col.index_ = col2;
sort_col.cs_type_ = cs_type2;
sort_col.set_is_ascending(true);
ASSERT_EQ(OB_SUCCESS, sort_cols.push_back(sort_col));
return;
}
void TestMergeSortTest::copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size)
{
ObString str;
ASSERT_EQ(OB_SUCCESS, cell.get_varchar(str));
ASSERT_TRUE(str.length() < buf_size);
memcpy(buf, str.ptr(), str.length());
str.assign_ptr(buf, str.length());
cell.set_varchar(str);
return;
}
void TestMergeSortTest::cons_new_row(ObNewRow& row, int64_t column_count)
{
row.cells_ = static_cast<ObObj*>(malloc(column_count * sizeof(ObObj)));
row.count_ = column_count;
}
TEST_F(TestMergeSortTest, sorted_int_int_test)
{
int64_t sort_col1 = 1;
int64_t sort_col2 = 2;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(4, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 65536, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256, 4, sort_col1, cs_type1, sort_col2, cs_type2);
}
TEST_F(TestMergeSortTest, random_int_int_test)
{
int64_t sort_col1 = 12;
int64_t sort_col2 = 13;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 65536, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256, 4, sort_col1, cs_type1, sort_col2, cs_type2);
}
TEST_F(TestMergeSortTest, random_varchar_int_test)
{
int64_t sort_col1 = 0;
int64_t sort_col2 = 1;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 65536, sort_col1, cs_type1, sort_col2, cs_type2);
cs_type1 = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(4, 16, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 256, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 4096, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(16, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256, 4, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(4, 65536, sort_col1, cs_type1, sort_col2, cs_type2);
}
#define SORT_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
} while (0)
TEST_F(TestMergeSortTest, sort_exception)
{
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "init", "t1", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "dump_base_run", "t2", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "dump_base_run", "t3", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "build_merge_heap", "t3", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "build_merge_heap", "t4", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "dump_merge_run", "t4", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "dump_merge_run", "t5", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "do_one_round", "t5", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "do_one_round", "t6", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "do_merge_sort", "t6", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_merge_sort.cpp", "get_next_row", "t7", OB_ERROR, OB_ERROR);
}
int main(int argc, char** argv)
{
system("rm -f test_merge_sort.log*");
OB_LOGGER.set_file_name("test_merge_sort.log", true, true);
// OB_LOGGER.set_log_level("INFO");
::testing::InitGoogleTest(&argc, argv);
oceanbase::common::ObLogger::get_logger().set_log_level("WARN");
return RUN_ALL_TESTS();
}

753
unittest/sql/engine/sort/ob_sort_test.cpp Normal file
View File

@ -0,0 +1,753 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "sql/engine/sort/ob_sort.h"
#include "sql/session/ob_sql_session_info.h"
#include "sql/engine/ob_physical_plan.h"
#include "lib/utility/ob_test_util.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/container/ob_se_array.h"
#include <gtest/gtest.h>
#include "ob_fake_table.h"
#include "sql/engine/ob_exec_context.h"
#include "share/ob_worker.h"
#include "observer/ob_signal_handle.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include <thread>
#include <vector>
#include <gtest/gtest.h>
using namespace oceanbase;
using namespace oceanbase::sql;
using namespace oceanbase::omt;
using namespace oceanbase::common;
using oceanbase::sql::test::ObFakeTable;
#define TEST_SORT_DUMP_GET_HASH_AREA_SIZE() (get_sort_area_size())
#define TEST_SORT_DUMP_SET_HASH_AREA_SIZE(size) (set_sort_area_size(size))
class ObSortTest : public blocksstable::TestDataFilePrepare {
public:
ObSortTest();
virtual ~ObSortTest();
private:
// disallow copy
ObSortTest(const ObSortTest& other);
ObSortTest& operator=(const ObSortTest& other);
protected:
virtual void SetUp() override
{
GCONF.enable_sql_operator_dump.set_value("True");
ASSERT_EQ(OB_SUCCESS, init_tenant_mgr());
blocksstable::TestDataFilePrepare::SetUp();
}
virtual void TearDown() override
{
blocksstable::TestDataFilePrepare::TearDown();
destroy_tenant_mgr();
}
int init_tenant_mgr();
int64_t get_sort_area_size()
{
int64_t sort_area_size = 0;
int ret = OB_SUCCESS;
ret = ObSqlWorkareaUtil::get_workarea_size(SORT_WORK_AREA, OB_SYS_TENANT_ID, sort_area_size);
if (OB_FAIL(ret)) {
SQL_ENG_LOG(WARN, "failed to get hash area size", K(ret), K(sort_area_size));
}
return sort_area_size;
}
void set_sort_area_size(int64_t size)
{
int ret = OB_SUCCESS;
int64_t tenant_id = OB_SYS_TENANT_ID;
ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
if (tenant_config.is_valid()) {
tenant_config->_sort_area_size = size;
} else {
ret = OB_ERR_UNEXPECTED;
SQL_ENG_LOG(WARN, "unexpected status: config is invalid", K(tenant_id));
}
// ASSERT_EQ(OB_SUCCESS, ret);
}
void destroy_tenant_mgr()
{
ObTenantManager::get_instance().destroy();
}
template <typename SortInit>
void sort_test(int64_t row_count, int64_t mem_limit, SortInit sort_init, int64_t verify_row_cnt = -1,
bool local_merge_sort = false);
void sort_test(int64_t row_count, int64_t mem_limit, int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2,
ObCollationType cs_type2);
void local_merge_sort_test(int64_t row_count, int64_t mem_limit, int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2);
void serialize_test();
void sort_exception_test(int expect_ret);
private:
static void copy_cell_varchar(ObObj& cell, char* buf, int64_t buf_size)
{
ObString str;
ASSERT_EQ(OB_SUCCESS, cell.get_varchar(str));
ASSERT_TRUE(str.length() < buf_size);
memcpy(buf, str.ptr(), str.length());
str.assign_ptr(buf, str.length());
cell.set_varchar(str);
return;
}
void cons_op_schema_objs(const ObIArray<ObSortColumn>& sort_columns, ObIArray<ObOpSchemaObj>& op_schema_objs)
{
for (int64_t i = 0; i < sort_columns.count(); i++) {
ObOpSchemaObj op_schema_obj;
if (0 == sort_columns.at(i).index_) {
op_schema_obj.obj_type_ = common::ObVarcharType;
} else {
op_schema_obj.obj_type_ = common::ObIntType;
}
op_schema_objs.push_back(op_schema_obj);
}
return;
}
};
class ObSortPlan {
public:
static ObSort& get_instance()
{
return *sort_;
}
template <typename SortInit>
static int init(int64_t row_count, int64_t mem_limit, SortInit sort_init)
{
if (mem_limit <= 0) {
mem_limit = 1 << 20;
}
int ret = OB_SUCCESS;
sort_->set_id(0);
input_table_->set_id(1);
sort_->set_column_count(input_table_->get_column_count());
sort_->set_mem_limit(mem_limit);
int64_t tenant_id = OB_SYS_TENANT_ID;
ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
if (tenant_config.is_valid()) {
tenant_config->_sort_area_size = mem_limit;
} else {
ret = OB_ERR_UNEXPECTED;
SQL_ENG_LOG(WARN, "unexpected status: config is invalid", K(tenant_id));
}
cons_run_filename(*filename_);
input_table_->set_row_count(row_count);
input_table_->set_phy_plan(physical_plan_);
sort_->set_phy_plan(physical_plan_);
row_count_ = row_count;
if (OB_FAIL(sort_->set_child(0, *input_table_))) {
} else if (OB_FAIL(sort_init(*sort_))) {
}
return ret;
}
static void set_local_merge_sort()
{
sort_->set_local_merge_sort(true);
}
static int init(int64_t row_count, int64_t mem_limit, int64_t sort_col1, ObCollationType cs_type1, int64_t sort_col2,
ObCollationType cs_type2)
{
return init(row_count, mem_limit, [&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, false, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true, ObMaxType, default_asc_direction()))) {
}
return ret;
});
}
static void reset()
{
sort_->reset();
input_table_->reset();
row_count_ = -1;
}
static void reuse()
{
sort_->reuse();
input_table_->reuse();
row_count_ = -1;
}
private:
ObSortPlan();
static void cons_run_filename(ObString& filename)
{
char* filename_buf = (char*)"ob_sort_test.run";
filename.assign_ptr(filename_buf, (int32_t)strlen(filename_buf));
return;
}
public:
private:
static ObPhysicalPlan* physical_plan_;
static ObFakeTable* input_table_;
static ObSort* sort_;
static int64_t row_count_;
static ObString* filename_;
};
ObSortTest::ObSortTest() : blocksstable::TestDataFilePrepare("TestDiskIR", 2 << 20, 2000)
{}
ObSortTest::~ObSortTest()
{}
int ObSortTest::init_tenant_mgr()
{
int ret = OB_SUCCESS;
ObTenantManager& tm = ObTenantManager::get_instance();
ObAddr self;
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
oceanbase::obrpc::ObCommonRpcProxy rs_rpc_proxy;
oceanbase::share::ObRsMgr rs_mgr;
int64_t tenant_id = 1;
self.set_ip_addr("127.0.0.1", 8086);
ret = ObTenantConfigMgr::get_instance().add_tenant_config(tenant_id);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(tenant_id);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.set_tenant_mem_limit(tenant_id, 2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SYS_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
const int64_t ulmt = 128LL << 30;
const int64_t llmt = 128LL << 30;
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
EXPECT_EQ(OB_SUCCESS, ret);
oceanbase::lib::set_memory_limit(128LL << 32);
return ret;
}
#define BEGIN_THREAD_CODE_V2(num) \
{ \
std::vector<std::thread*> _threads; \
for (int _i = 0; _i < (num); _i++) _threads.push_back(new std::thread([&]
#define END_THREAD_CODE_V2() )); \
for (auto t : _threads) \
t->join(); \
}
template <typename SortInit>
void ObSortTest::sort_test(
int64_t row_count, int64_t mem_limit, SortInit sort_init, int64_t verify_row_cnt, bool local_merge_sort)
{
ASSERT_EQ(OB_SUCCESS, ObSortPlan::init(row_count, mem_limit, sort_init));
if (local_merge_sort) {
ObSortPlan::set_local_merge_sort();
}
BEGIN_THREAD_CODE_V2(2)
{
ObExecContext exec_ctx;
ASSERT_EQ(OB_SUCCESS, exec_ctx.init_phy_op(2));
ASSERT_EQ(OB_SUCCESS, exec_ctx.create_physical_plan_ctx());
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
my_session.init_tenant("sys", 1);
exec_ctx.set_my_session(&my_session);
THIS_WORKER.set_timeout_ts(ObTimeUtility::current_time() + 600000000);
// do sort.
ObSort& sort = ObSortPlan::get_instance();
ASSERT_EQ(OB_SUCCESS, sort.open(exec_ctx));
auto& sort_columns = sort.get_sort_columns();
ObSEArray<ObOpSchemaObj, 8> op_schema_objs;
cons_op_schema_objs(sort_columns, op_schema_objs);
sort.get_op_schema_objs_for_update().assign(op_schema_objs);
ObObj pre[sort_columns.count()];
char varchar_buf[1024];
const ObNewRow* row = NULL;
int64_t cnt = verify_row_cnt > 0 ? verify_row_cnt : row_count;
for (int64_t i = 0; i < cnt; ++i) {
ASSERT_EQ(OB_SUCCESS, sort.get_next_row(exec_ctx, row)) << i;
// check order
for (int64_t j = 0; i > 0 && j < sort_columns.count(); j++) {
auto& col = sort_columns.at(j);
int cmp = pre[j].compare(row->cells_[col.index_], col.cs_type_);
if (cmp != 0) {
ASSERT_TRUE(col.is_ascending() ? cmp < 0 : cmp > 0);
break;
}
}
// save previous row
int64_t pos = 0;
for (int64_t j = 0; j < sort_columns.count(); j++) {
pre[j] = row->cells_[sort_columns.at(j).index_];
auto& c = pre[j];
if (c.is_string_type()) {
auto len = std::min((uint64_t)c.val_len_, sizeof(varchar_buf) - pos);
MEMCPY(varchar_buf + pos, c.v_.string_, len);
c.v_.string_ = varchar_buf + pos;
pos += len;
}
}
} // end for
ASSERT_EQ(OB_ITER_END, sort.get_next_row(exec_ctx, row));
ASSERT_EQ(OB_SUCCESS, sort.close(exec_ctx));
int64_t sort_row_count = 0;
ASSERT_EQ(OB_SUCCESS, sort.get_sort_row_count(exec_ctx, sort_row_count));
ASSERT_EQ(row_count, sort_row_count);
ob_print_mod_memory_usage();
ObTenantManager::get_instance().print_tenant_usage();
}
END_THREAD_CODE_V2();
ObSortPlan::reset();
}
void ObSortTest::sort_test(int64_t row_count, int64_t mem_limit, int64_t sort_col1, ObCollationType cs_type1,
int64_t sort_col2, ObCollationType cs_type2)
{
sort_test(row_count, mem_limit, [&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, false, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true, ObMaxType, default_asc_direction()))) {
}
return ret;
});
}
void ObSortTest::local_merge_sort_test(int64_t row_count, int64_t mem_limit, int64_t sort_col1,
ObCollationType cs_type1, int64_t sort_col2, ObCollationType cs_type2)
{
sort_test(
row_count,
mem_limit,
[&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col1, cs_type1, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(sort_col2, cs_type2, true, ObMaxType, default_asc_direction()))) {
}
return ret;
},
-1,
true);
}
void ObSortTest::serialize_test()
{
ObSort& sort_1 = ObSortPlan::get_instance();
ObArenaAllocator alloc;
ObSort sort_2(alloc);
const int64_t MAX_SERIALIZE_BUF_LEN = 1024;
char buf[MAX_SERIALIZE_BUF_LEN] = {'\0'};
ASSERT_EQ(OB_SUCCESS, ObSortPlan::init(1024, 1024, 0, CS_TYPE_INVALID, 1, CS_TYPE_INVALID));
int64_t pos = 0;
ASSERT_EQ(OB_SUCCESS, sort_1.serialize(buf, MAX_SERIALIZE_BUF_LEN, pos));
ASSERT_EQ(pos, sort_1.get_serialize_size());
int64_t data_len = pos;
sort_2.set_phy_plan(const_cast<ObPhysicalPlan*>(sort_1.get_phy_plan()));
pos = 0;
ASSERT_EQ(OB_SUCCESS, sort_2.deserialize(buf, data_len, pos));
ASSERT_EQ(pos, data_len);
const char* str_1 = to_cstring(sort_1);
const char* str_2 = to_cstring(sort_2);
ASSERT_EQ(0, strcmp(str_1, str_2)) << "sort_1: " << to_cstring(sort_1) << std::endl
<< "sort_2: " << to_cstring(sort_2);
ObSortPlan::reuse();
}
void ObSortTest::sort_exception_test(int expect_ret)
{
int ret = OB_SUCCESS;
ObExecContext exec_ctx;
const ObNewRow* row = NULL;
ObSQLSessionInfo my_session;
my_session.test_init(0, 0, 0, NULL);
my_session.init_tenant("sys", 1);
exec_ctx.set_my_session(&my_session);
ASSERT_EQ(OB_SUCCESS, exec_ctx.init_phy_op(2));
ASSERT_EQ(OB_SUCCESS, exec_ctx.create_physical_plan_ctx());
ObSort& sort = ObSortPlan::get_instance();
ObSortPlan::reset();
int64_t row_count = 16 * 1024;
if (OB_FAIL(ObSortPlan::init(row_count, 0, 0, CS_TYPE_UTF8MB4_BIN, 1, CS_TYPE_UTF8MB4_BIN))) {
} else if (OB_FAIL(sort.open(exec_ctx))) {
} else {
ObSEArray<ObOpSchemaObj, 8> op_schema_objs;
cons_op_schema_objs(sort.get_sort_columns(), op_schema_objs);
sort.get_op_schema_objs_for_update().assign(op_schema_objs);
while (OB_SUCC(ret)) {
ret = sort.get_next_row(exec_ctx, row);
}
if (OB_ITER_END == ret) {
int64_t sort_row_count = 0;
if (OB_FAIL(sort.close(exec_ctx))) {
} else if (OB_FAIL(sort.get_sort_row_count(exec_ctx, sort_row_count))) {
} else {
ASSERT_EQ(row_count, sort_row_count);
}
}
}
sort.close(exec_ctx);
ObSortPlan::reuse();
if (OB_FAIL(ret)) {
ASSERT_EQ(expect_ret, ret);
}
}
TEST_F(ObSortTest, varchar_int_item_in_mem_test)
{
int64_t sort_col1 = 0;
int64_t sort_col2 = 1;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(16 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(64 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
cs_type2 = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(16 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(64 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
}
TEST_F(ObSortTest, varchar_int_item_merge_sort_test)
{
int64_t sort_col1 = 0;
int64_t sort_col2 = 1;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(128 * 1024, 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256 * 1024, 4 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
// recursive merge needed.
sort_test(256 * 1024, 512 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
cs_type2 = CS_TYPE_UTF8MB4_GENERAL_CI;
sort_test(128 * 1024, 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256 * 1024, 4 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
}
TEST_F(ObSortTest, int_int_item_in_mem_test)
{
int64_t sort_col1 = 1;
int64_t sort_col2 = 2;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
sort_test(16 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(64 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
}
TEST_F(ObSortTest, int_int_item_merge_sort_test)
{
int64_t sort_col1 = 1;
int64_t sort_col2 = 2;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
// sort_test(64 * 1024, 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
sort_test(256 * 1024, 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
}
TEST_F(ObSortTest, test_conf)
{
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(128 * 1024 * 1024);
int64_t sort_mem = 0;
sort_mem = TEST_SORT_DUMP_GET_HASH_AREA_SIZE();
ASSERT_EQ((128 * 1024 * 1024), sort_mem);
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(1 * 1024 * 1024 * 1024);
sort_mem = TEST_SORT_DUMP_GET_HASH_AREA_SIZE();
ASSERT_EQ((1024 * 1024 * 1024), sort_mem);
TEST_SORT_DUMP_SET_HASH_AREA_SIZE(128 * 1024 * 1024);
}
TEST_F(ObSortTest, prefix_sort_test1)
{
sort_test(1024 * 1024, 1024 * 1024, [](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_BINARY, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
}
TEST_F(ObSortTest, prefix_sort_test3)
{
sort_test(256 * 1024, 1024 * 1024, [](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL5_ROW_ID_DIV_3, CS_TYPE_BINARY, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
}
TEST_F(ObSortTest, prefix_merge_sort_test)
{
sort_test(256 * 1024, 1024 * 1024, [](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(3))) {
} else if (OB_FAIL(sort.add_sort_column(ObFakeTable::COL11_ROW_ID_MULTIPLY_3_DIV_COUNT,
CS_TYPE_BINARY,
true,
ObMaxType,
default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_prefix_pos(1);
}
return ret;
});
}
class MockExpr : public ObSqlExpression {
public:
MockExpr(ObIAllocator& alloc, int64_t cnt) : ObSqlExpression(alloc)
{
set_item_count(1);
start_gen_infix_exr();
ObPostExprItem item;
item.set_int(cnt);
item.set_item_type(T_INT);
add_expr_item(item);
}
int calc(common::ObExprCtx&, const common::ObNewRow&, common::ObObj& result) const
{
result.set_int(get_expr_items().at(0).get_obj().get_int());
return OB_SUCCESS;
}
};
TEST_F(ObSortTest, topn_sort_test)
{
ObArenaAllocator alloc;
MockExpr expr(alloc, 4);
sort_test(
20,
10 << 20,
[&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_topn_expr(&expr);
}
return ret;
},
4);
}
TEST_F(ObSortTest, topn_disk_sort_test)
{
ObArenaAllocator alloc;
MockExpr expr(alloc, 4);
sort_test(
20,
0,
[&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_topn_expr(&expr);
}
return ret;
},
4);
}
TEST_F(ObSortTest, local_merge_sort_test)
{
int64_t sort_col1 = 1;
int64_t sort_col2 = 2;
ObCollationType cs_type1 = CS_TYPE_UTF8MB4_BIN;
ObCollationType cs_type2 = CS_TYPE_UTF8MB4_BIN;
local_merge_sort_test(0, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
local_merge_sort_test(16, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
local_merge_sort_test(256, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
local_merge_sort_test(16, 0, sort_col1, cs_type1, sort_col2, cs_type2);
local_merge_sort_test(256, 0, sort_col1, cs_type1, sort_col2, cs_type2);
local_merge_sort_test(64 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
local_merge_sort_test(256 * 1024, 256 * 1024 * 1024, sort_col1, cs_type1, sort_col2, cs_type2);
}
TEST_F(ObSortTest, local_merge_sort_disk_test)
{
ObArenaAllocator alloc;
MockExpr expr(alloc, 4);
sort_test(
256 * 1024,
1 << 20,
[&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_topn_expr(&expr);
}
return ret;
},
4,
true);
}
TEST_F(ObSortTest, local_merge_sort_topn_test)
{
ObArenaAllocator alloc;
MockExpr expr(alloc, 4);
sort_test(
1024,
10 << 20,
[&](ObSort& sort) {
int ret = OB_SUCCESS;
if (OB_FAIL(sort.init_sort_columns(2))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL0_RAND_STR, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else if (OB_FAIL(sort.add_sort_column(
ObFakeTable::COL1_ROW_ID, CS_TYPE_UTF8MB4_BIN, true, ObMaxType, default_asc_direction()))) {
} else {
sort.set_topn_expr(&expr);
}
return ret;
},
4,
true);
}
TEST_F(ObSortTest, ser)
{
serialize_test();
}
#define SORT_EXCEPTION_TEST(file, func, key, err, expect_ret) \
do { \
TP_SET_ERROR("engine/sort/" file, func, key, err); \
sort_exception_test(expect_ret); \
TP_SET_ERROR("engine/sort/" file, func, key, NULL); \
ASSERT_FALSE(HasFatalFailure()); \
} while (0)
TEST_F(ObSortTest, sort_exception)
{
THIS_WORKER.set_timeout_ts(ObTimeUtility::current_time() + 600000000);
SORT_EXCEPTION_TEST("ob_sort.cpp", "add_sort_column", "t1", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "open", "t1", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "open", "t3", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "open", "t5", 1, OB_ERR_UNEXPECTED);
SORT_EXCEPTION_TEST("ob_sort.cpp", "open", "t7", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "open", "t9", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "close", "t1", 1, OB_ERR_UNEXPECTED);
SORT_EXCEPTION_TEST("ob_sort.cpp", "close", "t3", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "do_sort", "t1", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "do_sort", "t3", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "do_sort", "t5", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "do_sort", "t7", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "do_sort", "t9", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "do_sort", "t11", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "do_sort", "t13", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "inner_get_next_row", "t1", 1, OB_ERR_UNEXPECTED);
SORT_EXCEPTION_TEST("ob_sort.cpp", "inner_get_next_row", "t3", 1, OB_TIMEOUT);
// see comments for tracepoint t5 in inner_get_next_row() of ob_sort.cpp.
// SORT_EXCEPTION_TEST("ob_sort.cpp", "inner_get_next_row", "t5", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "inner_get_next_row", "t7", 1, OB_ERR_UNEXPECTED);
SORT_EXCEPTION_TEST("ob_sort.cpp", "inner_get_next_row", "t9", OB_ERROR, OB_ERROR);
SORT_EXCEPTION_TEST("ob_sort.cpp", "get_sort_row_count", "t1", 1, OB_ERR_UNEXPECTED);
}
ObPhysicalPlan* ObSortPlan::physical_plan_ = nullptr;
ObSort* ObSortPlan::sort_ = nullptr;
ObFakeTable* ObSortPlan::input_table_ = nullptr;
int64_t ObSortPlan::row_count_ = -1;
ObString* ObSortPlan::filename_ = nullptr;
int main(int argc, char** argv)
{
ObClockGenerator::init();
system("rm -f test_sort.log*");
OB_LOGGER.set_file_name("test_sort.log", true, true);
OB_LOGGER.set_log_level("INFO");
oceanbase::observer::ObSignalHandle signal_handle;
oceanbase::observer::ObSignalHandle::change_signal_mask();
signal_handle.start();
void* buf = nullptr;
ObArenaAllocator allocator;
buf = allocator.alloc(sizeof(ObPhysicalPlan));
ObSortPlan::physical_plan_ = new (buf) ObPhysicalPlan();
buf = allocator.alloc(sizeof(ObSort));
ObSortPlan::sort_ = new (buf) ObSort(ObSortPlan::physical_plan_->get_allocator());
buf = allocator.alloc(sizeof(ObFakeTable));
ObSortPlan::input_table_ = new (buf) ObFakeTable();
ObSortPlan::row_count_ = -1;
buf = allocator.alloc(sizeof(ObString));
ObSortPlan::filename_ = new (buf) ObString();
::testing::InitGoogleTest(&argc, argv);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
return RUN_ALL_TESTS();
}

1195
unittest/sql/engine/sort/ob_specific_columns_sort_test.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

285
unittest/sql/engine/sort/test_sort_impl.cpp Normal file
View File

@ -0,0 +1,285 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "sql/engine/sort/ob_sort.h"
#include "sql/session/ob_sql_session_info.h"
#include "sql/engine/ob_physical_plan.h"
#include "lib/utility/ob_test_util.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/container/ob_se_array.h"
#include <gtest/gtest.h>
#include "ob_fake_table.h"
#include "sql/engine/ob_exec_context.h"
#include "share/ob_worker.h"
#include "observer/ob_signal_handle.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "storage/blocksstable/ob_tmp_file.h"
#include "sql/engine/join/join_data_generator.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include <thread>
#include <vector>
#include <gtest/gtest.h>
using namespace oceanbase;
using namespace oceanbase::sql;
using namespace oceanbase::common;
using namespace oceanbase::omt;
class TestSortImpl : public blocksstable::TestDataFilePrepare {
public:
TestSortImpl() : blocksstable::TestDataFilePrepare("TestDiskIR", 2 << 20, 1000), data_(alloc_)
{}
void set_sort_area_size(int64_t size)
{
int ret = OB_SUCCESS;
int64_t tenant_id = OB_SYS_TENANT_ID;
ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
if (tenant_config.is_valid()) {
tenant_config->_sort_area_size = size;
} else {
ret = OB_ERR_UNEXPECTED;
SQL_ENG_LOG(WARN, "unexpected status: config is invalid", K(tenant_id));
}
// ASSERT_EQ(OB_SUCCESS, ret);
}
virtual void SetUp() override
{
GCONF.enable_sql_operator_dump.set_value("True");
ASSERT_EQ(OB_SUCCESS, init_tenant_mgr());
blocksstable::TestDataFilePrepare::SetUp();
ASSERT_EQ(OB_SUCCESS, blocksstable::ObTmpFileManager::get_instance().init());
ASSERT_EQ(OB_SUCCESS, blocksstable::ObTmpFileManager::get_instance().start());
for (int64_t i = 0; i < ARRAYSIZEOF(cols_); i++) {
ObSortColumn& sc = cols_[i];
sc.cs_type_ = CS_TYPE_UTF8MB4_BIN;
sc.set_is_ascending(true);
}
cols_[0].index_ = JoinDataGenerator::ROW_ID_CELL;
cols_[1].index_ = JoinDataGenerator::IDX_CELL;
cols_[2].index_ = JoinDataGenerator::RAND_CELL;
data_.gen_varchar_cell_ = true;
data_.string_size_ = 500;
set_sort_area_size(1 * 1024 * 1024);
}
virtual void TearDown() override
{
blocksstable::ObTmpFileManager::get_instance().destroy();
blocksstable::TestDataFilePrepare::TearDown();
destroy_tenant_mgr();
}
int init_tenant_mgr()
{
int ret = OB_SUCCESS;
ObTenantManager& tm = ObTenantManager::get_instance();
ObAddr self;
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
oceanbase::obrpc::ObCommonRpcProxy rs_rpc_proxy;
oceanbase::share::ObRsMgr rs_mgr;
self.set_ip_addr("127.0.0.1", 8086);
ret = ObTenantConfigMgr::get_instance().add_tenant_config(tenant_id_);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(tenant_id_);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.set_tenant_mem_limit(tenant_id_, 2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SYS_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
EXPECT_EQ(OB_SUCCESS, ret);
const int64_t ulmt = 128LL << 30;
const int64_t llmt = 128LL << 30;
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
EXPECT_EQ(OB_SUCCESS, ret);
oceanbase::lib::set_memory_limit(128LL << 32);
return ret;
}
void destroy_tenant_mgr()
{
ObTenantManager::get_instance().destroy();
}
template <typename T>
void verify_sort(T& sort, int64_t cols, bool unique)
{
ObExecContext ctx;
const ObNewRow* r = NULL;
int64_t rows = data_.row_cnt_;
for (int64_t i = 0; i < rows; i++) {
ASSERT_EQ(OB_SUCCESS, data_.get_next_row(ctx, r));
ASSERT_EQ(OB_SUCCESS, sort.add_row(*r));
if (i == rows / 2 || i == (rows - rows / 16)) {
ASSERT_EQ(OB_SUCCESS, sort.sort());
for (int j = 0; j < std::min(1000L, i - 1); j++) {
int ret = sort.get_next_row(r);
if (!unique || 0 == j) {
ASSERT_EQ(OB_SUCCESS, ret);
} else {
ASSERT_TRUE(OB_SUCCESS == ret || OB_ITER_END == ret);
}
}
}
}
ASSERT_EQ(OB_SUCCESS, sort.sort());
int ret = 0;
ObNewRow prev;
char copy_buf[2048];
for (int loop = 0; loop < 2; loop += 1) {
if (loop == 1) {
ASSERT_EQ(0, sort.rewind());
}
for (int64_t i = 0; i < rows; i++) {
ret = sort.get_next_row(r);
if (!unique) {
ASSERT_EQ(0, ret);
} else {
ASSERT_TRUE(0 == ret || (OB_ITER_END == ret && (0 == rows || i > 0)));
}
if (0 == ret) {
if (i < 10) {
LOG_INFO("get row", K(*r));
}
if (i > 0) {
int cmp = 0;
for (int64_t i = 0; cmp == 0 && i < cols; i++) {
auto& col = cols_[i];
cmp = prev.get_cell(col.index_).compare(r->get_cell(col.index_), col.cs_type_);
if (cmp != 0) {
if (!col.is_ascending()) {
cmp *= -1;
}
}
}
ASSERT_LE(cmp, 0);
if (unique) {
ASSERT_LT(cmp, 0);
}
}
int64_t pos = 0;
ASSERT_EQ(0, prev.deep_copy(*r, copy_buf, sizeof(copy_buf), pos));
}
}
if (!unique) {
ASSERT_EQ(OB_ITER_END, sort.get_next_row(r));
}
}
}
ObArenaAllocator alloc_;
JoinDataGenerator data_;
ObSortColumn cols_[3];
uint64_t tenant_id_ = 1;
};
TEST_F(TestSortImpl, sort_impl)
{
ObSortImpl sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 20; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.idx_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObArrayHelper<ObSortColumn> helper;
helper.init(2, cols_, 2);
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, false, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.test_init();
sort.reuse();
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
}
TEST_F(TestSortImpl, local_order)
{
ObSortImpl sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 20; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObArrayHelper<ObSortColumn> helper;
helper.init(2, cols_, 2);
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, true, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 10000; };
data_.test_init();
sort.reset();
ASSERT_EQ(0, sort.init(tenant_id_, helper, NULL, true, true));
verify_sort(sort, 2, false);
ASSERT_FALSE(HasFatalFailure());
}
TEST_F(TestSortImpl, unique)
{
ObUniqueSort sort;
data_.row_cnt_ = 100;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 3; };
data_.row_id_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.idx_val_ = [](const int64_t v) { return murmurhash64A(&v, sizeof(v), 0); };
data_.test_init();
ObArrayHelper<ObSortColumn> helper;
helper.init(1, cols_, 1);
ASSERT_EQ(0, sort.init(tenant_id_, helper, true));
verify_sort(sort, 1, true);
ASSERT_FALSE(HasFatalFailure());
// test external disk merge sort
data_.row_cnt_ = (100L << 20) / data_.string_size_;
data_.idx_cnt_func_ = [](const int64_t, const int64_t) { return 50000; };
data_.test_init();
cols_[0].index_ = JoinDataGenerator::IDX_CELL;
sort.reset();
ASSERT_EQ(0, sort.init(tenant_id_, helper, true));
verify_sort(sort, 1, true);
ASSERT_FALSE(HasFatalFailure());
}
int main(int argc, char** argv)
{
ObClockGenerator::init();
system("rm -f test_sort_impl.log*");
OB_LOGGER.set_file_name("test_sort_impl.log", true, true);
OB_LOGGER.set_log_level("INFO");
oceanbase::observer::ObSignalHandle signal_handle;
oceanbase::observer::ObSignalHandle::change_signal_mask();
signal_handle.start();
::testing::InitGoogleTest(&argc, argv);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
return RUN_ALL_TESTS();
}