patch 4.0
This commit is contained in:
@ -1,4 +1,4 @@
|
||||
#ob_unittest(test_ra_row_store)
|
||||
ob_unittest(test_ra_row_store_projector)
|
||||
ob_unittest(test_chunk_row_store)
|
||||
ob_unittest(test_chunk_datum_store)
|
||||
sql_unittest(test_ra_row_store_projector)
|
||||
sql_unittest(test_chunk_row_store)
|
||||
sql_unittest(test_chunk_datum_store)
|
||||
|
||||
@ -24,21 +24,27 @@
|
||||
#include "sql/ob_sql_init.h"
|
||||
#include "share/datum/ob_datum.h"
|
||||
#include "sql/engine/expr/ob_expr.h"
|
||||
#include "share/ob_simple_mem_limit_getter.h"
|
||||
|
||||
namespace oceanbase {
|
||||
namespace sql {
|
||||
namespace oceanbase
|
||||
{
|
||||
namespace sql
|
||||
{
|
||||
using namespace common;
|
||||
static ObSimpleMemLimitGetter getter;
|
||||
|
||||
class TestEnv : public ::testing::Environment {
|
||||
class TestEnv : public ::testing::Environment
|
||||
{
|
||||
public:
|
||||
virtual void SetUp() override
|
||||
{
|
||||
GCONF.enable_sql_operator_dump.set_value("True");
|
||||
int ret = OB_SUCCESS;
|
||||
lib::ObMallocAllocator* malloc_allocator = lib::ObMallocAllocator::get_instance();
|
||||
// ret = malloc_allocator->create_tenant_ctx_allocator(OB_SYS_TENANT_ID);
|
||||
// ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = malloc_allocator->create_tenant_ctx_allocator(OB_SYS_TENANT_ID, common::ObCtxIds::WORK_AREA);
|
||||
lib::ObMallocAllocator *malloc_allocator = lib::ObMallocAllocator::get_instance();
|
||||
//ret = malloc_allocator->create_tenant_ctx_allocator(OB_SYS_TENANT_ID);
|
||||
//ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = malloc_allocator->create_tenant_ctx_allocator(
|
||||
OB_SYS_TENANT_ID, common::ObCtxIds::WORK_AREA);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
int s = (int)time(NULL);
|
||||
LOG_INFO("initial setup random seed", K(s));
|
||||
@ -46,52 +52,109 @@ public:
|
||||
}
|
||||
|
||||
virtual void TearDown() override
|
||||
{}
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
#define CALL(func, ...) \
|
||||
func(__VA_ARGS__); \
|
||||
ASSERT_FALSE(HasFatalFailure());
|
||||
#define CALL(func, ...) func(__VA_ARGS__); ASSERT_FALSE(HasFatalFailure());
|
||||
|
||||
class TestChunkDatumStore : public blocksstable::TestDataFilePrepare {
|
||||
struct MyAllocator : public DefaultPageAllocator
|
||||
{
|
||||
void *alloc(const int64_t sz, const ObMemAttr &attr) override
|
||||
{
|
||||
int64_t size = sz + 8;
|
||||
uint64_t *mem = (uint64_t *)DefaultPageAllocator::alloc(size, attr);
|
||||
if (NULL != mem) {
|
||||
*mem = sz;
|
||||
total_ += sz;
|
||||
return mem + 1;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void free(void *p)
|
||||
{
|
||||
if (NULL != p) {
|
||||
uint64_t *mem = ((uint64_t *)p - 1);
|
||||
total_ -= *mem;
|
||||
memset(p, 0xAA, *mem);
|
||||
DefaultPageAllocator::free(mem);
|
||||
}
|
||||
}
|
||||
|
||||
int64_t total_ = 0;
|
||||
};
|
||||
|
||||
class TestChunkDatumStore : public blocksstable::TestDataFilePrepare
|
||||
{
|
||||
public:
|
||||
TestChunkDatumStore()
|
||||
: blocksstable::TestDataFilePrepare("TestDisk_chunk_datum_store", 2 << 20, 5000),
|
||||
eval_ctx_(exec_ctx_, eval_res_, eval_tmp_)
|
||||
{}
|
||||
TestChunkDatumStore() : blocksstable::TestDataFilePrepare(&getter,
|
||||
"TestDisk_chunk_datum_store", 2<<20, 5000),
|
||||
plan_ctx_(alloc_),
|
||||
exec_ctx_(alloc_),
|
||||
eval_ctx_(exec_ctx_)
|
||||
{
|
||||
}
|
||||
|
||||
struct BatchGuard
|
||||
{
|
||||
BatchGuard(TestChunkDatumStore &t) : t_(t)
|
||||
{
|
||||
FOREACH_CNT(e, t_.cells_) {
|
||||
(*e)->batch_result_ = true;
|
||||
}
|
||||
FOREACH_CNT(e, t_.ver_cells_) {
|
||||
(*e)->batch_result_ = true;
|
||||
}
|
||||
}
|
||||
~BatchGuard()
|
||||
{
|
||||
FOREACH_CNT(e, t_.cells_) {
|
||||
(*e)->batch_result_ = false;
|
||||
}
|
||||
FOREACH_CNT(e, t_.ver_cells_) {
|
||||
(*e)->batch_result_ = false;
|
||||
}
|
||||
}
|
||||
TestChunkDatumStore &t_;
|
||||
};
|
||||
|
||||
void init_exprs()
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
int64_t pos = 0;
|
||||
eval_ctx_.frames_ = static_cast<char**>(alloc_.alloc(sizeof(void*) * 2));
|
||||
ASSERT_EQ(true, nullptr != eval_ctx_.frames_);
|
||||
eval_ctx_.frames_[0] = (char*)alloc_.alloc(4096);
|
||||
int64_t frame_size = (sizeof(ObDatum) + 16) * COLS * batch_size_ * 2;
|
||||
eval_ctx_.frames_[0] = (char *)alloc_.alloc(frame_size);
|
||||
memset(eval_ctx_.frames_[0], 0, frame_size);
|
||||
ASSERT_EQ(true, nullptr != eval_ctx_.frames_[0]);
|
||||
const int64_t datum_eval_info_size = sizeof(ObDatum) + sizeof(ObEvalInfo);
|
||||
for (int64_t i = 0; i < COLS; ++i) {
|
||||
ObExpr* expr = static_cast<ObExpr*>(alloc_.alloc(sizeof(ObExpr)));
|
||||
ObExpr *expr = new (alloc_.alloc(sizeof(ObExpr))) ObExpr();
|
||||
ASSERT_EQ(OB_SUCCESS, cells_.push_back(expr));
|
||||
expr->frame_idx_ = 0;
|
||||
expr->datum_off_ = pos;
|
||||
expr->eval_info_off_ = pos + sizeof(ObDatum);
|
||||
pos += datum_eval_info_size;
|
||||
ObDatum* expr_datum = &expr->locate_expr_datum(eval_ctx_);
|
||||
new (expr_datum) ObDatum;
|
||||
expr_datum->ptr_ = eval_ctx_.frames_[0] + pos;
|
||||
pos += sizeof(int64_t);
|
||||
pos += sizeof(ObDatum) * batch_size_;
|
||||
expr->eval_info_off_ = pos;
|
||||
pos += sizeof(ObEvalInfo);
|
||||
ObDatum *datums = expr->locate_batch_datums(eval_ctx_);
|
||||
for (int64_t j = 0; j < batch_size_; j++) {
|
||||
datums[j].ptr_ = eval_ctx_.frames_[0] + pos;
|
||||
pos += 8;
|
||||
}
|
||||
}
|
||||
for (int64_t i = 0; i < COLS; ++i) {
|
||||
ObExpr* expr = static_cast<ObExpr*>(alloc_.alloc(sizeof(ObExpr)));
|
||||
ObExpr *expr = new (alloc_.alloc(sizeof(ObExpr))) ObExpr();
|
||||
ASSERT_EQ(OB_SUCCESS, ver_cells_.push_back(expr));
|
||||
expr->frame_idx_ = 0;
|
||||
expr->datum_off_ = pos;
|
||||
expr->eval_info_off_ = pos + sizeof(ObDatum);
|
||||
pos += datum_eval_info_size;
|
||||
ObDatum* expr_datum = &expr->locate_expr_datum(eval_ctx_);
|
||||
new (expr_datum) ObDatum;
|
||||
expr_datum->ptr_ = eval_ctx_.frames_[0] + pos;
|
||||
pos += sizeof(int64_t);
|
||||
pos += sizeof(ObDatum) * batch_size_;
|
||||
expr->eval_info_off_ = pos;
|
||||
pos += sizeof(ObEvalInfo);
|
||||
ObDatum *datums = expr->locate_batch_datums(eval_ctx_);
|
||||
for (int64_t j = 0; j < batch_size_; j++) {
|
||||
datums[j].ptr_ = eval_ctx_.frames_[0] + pos;
|
||||
pos += 8;
|
||||
}
|
||||
}
|
||||
}
|
||||
virtual void SetUp() override
|
||||
@ -101,16 +164,31 @@ public:
|
||||
blocksstable::TestDataFilePrepare::SetUp();
|
||||
ret = blocksstable::ObTmpFileManager::get_instance().init();
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
static ObTenantBase tenant_ctx(tenant_id_);
|
||||
ObTenantEnv::set_tenant(&tenant_ctx);
|
||||
ObTenantIOManager *io_service = nullptr;
|
||||
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
|
||||
|
||||
cell_cnt_ = COLS;
|
||||
init_exprs();
|
||||
|
||||
// mem limit 1M
|
||||
plan_.set_batch_size(batch_size_);
|
||||
plan_ctx_.set_phy_plan(&plan_);
|
||||
eval_ctx_.set_max_batch_size(batch_size_);
|
||||
exec_ctx_.set_physical_plan_ctx(&plan_ctx_);
|
||||
|
||||
skip_ = (ObBitVector *)alloc_.alloc(ObBitVector::memory_size(batch_size_));
|
||||
|
||||
//mem limit 1M
|
||||
rs_.set_allocator(rs_alloc_);
|
||||
ret = rs_.init(1L << 20, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ASSERT_EQ(OB_SUCCESS, rs_.alloc_dir_id());
|
||||
|
||||
memset(str_buf_, 'z', BUF_SIZE);
|
||||
memset(str_buf_, 'a', BUF_SIZE);
|
||||
for (int64_t i = 0; i < BUF_SIZE; i++) {
|
||||
str_buf_[i] += i % 26;
|
||||
}
|
||||
LOG_INFO("setup finished");
|
||||
}
|
||||
|
||||
@ -124,39 +202,45 @@ public:
|
||||
|
||||
blocksstable::ObTmpFileManager::get_instance().destroy();
|
||||
blocksstable::TestDataFilePrepare::TearDown();
|
||||
ObTenantManager::get_instance().destroy();
|
||||
LOG_WARN("TearDown finished", K_(rs));
|
||||
}
|
||||
|
||||
void gen_row(int64_t row_id)
|
||||
void gen_row(int64_t row_id, int64_t idx = 0)
|
||||
{
|
||||
ObDatum* expr_datum_0 = &cells_.at(0)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_0 = &cells_.at(0)->locate_batch_datums(eval_ctx_)[idx];
|
||||
expr_datum_0->set_int(row_id);
|
||||
cells_.at(0)->get_eval_info(eval_ctx_).evaluated_ = true;
|
||||
cells_.at(0)->get_eval_info(eval_ctx_).projected_ = true;
|
||||
int64_t max_size = 512;
|
||||
if (enable_big_row_ && row_id > 0 && random() % 100000 < 5) {
|
||||
max_size = 1 << 20;
|
||||
}
|
||||
ObDatum* expr_datum_1 = &cells_.at(1)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_1 = &cells_.at(1)->locate_batch_datums(eval_ctx_)[idx];
|
||||
expr_datum_1->set_null();
|
||||
cells_.at(1)->get_eval_info(eval_ctx_).evaluated_ = true;
|
||||
cells_.at(1)->get_eval_info(eval_ctx_).projected_ = true;
|
||||
|
||||
int64_t size = 10 + random() % max_size;
|
||||
ObDatum* expr_datum_2 = &cells_.at(2)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_2 = &cells_.at(2)->locate_batch_datums(eval_ctx_)[idx];
|
||||
expr_datum_2->set_string(str_buf_, (int)size);
|
||||
cells_.at(2)->get_eval_info(eval_ctx_).evaluated_ = true;
|
||||
cells_.at(2)->get_eval_info(eval_ctx_).projected_ = true;
|
||||
}
|
||||
|
||||
// varify next row
|
||||
// template <typename T>
|
||||
//varify next row
|
||||
//template <typename T>
|
||||
void verify_row(ObChunkDatumStore::Iterator& it, int64_t n, bool verify_all = false)
|
||||
{
|
||||
int ret = it.get_next_row(ver_cells_, eval_ctx_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
CALL(verify_row_data, n, verify_all);
|
||||
}
|
||||
|
||||
ObDatum* expr_datum_0 = &ver_cells_.at(0)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum* expr_datum_1 = &ver_cells_.at(1)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum* expr_datum_2 = &ver_cells_.at(2)->locate_expr_datum(eval_ctx_);
|
||||
void verify_row_data(int64_t n, bool verify_all = false, int64_t idx = 0)
|
||||
{
|
||||
ObDatum *expr_datum_0 = &ver_cells_.at(0)->locate_batch_datums(eval_ctx_)[idx];
|
||||
ObDatum *expr_datum_1 = &ver_cells_.at(1)->locate_batch_datums(eval_ctx_)[idx];
|
||||
ObDatum *expr_datum_2 = &ver_cells_.at(2)->locate_batch_datums(eval_ctx_)[idx];
|
||||
int64_t v = expr_datum_0->get_int();
|
||||
if (verify_all) {
|
||||
expr_datum_1->is_null();
|
||||
@ -181,8 +265,8 @@ public:
|
||||
return verify_row(it_, id, verify_all);
|
||||
}
|
||||
|
||||
void verify_n_rows(ObChunkDatumStore& rs, ObChunkDatumStore::Iterator& it, int64_t n, bool verify_all = false,
|
||||
int64_t chunk_size = 0, int64_t start = 0)
|
||||
void verify_n_rows(ObChunkDatumStore &rs, ObChunkDatumStore::Iterator& it,
|
||||
int64_t n, bool verify_all = false, int64_t chunk_size = 0, int64_t start = 0)
|
||||
{
|
||||
if (!it.is_valid()) {
|
||||
ASSERT_EQ(rs.begin(it), 0);
|
||||
@ -198,7 +282,7 @@ public:
|
||||
return verify_n_rows(rs_, it_, n, verify_all);
|
||||
}
|
||||
|
||||
void append_rows(ObChunkDatumStore& rs, int64_t cnt)
|
||||
void append_rows(ObChunkDatumStore &rs, int64_t cnt)
|
||||
{
|
||||
int64_t ret = OB_SUCCESS;
|
||||
int64_t base = rs.get_row_cnt();
|
||||
@ -215,6 +299,45 @@ public:
|
||||
return append_rows(rs_, cnt);
|
||||
}
|
||||
|
||||
void batch_append_rows(int64_t cnt)
|
||||
{
|
||||
int64_t ret = 0;
|
||||
int64_t base = rs_.get_row_cnt();
|
||||
skip_->reset(batch_size_);
|
||||
for (int64_t i = 0; i < cnt;) {
|
||||
int64_t bcnt = std::min(cnt - i, batch_size_);
|
||||
for (int64_t j = 0; j < bcnt; j++, i++) {
|
||||
gen_row(base + i, j);
|
||||
}
|
||||
int64_t stored_row_cnt = 0;
|
||||
ret = rs_.add_batch(cells_, eval_ctx_, *skip_, bcnt, stored_row_cnt);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ASSERT_EQ(stored_row_cnt, bcnt);
|
||||
}
|
||||
ASSERT_EQ(base + cnt, rs_.get_row_cnt());
|
||||
}
|
||||
|
||||
void batch_verify_all(int64_t chunk_size)
|
||||
{
|
||||
it_.reset();
|
||||
ASSERT_EQ(OB_SUCCESS, rs_.begin(it_, chunk_size));
|
||||
int64_t read_cnt = 0;
|
||||
int ret = OB_SUCCESS;
|
||||
while (OB_SUCC(ret)) {
|
||||
int64_t cnt = 0;
|
||||
ret = it_.get_next_batch(ver_cells_, eval_ctx_, batch_size_, cnt);
|
||||
if (OB_SUCC(ret)) {
|
||||
ASSERT_GT(cnt, 0);
|
||||
for (int64_t i = 0; i < cnt; i++) {
|
||||
CALL(verify_row_data, read_cnt + i, true, i);
|
||||
}
|
||||
read_cnt += cnt;
|
||||
}
|
||||
}
|
||||
ASSERT_EQ(OB_ITER_END, ret);
|
||||
ASSERT_EQ(read_cnt, rs_.get_row_cnt());
|
||||
}
|
||||
|
||||
void test_time(int64_t block_size, int64_t rows)
|
||||
{
|
||||
ObArenaAllocator alloc(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20);
|
||||
@ -232,18 +355,15 @@ public:
|
||||
ret = rs.add_row(cells_, &eval_ctx_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
}
|
||||
LOG_INFO("rs write time:",
|
||||
K(block_size),
|
||||
K(rows),
|
||||
K(rs.get_block_cnt()),
|
||||
K(rs.get_block_list_cnt()),
|
||||
LOG_INFO("rs write time:", K(block_size), K(rows),
|
||||
K(rs.get_block_cnt()), K(rs.get_block_list_cnt()),
|
||||
K(ObTimeUtil::current_time() - begin));
|
||||
begin = ObTimeUtil::current_time();
|
||||
ObChunkDatumStore::Iterator it;
|
||||
ASSERT_EQ(rs.begin(it), 0);
|
||||
i = 0;
|
||||
while (OB_SUCC(it.get_next_row(ver_cells_, eval_ctx_))) {
|
||||
ObDatum* expr_datum_0 = &ver_cells_.at(0)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_0 = &ver_cells_.at(0)->locate_expr_datum(eval_ctx_);
|
||||
v = *expr_datum_0->int_;
|
||||
ASSERT_EQ(i, v);
|
||||
i++;
|
||||
@ -252,23 +372,28 @@ public:
|
||||
}
|
||||
|
||||
void with_or_without_chunk(bool is_with);
|
||||
|
||||
protected:
|
||||
const static int64_t COLS = 3;
|
||||
bool enable_big_row_ = false;
|
||||
int64_t cell_cnt_;
|
||||
ObSEArray<ObExpr*, COLS> cells_;
|
||||
ObSEArray<ObExpr*, COLS> ver_cells_;
|
||||
MyAllocator rs_alloc_;
|
||||
ObChunkDatumStore rs_;
|
||||
ObChunkDatumStore::Iterator it_;
|
||||
// 256 is average row length, contain 5 blocks per batch.
|
||||
int64_t batch_size_ = (64L << 10) * 5 / 256;
|
||||
ObBitVector *skip_;
|
||||
|
||||
int64_t tenant_id_ = OB_SYS_TENANT_ID;
|
||||
int64_t ctx_id_ = ObCtxIds::WORK_AREA;
|
||||
const char* label_ = ObModIds::OB_SQL_ROW_STORE;
|
||||
const char *label_ = ObModIds::OB_SQL_ROW_STORE;
|
||||
|
||||
const static int64_t BUF_SIZE = 2 << 20;
|
||||
char str_buf_[BUF_SIZE];
|
||||
ObArenaAllocator alloc_;
|
||||
ObPhysicalPlan plan_;
|
||||
ObPhysicalPlanCtx plan_ctx_;
|
||||
ObExecContext exec_ctx_;
|
||||
ObArenaAllocator eval_res_;
|
||||
ObArenaAllocator eval_tmp_;
|
||||
@ -278,7 +403,6 @@ protected:
|
||||
int TestChunkDatumStore::init_tenant_mgr()
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
ObTenantManager& tm = ObTenantManager::get_instance();
|
||||
ObAddr self;
|
||||
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
|
||||
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
|
||||
@ -286,36 +410,32 @@ int TestChunkDatumStore::init_tenant_mgr()
|
||||
oceanbase::share::ObRsMgr rs_mgr;
|
||||
int64_t tenant_id = OB_SYS_TENANT_ID;
|
||||
self.set_ip_addr("127.0.0.1", 8086);
|
||||
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
ret = tm.add_tenant(tenant_id);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
ret = tm.set_tenant_mem_limit(tenant_id, 2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
ret = tm.add_tenant(OB_SYS_TENANT_ID);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
|
||||
ret = getter.add_tenant(tenant_id,
|
||||
2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
const int64_t ulmt = 128LL << 30;
|
||||
const int64_t llmt = 128LL << 30;
|
||||
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
|
||||
ret = getter.add_tenant(OB_SERVER_TENANT_ID,
|
||||
ulmt,
|
||||
llmt);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
oceanbase::lib::set_memory_limit(128LL << 32);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
// Test start
|
||||
TEST_F(TestChunkDatumStore, basic)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
LOG_WARN("starting basic test: append 3000 rows");
|
||||
CALL(append_rows, 3000); // approximate 1MB, no need to dump
|
||||
CALL(append_rows, 3000); // approximate 1MB, no need to dump
|
||||
CALL(verify_n_rows, rs_.get_row_cnt(), true);
|
||||
LOG_WARN("basic test: varified rows", K(rs_.get_row_cnt()));
|
||||
it_.reset();
|
||||
|
||||
LOG_WARN("starting basic test: append 10000 rows");
|
||||
CALL(append_rows, 10000); // need to dump
|
||||
CALL(append_rows, 10000); //need to dump
|
||||
rs_.finish_add_row();
|
||||
LOG_WARN("mem", K(rs_.get_mem_hold()), K(rs_.get_mem_used()));
|
||||
ASSERT_EQ(13000, rs_.get_row_cnt());
|
||||
@ -350,6 +470,103 @@ TEST_F(TestChunkDatumStore, basic)
|
||||
rs_.reset();
|
||||
}
|
||||
|
||||
TEST_F(TestChunkDatumStore, has_next_bug)
|
||||
{
|
||||
rs_.reset();
|
||||
int ret = rs_.init(1024, tenant_id_, ctx_id_, label_);
|
||||
rs_.set_allocator(rs_alloc_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
CALL(append_rows, 10000);
|
||||
ASSERT_EQ(OB_SUCCESS, rs_.finish_add_row(true));
|
||||
int64_t mem_before_iterate = rs_alloc_.total_;
|
||||
ObChunkDatumStore::Iterator it;
|
||||
ASSERT_EQ(OB_SUCCESS, rs_.begin(it));
|
||||
for (int64_t i = 0; i < 10000; i++) {
|
||||
ASSERT_TRUE(it.has_next());
|
||||
CALL(verify_row, it, i, true);
|
||||
}
|
||||
ASSERT_FALSE(it.has_next());
|
||||
ASSERT_EQ(OB_ITER_END, it.get_next_row(ver_cells_, eval_ctx_));
|
||||
|
||||
int64_t mem_used = rs_alloc_.total_ - mem_before_iterate;
|
||||
ASSERT_LT(mem_used, 500L * 1024L)
|
||||
<< " mem_before_iterate: " << mem_before_iterate
|
||||
<< " cur_mem: " << rs_alloc_.total_;
|
||||
}
|
||||
|
||||
TEST_F(TestChunkDatumStore, iteration_age)
|
||||
{
|
||||
rs_.reset();
|
||||
int ret = rs_.init(1024, tenant_id_, ctx_id_, label_);
|
||||
rs_.set_allocator(rs_alloc_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
CALL(append_rows, 20000);
|
||||
ASSERT_EQ(OB_SUCCESS, rs_.finish_add_row(true));
|
||||
int64_t mem_before_iterate = rs_alloc_.total_;
|
||||
ObChunkDatumStore::Iterator it;
|
||||
ObChunkDatumStore::IterationAge age;
|
||||
ASSERT_EQ(OB_SUCCESS, rs_.begin(it));
|
||||
// hold the iterated blocks
|
||||
it.set_iteration_age(&age);
|
||||
age.inc();
|
||||
for (int64_t i = 0; i < 10000; i++) {
|
||||
ASSERT_TRUE(it.has_next());
|
||||
CALL(verify_row, it, i, true);
|
||||
}
|
||||
int64_t mem_used = rs_alloc_.total_ - mem_before_iterate;
|
||||
ASSERT_GT(mem_used, 1024L * 1024L)
|
||||
<< " mem_before_iterate: " << mem_before_iterate
|
||||
<< " cur_mem: " << rs_alloc_.total_;
|
||||
|
||||
mem_before_iterate = rs_alloc_.total_;
|
||||
// release the iterated blocks.
|
||||
for (int64_t i = 10000; i < 20000; i++) {
|
||||
age.inc();
|
||||
ASSERT_TRUE(it.has_next());
|
||||
CALL(verify_row, it, i, true);
|
||||
}
|
||||
|
||||
mem_used = rs_alloc_.total_ - mem_before_iterate;
|
||||
ASSERT_LT(mem_used, 500L * 1024L)
|
||||
<< " mem_before_iterate: " << mem_before_iterate
|
||||
<< " cur_mem: " << rs_alloc_.total_;
|
||||
}
|
||||
|
||||
TEST_F(TestChunkDatumStore, batch_basic)
|
||||
{
|
||||
BatchGuard g(*this);
|
||||
CALL(batch_append_rows, 3000); // approximate 1MB, no need to dump
|
||||
CALL(verify_n_rows, rs_.get_row_cnt(), true);
|
||||
it_.reset();
|
||||
CALL(batch_verify_all, 0);
|
||||
it_.reset();
|
||||
rs_.reset();
|
||||
ASSERT_EQ(0, rs_alloc_.total_);
|
||||
|
||||
CALL(batch_append_rows, 30000); // need dump
|
||||
rs_.finish_add_row();
|
||||
ASSERT_EQ(30000, rs_.get_row_cnt());
|
||||
ASSERT_GT(rs_.get_file_size(), 1000000);
|
||||
CALL(batch_verify_all, 0);
|
||||
CALL(batch_verify_all, 512L << 10);
|
||||
it_.reset();
|
||||
rs_.reset();
|
||||
ASSERT_EQ(0, rs_alloc_.total_);
|
||||
|
||||
enable_big_row_ = true;
|
||||
int ret = rs_.init(1L << 20, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ASSERT_EQ(OB_SUCCESS, rs_.alloc_dir_id());
|
||||
CALL(batch_append_rows, 20000);
|
||||
ASSERT_EQ(OB_SUCCESS, rs_.finish_add_row());
|
||||
ASSERT_EQ(20000, rs_.get_row_cnt());
|
||||
CALL(batch_verify_all, 0);
|
||||
|
||||
it_.reset();
|
||||
rs_.reset();
|
||||
ASSERT_EQ(0, rs_alloc_.total_);
|
||||
}
|
||||
|
||||
TEST_F(TestChunkDatumStore, multi_iter)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
@ -362,7 +579,7 @@ TEST_F(TestChunkDatumStore, multi_iter)
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
LOG_WARN("starting basic test: append 3000 rows");
|
||||
CALL(append_rows, rs, total); // approximate 1MB, no need to dump
|
||||
CALL(append_rows, rs, total); // approximate 1MB, no need to dump
|
||||
rs.finish_add_row();
|
||||
LOG_WARN("Multi_iter", K_(rs.mem_hold), K_(rs.mem_used));
|
||||
|
||||
@ -400,7 +617,7 @@ TEST_F(TestChunkDatumStore, basic2)
|
||||
int ret = OB_SUCCESS;
|
||||
ObChunkDatumStore rs;
|
||||
ObChunkDatumStore::Iterator it;
|
||||
// mem limit 5M
|
||||
//mem limit 5M
|
||||
ret = rs.init(5L << 20, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
@ -422,7 +639,7 @@ TEST_F(TestChunkDatumStore, chunk_iterator)
|
||||
ObChunkDatumStore rs;
|
||||
ObChunkDatumStore::ChunkIterator chunk_it;
|
||||
ObChunkDatumStore::RowIterator it;
|
||||
// mem limit 5M
|
||||
//mem limit 5M
|
||||
ret = rs.init(5L << 20, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
@ -434,7 +651,7 @@ TEST_F(TestChunkDatumStore, chunk_iterator)
|
||||
ASSERT_EQ(rs.get_row_cnt_in_memory() + rs.get_row_cnt_on_disk(), 100000);
|
||||
rs.finish_add_row();
|
||||
ASSERT_EQ(OB_SUCCESS, rs.begin(chunk_it, ObChunkDatumStore::BLOCK_SIZE * 4));
|
||||
const ObChunkDatumStore::StoredRow* v_sr = NULL;
|
||||
const ObChunkDatumStore::StoredRow *v_sr = NULL;
|
||||
int64_t v;
|
||||
int64_t extend_v;
|
||||
int64_t row_cnt = 0;
|
||||
@ -443,7 +660,7 @@ TEST_F(TestChunkDatumStore, chunk_iterator)
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
row_cnt += chunk_it.get_cur_chunk_row_cnt();
|
||||
LOG_WARN("got chunk of rows:", K(row_cnt));
|
||||
while (OB_SUCC(ret)) {
|
||||
while(OB_SUCC(ret)) {
|
||||
ret = it.get_next_row(v_sr);
|
||||
if (ret == OB_ITER_END) {
|
||||
ret = chunk_it.load_next_chunk(it);
|
||||
@ -458,9 +675,9 @@ TEST_F(TestChunkDatumStore, chunk_iterator)
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = it.convert_to_row(v_sr, ver_cells_, eval_ctx_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ObDatum* expr_datum_0 = &ver_cells_.at(0)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum* expr_datum_1 = &ver_cells_.at(1)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum* expr_datum_2 = &ver_cells_.at(2)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_0 = &ver_cells_.at(0)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_1 = &ver_cells_.at(1)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_2 = &ver_cells_.at(2)->locate_expr_datum(eval_ctx_);
|
||||
v = expr_datum_0->get_int();
|
||||
expr_datum_1->is_null();
|
||||
if (0 != strncmp(str_buf_, expr_datum_2->ptr_, expr_datum_2->len_)) {
|
||||
@ -482,7 +699,7 @@ TEST_F(TestChunkDatumStore, test_copy_row)
|
||||
int64_t rows = 1000;
|
||||
ObChunkDatumStore rs;
|
||||
ObChunkDatumStore::Iterator it;
|
||||
const ObChunkDatumStore::StoredRow* sr;
|
||||
const ObChunkDatumStore::StoredRow *sr;
|
||||
LOG_WARN("starting mem_perf test: append rows", K(rows));
|
||||
int64_t begin = ObTimeUtil::current_time();
|
||||
ret = rs.init(0, tenant_id_, ctx_id_, label_);
|
||||
@ -535,11 +752,13 @@ TEST_F(TestChunkDatumStore, disk)
|
||||
write_time += common::ObTimeUtil::current_time() - begin;
|
||||
}
|
||||
|
||||
LOG_INFO("mem and disk", K(rows), K(rs.get_mem_hold()), K(rs.get_mem_used()), K(rs.get_file_size()));
|
||||
LOG_INFO("mem and disk", K(rows), K(rs.get_mem_hold()),
|
||||
K(rs.get_mem_used()), K(rs.get_file_size()));
|
||||
LOG_INFO("disk write:", K(rows), K(write_time));
|
||||
|
||||
ASSERT_EQ(OB_SUCCESS, rs.finish_add_row());
|
||||
LOG_WARN("mem and disk after finish", K(rows), K(rs.get_mem_hold()), K(rs.get_mem_used()), K(rs.get_file_size()));
|
||||
LOG_WARN("mem and disk after finish", K(rows), K(rs.get_mem_hold()),
|
||||
K(rs.get_mem_used()), K(rs.get_file_size()));
|
||||
|
||||
it.reset();
|
||||
begin = ObTimeUtil::current_time();
|
||||
@ -568,18 +787,14 @@ TEST_F(TestChunkDatumStore, disk_with_chunk)
|
||||
write_time += common::ObTimeUtil::current_time() - begin;
|
||||
}
|
||||
|
||||
LOG_INFO("mem and disk", K(rows), K(rs.get_mem_hold()), K(rs.get_mem_used()), K(rs.get_file_size()));
|
||||
LOG_INFO("mem and disk", K(rows), K(rs.get_mem_hold()),
|
||||
K(rs.get_mem_used()), K(rs.get_file_size()));
|
||||
LOG_INFO("disk write:", K(rows), K(write_time));
|
||||
|
||||
ASSERT_EQ(OB_SUCCESS, rs.finish_add_row());
|
||||
LOG_INFO("mem and disk after finish",
|
||||
K(rows),
|
||||
K(rs.get_mem_hold()),
|
||||
K(rs.get_mem_used()),
|
||||
K(rs.get_file_size()),
|
||||
K(rs.max_blk_size_),
|
||||
K(rs.min_blk_size_),
|
||||
K(rs.n_block_in_file_));
|
||||
LOG_INFO("mem and disk after finish", K(rows), K(rs.get_mem_hold()),
|
||||
K(rs.get_mem_used()), K(rs.get_file_size()), K(rs.max_blk_size_),
|
||||
K(rs.min_blk_size_), K(rs.n_block_in_file_));
|
||||
|
||||
it.reset();
|
||||
begin = ObTimeUtil::current_time();
|
||||
@ -628,9 +843,9 @@ TEST_F(TestChunkDatumStore, disk_with_chunk)
|
||||
TEST_F(TestChunkDatumStore, test_add_block)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
// send
|
||||
//send
|
||||
ObChunkDatumStore rs;
|
||||
ObChunkDatumStore::Block* block;
|
||||
ObChunkDatumStore::Block *block;
|
||||
ObArenaAllocator alloc(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20);
|
||||
|
||||
gen_row(1);
|
||||
@ -641,7 +856,7 @@ TEST_F(TestChunkDatumStore, test_add_block)
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
int64_t min_size = rs.min_blk_size(row_size);
|
||||
void* mem = alloc.alloc(min_size);
|
||||
void *mem = alloc.alloc(min_size);
|
||||
|
||||
ret = rs.init_block_buffer(mem, min_size, block);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
@ -657,12 +872,12 @@ TEST_F(TestChunkDatumStore, test_add_block)
|
||||
LOG_INFO("Molly size", K(block->get_buffer()->data_size()), K(block->get_buffer()->capacity()));
|
||||
|
||||
ObArenaAllocator alloc2(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20);
|
||||
void* mem2 = alloc2.alloc(block->get_buffer()->data_size());
|
||||
void *mem2 = alloc2.alloc(block->get_buffer()->data_size());
|
||||
memcpy(mem2, mem, block->get_buffer()->data_size());
|
||||
|
||||
// recv
|
||||
//recv
|
||||
ObChunkDatumStore rs2;
|
||||
ObChunkDatumStore::Block* block2 = reinterpret_cast<ObChunkDatumStore::Block*>(mem2);
|
||||
ObChunkDatumStore::Block *block2 = reinterpret_cast<ObChunkDatumStore::Block *>(mem2);
|
||||
ret = rs2.init(0, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = rs2.add_block(block2, true);
|
||||
@ -673,7 +888,7 @@ TEST_F(TestChunkDatumStore, test_add_block)
|
||||
CALL(verify_n_rows, rs2, it2, 99, true);
|
||||
ASSERT_EQ(true, it2.has_next());
|
||||
CALL(verify_n_rows, rs2, it2, 100, true, 0, 99);
|
||||
ASSERT_EQ(false, it2.has_next());
|
||||
ASSERT_FALSE(it2.has_next());
|
||||
ASSERT_EQ(OB_ITER_END, it2.get_next_row(ver_cells_, eval_ctx_));
|
||||
rs2.remove_added_blocks();
|
||||
it2.reset();
|
||||
@ -710,9 +925,9 @@ TEST_F(TestChunkDatumStore, row_with_extend_size)
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = it.convert_to_row(v_sr, ver_cells_, eval_ctx_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ObDatum* expr_datum_0 = &ver_cells_.at(0)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum* expr_datum_1 = &ver_cells_.at(1)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum* expr_datum_2 = &ver_cells_.at(2)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_0 = &ver_cells_.at(0)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_1 = &ver_cells_.at(1)->locate_expr_datum(eval_ctx_);
|
||||
ObDatum *expr_datum_2 = &ver_cells_.at(2)->locate_expr_datum(eval_ctx_);
|
||||
v = expr_datum_0->get_int();
|
||||
expr_datum_1->is_null();
|
||||
if (0 != strncmp(str_buf_, expr_datum_2->ptr_, expr_datum_2->len_)) {
|
||||
@ -787,22 +1002,85 @@ TEST_F(TestChunkDatumStore, test_only_disk_data1)
|
||||
rs.reset();
|
||||
}
|
||||
|
||||
} // end namespace sql
|
||||
} // end namespace oceanbase
|
||||
TEST_F(TestChunkDatumStore, test_append_block)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
//send
|
||||
ObChunkDatumStore rs;
|
||||
ObChunkDatumStore::Block *block;
|
||||
ObArenaAllocator alloc(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20);
|
||||
|
||||
gen_row(1);
|
||||
int64_t row_size = 0;
|
||||
ASSERT_EQ(OB_SUCCESS, ObChunkDatumStore::Block::row_store_size(cells_, eval_ctx_, row_size));
|
||||
|
||||
ret = rs.init(0, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
int64_t min_size = rs.min_blk_size(row_size);
|
||||
void *mem = alloc.alloc(min_size);
|
||||
|
||||
ret = rs.init_block_buffer(mem, min_size, block);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = rs.add_block(block, false);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
CALL(append_rows, rs, 100);
|
||||
rs.finish_add_row();
|
||||
ret = block->unswizzling();
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
rs.remove_added_blocks();
|
||||
rs.reset();
|
||||
LOG_INFO("Molly size", K(block->get_buffer()->data_size()), K(block->get_buffer()->capacity()));
|
||||
|
||||
ObArenaAllocator alloc2(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20);
|
||||
void *mem2 = alloc2.alloc(block->get_buffer()->data_size());
|
||||
memcpy(mem2, mem, block->get_buffer()->data_size());
|
||||
|
||||
//recv
|
||||
ObChunkDatumStore rs2;
|
||||
rs2.alloc_dir_id();
|
||||
ObChunkDatumStore::Block *block2 = reinterpret_cast<ObChunkDatumStore::Block *>(mem2);
|
||||
ret = rs2.init(0, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
for (int64_t i = 0; OB_SUCC(ret) && i < 100; ++i) {
|
||||
ret = rs2.append_block(block->get_buffer()->data(), block->get_buffer()->data_size(), true);
|
||||
}
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = rs2.dump(false, true);
|
||||
rs2.finish_add_row();
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ObChunkDatumStore::Iterator it2;
|
||||
ASSERT_EQ(OB_SUCCESS, rs2.begin(it2));
|
||||
for (int64_t i = 0; OB_SUCC(ret) && i < 100; ++i) {
|
||||
CALL(verify_n_rows, rs2, it2, 99, true);
|
||||
ASSERT_EQ(true, it2.has_next());
|
||||
CALL(verify_n_rows, rs2, it2, 100, true, 0, 99);
|
||||
}
|
||||
ASSERT_FALSE(it2.has_next());
|
||||
ASSERT_EQ(OB_ITER_END, it2.get_next_row(ver_cells_, eval_ctx_));
|
||||
rs2.remove_added_blocks();
|
||||
it2.reset();
|
||||
rs2.reset();
|
||||
}
|
||||
|
||||
} // end namespace sql
|
||||
} // end namespace oceanbase
|
||||
|
||||
|
||||
void ignore_sig(int sig)
|
||||
{
|
||||
UNUSED(sig);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
signal(49, ignore_sig);
|
||||
oceanbase::sql::init_sql_factories();
|
||||
oceanbase::common::ObLogger::get_logger().set_file_name("test_chunk_datum_store.log", true);
|
||||
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
auto* env = new (oceanbase::sql::TestEnv);
|
||||
auto *env = new (oceanbase::sql::TestEnv);
|
||||
testing::AddGlobalTestEnvironment(env);
|
||||
int ret = RUN_ALL_TESTS();
|
||||
OB_LOGGER.disable();
|
||||
|
||||
@ -22,20 +22,26 @@
|
||||
#include "common/row/ob_row_store.h"
|
||||
#include "share/config/ob_server_config.h"
|
||||
#include "sql/ob_sql_init.h"
|
||||
#include "share/ob_simple_mem_limit_getter.h"
|
||||
|
||||
namespace oceanbase {
|
||||
namespace sql {
|
||||
|
||||
namespace oceanbase
|
||||
{
|
||||
namespace sql
|
||||
{
|
||||
using namespace common;
|
||||
static ObSimpleMemLimitGetter getter;
|
||||
|
||||
class TestEnv : public ::testing::Environment {
|
||||
class TestEnv : public ::testing::Environment
|
||||
{
|
||||
public:
|
||||
virtual void SetUp() override
|
||||
{
|
||||
GCONF.enable_sql_operator_dump.set_value("True");
|
||||
int ret = OB_SUCCESS;
|
||||
lib::ObMallocAllocator* malloc_allocator = lib::ObMallocAllocator::get_instance();
|
||||
// ret = malloc_allocator->create_tenant_ctx_allocator(OB_SYS_TENANT_ID);
|
||||
// ASSERT_EQ(OB_SUCCESS, ret);
|
||||
lib::ObMallocAllocator *malloc_allocator = lib::ObMallocAllocator::get_instance();
|
||||
//ret = malloc_allocator->create_tenant_ctx_allocator(OB_SYS_TENANT_ID);
|
||||
//ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = malloc_allocator->create_tenant_ctx_allocator(OB_SYS_TENANT_ID, common::ObCtxIds::WORK_AREA);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
int s = (int)time(NULL);
|
||||
@ -44,17 +50,19 @@ public:
|
||||
}
|
||||
|
||||
virtual void TearDown() override
|
||||
{}
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
#define CALL(func, ...) \
|
||||
func(__VA_ARGS__); \
|
||||
ASSERT_FALSE(HasFatalFailure());
|
||||
#define CALL(func, ...) func(__VA_ARGS__); ASSERT_FALSE(HasFatalFailure());
|
||||
|
||||
class TestChunkRowStore : public blocksstable::TestDataFilePrepare {
|
||||
class TestChunkRowStore : public blocksstable::TestDataFilePrepare
|
||||
{
|
||||
public:
|
||||
TestChunkRowStore() : blocksstable::TestDataFilePrepare("TestDisk_chunk_row_store", 2 << 20, 5000)
|
||||
{}
|
||||
TestChunkRowStore() : blocksstable::TestDataFilePrepare(&getter,
|
||||
"TestDisk_chunk_row_store", 2<<20, 5000)
|
||||
{
|
||||
}
|
||||
|
||||
virtual void SetUp() override
|
||||
{
|
||||
@ -63,8 +71,10 @@ public:
|
||||
blocksstable::TestDataFilePrepare::SetUp();
|
||||
ret = blocksstable::ObTmpFileManager::get_instance().init();
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = blocksstable::ObTmpFileManager::get_instance().start();
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
static ObTenantBase tenant_ctx(tenant_id_);
|
||||
ObTenantEnv::set_tenant(&tenant_ctx);
|
||||
ObTenantIOManager *io_service = nullptr;
|
||||
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
|
||||
|
||||
row_.count_ = COLS;
|
||||
row_.cells_ = cells_;
|
||||
@ -73,7 +83,7 @@ public:
|
||||
row_verify_.count_ = COLS;
|
||||
row_verify_.cells_ = cells_;
|
||||
|
||||
// mem limit 1M
|
||||
//mem limit 1M
|
||||
ret = rs_.init(1L << 20, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
@ -93,11 +103,10 @@ public:
|
||||
|
||||
blocksstable::ObTmpFileManager::get_instance().destroy();
|
||||
blocksstable::TestDataFilePrepare::TearDown();
|
||||
ObTenantManager::get_instance().destroy();
|
||||
LOG_WARN("TearDown finished", K_(rs));
|
||||
}
|
||||
|
||||
ObNewRow& gen_row(int64_t row_id)
|
||||
ObNewRow &gen_row(int64_t row_id)
|
||||
{
|
||||
cells_[0].set_int(row_id);
|
||||
int64_t max_size = 512;
|
||||
@ -109,8 +118,8 @@ public:
|
||||
return row_;
|
||||
}
|
||||
|
||||
// varify next row
|
||||
// template <typename T>
|
||||
//varify next row
|
||||
//template <typename T>
|
||||
void verify_row(ObChunkRowStore::Iterator& it, int64_t n, bool verify_all = false)
|
||||
{
|
||||
int ret = it.get_next_row(row_verify_);
|
||||
@ -142,8 +151,8 @@ public:
|
||||
return verify_row(it_, id, verify_all);
|
||||
}
|
||||
|
||||
void verify_n_rows(ObChunkRowStore& rs, ObChunkRowStore::Iterator& it, int64_t n, bool verify_all = false,
|
||||
int64_t chunk_size = 0, int64_t start = 0)
|
||||
void verify_n_rows(ObChunkRowStore &rs, ObChunkRowStore::Iterator& it,
|
||||
int64_t n, bool verify_all = false, int64_t chunk_size = 0, int64_t start = 0)
|
||||
{
|
||||
if (!it.is_valid()) {
|
||||
ASSERT_EQ(rs.begin(it), 0);
|
||||
@ -151,9 +160,9 @@ public:
|
||||
}
|
||||
for (int64_t i = start; i < n; i++) {
|
||||
CALL(verify_row, it, i, verify_all);
|
||||
// if (i % 1000 == 0) {
|
||||
// LOG_WARN("verified rows:", K(i));
|
||||
// }
|
||||
// if (i % 1000 == 0) {
|
||||
// LOG_WARN("verified rows:", K(i));
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
@ -162,15 +171,15 @@ public:
|
||||
return verify_n_rows(rs_, it_, n, verify_all);
|
||||
}
|
||||
|
||||
void append_rows(ObChunkRowStore& rs, int64_t cnt)
|
||||
void append_rows(ObChunkRowStore &rs, int64_t cnt)
|
||||
{
|
||||
int64_t ret = OB_SUCCESS;
|
||||
int64_t base = rs.get_row_cnt();
|
||||
for (int64_t i = 0; i < cnt; i++) {
|
||||
ObNewRow& row = gen_row(base + i);
|
||||
ObNewRow &row = gen_row(base + i);
|
||||
ret = rs.add_row(row);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
// if (i % 1000 == 0) {
|
||||
//if (i % 1000 == 0) {
|
||||
// LOG_WARN("appended rows:", K(i));
|
||||
//}
|
||||
}
|
||||
@ -196,18 +205,14 @@ public:
|
||||
rs.set_block_size(block_size);
|
||||
begin = ObTimeUtil::current_time();
|
||||
for (i = 0; i < rows; i++) {
|
||||
ObNewRow& row = gen_row(i);
|
||||
ObNewRow &row = gen_row(i);
|
||||
ret = rs.add_row(row);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
// if (i % 1000 == 0) {
|
||||
//if (i % 1000 == 0) {
|
||||
// LOG_WARN("appended rows:", K(i));
|
||||
//}
|
||||
}
|
||||
LOG_WARN("rs write time:",
|
||||
K(block_size),
|
||||
K(rows),
|
||||
K(rs.get_block_cnt()),
|
||||
K(rs.get_block_list_cnt()),
|
||||
LOG_WARN("rs write time:", K(block_size), K(rows), K(rs.get_block_cnt()), K(rs.get_block_list_cnt()),
|
||||
K(ObTimeUtil::current_time() - begin));
|
||||
begin = ObTimeUtil::current_time();
|
||||
ObChunkRowStore::Iterator it;
|
||||
@ -222,8 +227,8 @@ public:
|
||||
}
|
||||
|
||||
void with_or_without_chunk(bool is_with);
|
||||
|
||||
protected:
|
||||
|
||||
const static int64_t COLS = 3;
|
||||
bool enable_big_row_ = false;
|
||||
ObObj cells_[COLS];
|
||||
@ -235,7 +240,7 @@ protected:
|
||||
|
||||
int64_t tenant_id_ = OB_SYS_TENANT_ID;
|
||||
int64_t ctx_id_ = ObCtxIds::WORK_AREA;
|
||||
const char* label_ = ObModIds::OB_SQL_ROW_STORE;
|
||||
const char *label_ = ObModIds::OB_SQL_ROW_STORE;
|
||||
|
||||
const static int64_t BUF_SIZE = 2 << 20;
|
||||
char str_buf_[BUF_SIZE];
|
||||
@ -244,7 +249,6 @@ protected:
|
||||
int TestChunkRowStore::init_tenant_mgr()
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
ObTenantManager& tm = ObTenantManager::get_instance();
|
||||
ObAddr self;
|
||||
oceanbase::rpc::frame::ObReqTransport req_transport(NULL, NULL);
|
||||
oceanbase::obrpc::ObSrvRpcProxy rpc_proxy;
|
||||
@ -252,19 +256,14 @@ int TestChunkRowStore::init_tenant_mgr()
|
||||
oceanbase::share::ObRsMgr rs_mgr;
|
||||
int64_t tenant_id = OB_SYS_TENANT_ID;
|
||||
self.set_ip_addr("127.0.0.1", 8086);
|
||||
ret = tm.init(self, rpc_proxy, rs_rpc_proxy, rs_mgr, &req_transport, &ObServerConfig::get_instance());
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
ret = tm.add_tenant(tenant_id);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
ret = tm.set_tenant_mem_limit(tenant_id, 2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
ret = tm.add_tenant(OB_SYS_TENANT_ID);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
ret = tm.add_tenant(OB_SERVER_TENANT_ID);
|
||||
ret = getter.add_tenant(tenant_id,
|
||||
2L * 1024L * 1024L * 1024L, 4L * 1024L * 1024L * 1024L);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
const int64_t ulmt = 128LL << 30;
|
||||
const int64_t llmt = 128LL << 30;
|
||||
ret = tm.set_tenant_mem_limit(OB_SYS_TENANT_ID, ulmt, llmt);
|
||||
ret = getter.add_tenant(OB_SERVER_TENANT_ID,
|
||||
ulmt,
|
||||
llmt);
|
||||
EXPECT_EQ(OB_SUCCESS, ret);
|
||||
oceanbase::lib::set_memory_limit(128LL << 32);
|
||||
return ret;
|
||||
@ -275,13 +274,13 @@ TEST_F(TestChunkRowStore, basic)
|
||||
int ret = OB_SUCCESS;
|
||||
|
||||
LOG_WARN("starting basic test: append 3000 rows");
|
||||
CALL(append_rows, 3000); // approximate 1MB, no need to dump
|
||||
CALL(append_rows, 3000); // approximate 1MB, no need to dump
|
||||
CALL(verify_n_rows, rs_.get_row_cnt(), true);
|
||||
LOG_WARN("basic test: varified rows", K(rs_.get_row_cnt()));
|
||||
it_.reset();
|
||||
|
||||
LOG_WARN("starting basic test: append 10000 rows");
|
||||
CALL(append_rows, 10000); // need to dump
|
||||
CALL(append_rows, 10000); //need to dump
|
||||
rs_.finish_add_row();
|
||||
LOG_WARN("mem", K(rs_.get_mem_hold()), K(rs_.get_mem_used()));
|
||||
ASSERT_EQ(13000, rs_.get_row_cnt());
|
||||
@ -329,7 +328,7 @@ TEST_F(TestChunkRowStore, multi_iter)
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
LOG_WARN("starting basic test: append 3000 rows");
|
||||
CALL(append_rows, rs, total); // approximate 1MB, no need to dump
|
||||
CALL(append_rows, rs, total); // approximate 1MB, no need to dump
|
||||
rs.finish_add_row();
|
||||
LOG_WARN("Multi_iter", K_(rs.mem_hold), K_(rs.mem_used));
|
||||
|
||||
@ -362,13 +361,13 @@ TEST_F(TestChunkRowStore, multi_iter)
|
||||
rs.reset();
|
||||
}
|
||||
|
||||
|
||||
TEST_F(TestChunkRowStore, keep_projector0)
|
||||
{
|
||||
ObChunkRowStore rs(NULL);
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ASSERT_EQ(OB_SUCCESS,
|
||||
rs.init(
|
||||
100 << 20, OB_SERVER_TENANT_ID, ObCtxIds::DEFAULT_CTX_ID, common::ObModIds::OB_SQL_CHUNK_ROW_STORE, true));
|
||||
ASSERT_EQ(OB_SUCCESS, rs.init(100 << 20, OB_SERVER_TENANT_ID, ObCtxIds::DEFAULT_CTX_ID,
|
||||
common::ObModIds::OB_SQL_CHUNK_ROW_STORE, true));
|
||||
const int64_t OBJ_CNT = 3;
|
||||
ObObj objs[OBJ_CNT];
|
||||
ObNewRow r;
|
||||
@ -395,7 +394,7 @@ TEST_F(TestChunkRowStore, keep_projector0)
|
||||
ObChunkRowStore::Iterator it1;
|
||||
ASSERT_EQ(OB_SUCCESS, rs.begin(it1));
|
||||
|
||||
r.projector_size_ = 0;
|
||||
r.projector_size_ =0;
|
||||
r.projector_ = NULL;
|
||||
|
||||
ASSERT_EQ(OB_SUCCESS, it1.get_next_row(r));
|
||||
@ -406,7 +405,7 @@ TEST_F(TestChunkRowStore, keep_projector0)
|
||||
ASSERT_EQ(2, r.get_cell(1).get_int());
|
||||
ASSERT_EQ(2, r.cells_[1].get_int());
|
||||
|
||||
ObNewRow* rr;
|
||||
ObNewRow *rr;
|
||||
ASSERT_EQ(OB_SUCCESS, it1.get_next_row(rr));
|
||||
|
||||
ASSERT_TRUE(NULL == rr->projector_);
|
||||
@ -420,17 +419,14 @@ TEST_F(TestChunkRowStore, keep_projector0)
|
||||
rs.reset();
|
||||
}
|
||||
|
||||
|
||||
TEST_F(TestChunkRowStore, keep_projector2)
|
||||
{
|
||||
ObChunkRowStore rs(NULL);
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ASSERT_EQ(OB_SUCCESS,
|
||||
rs.init(100 << 20,
|
||||
OB_SERVER_TENANT_ID,
|
||||
ObCtxIds::DEFAULT_CTX_ID,
|
||||
common::ObModIds::OB_SQL_CHUNK_ROW_STORE,
|
||||
true,
|
||||
ObChunkRowStore::STORE_MODE::FULL));
|
||||
ASSERT_EQ(OB_SUCCESS, rs.init(100 << 20, OB_SERVER_TENANT_ID, ObCtxIds::DEFAULT_CTX_ID,
|
||||
common::ObModIds::OB_SQL_CHUNK_ROW_STORE, true,
|
||||
ObChunkRowStore::STORE_MODE::FULL));
|
||||
const int64_t OBJ_CNT = 3;
|
||||
ObObj objs[OBJ_CNT];
|
||||
ObNewRow r;
|
||||
@ -466,7 +462,7 @@ TEST_F(TestChunkRowStore, keep_projector2)
|
||||
ASSERT_EQ(1, r.cells_[1].get_int());
|
||||
ASSERT_EQ(2, r.get_cell(1).get_int());
|
||||
|
||||
ObNewRow* rr;
|
||||
ObNewRow *rr;
|
||||
ASSERT_EQ(OB_SUCCESS, it1.get_next_row(rr));
|
||||
|
||||
ASSERT_TRUE(NULL != rr->projector_);
|
||||
@ -485,22 +481,14 @@ TEST_F(TestChunkRowStore, keep_projector2_with_copy)
|
||||
{
|
||||
ObChunkRowStore rs(NULL);
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ASSERT_EQ(OB_SUCCESS,
|
||||
rs.init(100 << 20,
|
||||
OB_SERVER_TENANT_ID,
|
||||
ObCtxIds::DEFAULT_CTX_ID,
|
||||
common::ObModIds::OB_SQL_CHUNK_ROW_STORE,
|
||||
true,
|
||||
ObChunkRowStore::STORE_MODE::FULL));
|
||||
ASSERT_EQ(OB_SUCCESS, rs.init(100 << 20, OB_SERVER_TENANT_ID, ObCtxIds::DEFAULT_CTX_ID,
|
||||
common::ObModIds::OB_SQL_CHUNK_ROW_STORE, true,
|
||||
ObChunkRowStore::STORE_MODE::FULL));
|
||||
ObChunkRowStore rs2(NULL);
|
||||
ASSERT_EQ(OB_SUCCESS, rs2.alloc_dir_id());
|
||||
ASSERT_EQ(OB_SUCCESS,
|
||||
rs2.init(100 << 20,
|
||||
OB_SERVER_TENANT_ID,
|
||||
ObCtxIds::DEFAULT_CTX_ID,
|
||||
common::ObModIds::OB_SQL_CHUNK_ROW_STORE,
|
||||
true,
|
||||
ObChunkRowStore::STORE_MODE::FULL));
|
||||
ASSERT_EQ(OB_SUCCESS, rs2.init(100 << 20, OB_SERVER_TENANT_ID, ObCtxIds::DEFAULT_CTX_ID,
|
||||
common::ObModIds::OB_SQL_CHUNK_ROW_STORE, true,
|
||||
ObChunkRowStore::STORE_MODE::FULL));
|
||||
const int64_t OBJ_CNT = 3;
|
||||
ObObj objs[OBJ_CNT];
|
||||
ObNewRow r;
|
||||
@ -529,7 +517,7 @@ TEST_F(TestChunkRowStore, keep_projector2_with_copy)
|
||||
ASSERT_EQ(OB_SUCCESS, rs2.copy_row(sr, &rs));
|
||||
ObChunkRowStore::Iterator it2;
|
||||
ASSERT_EQ(OB_SUCCESS, rs2.begin(it2));
|
||||
ObNewRow* rr;
|
||||
ObNewRow *rr;
|
||||
ASSERT_EQ(OB_SUCCESS, it2.get_next_row(rr));
|
||||
// only fill cells, projector_ unchanged.
|
||||
ASSERT_EQ(0, rr->get_cell(0).get_int());
|
||||
@ -554,7 +542,7 @@ TEST_F(TestChunkRowStore, basic2)
|
||||
ObChunkRowStore rs;
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ObChunkRowStore::Iterator it;
|
||||
// mem limit 5M
|
||||
//mem limit 5M
|
||||
ret = rs.init(5L << 20, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
CALL(append_rows, rs, 100000);
|
||||
@ -569,6 +557,7 @@ TEST_F(TestChunkRowStore, basic2)
|
||||
rs.reset();
|
||||
}
|
||||
|
||||
|
||||
TEST_F(TestChunkRowStore, chunk_iterator)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
@ -576,7 +565,7 @@ TEST_F(TestChunkRowStore, chunk_iterator)
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ObChunkRowStore::ChunkIterator chunk_it;
|
||||
ObChunkRowStore::RowIterator it;
|
||||
// mem limit 5M
|
||||
//mem limit 5M
|
||||
ret = rs.init(5L << 20, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
CALL(append_rows, rs, 100000);
|
||||
@ -587,7 +576,7 @@ TEST_F(TestChunkRowStore, chunk_iterator)
|
||||
ASSERT_EQ(rs.get_row_cnt_in_memory() + rs.get_row_cnt_on_disk(), 100000);
|
||||
rs.finish_add_row();
|
||||
ASSERT_EQ(OB_SUCCESS, rs.begin(chunk_it, ObChunkRowStore::BLOCK_SIZE * 4));
|
||||
const ObChunkRowStore::StoredRow* v_sr = NULL;
|
||||
const ObChunkRowStore::StoredRow *v_sr = NULL;
|
||||
int64_t v;
|
||||
int64_t extend_v;
|
||||
int64_t row_cnt = 0;
|
||||
@ -596,7 +585,7 @@ TEST_F(TestChunkRowStore, chunk_iterator)
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
row_cnt += chunk_it.get_cur_chunk_row_cnt();
|
||||
LOG_WARN("got chunk of rows:", K(row_cnt));
|
||||
while (OB_SUCC(ret)) {
|
||||
while(OB_SUCC(ret)) {
|
||||
ret = it.get_next_row(v_sr);
|
||||
if (ret == OB_ITER_END) {
|
||||
ret = chunk_it.load_next_chunk(it);
|
||||
@ -634,7 +623,7 @@ TEST_F(TestChunkRowStore, test_copy_row)
|
||||
ObChunkRowStore rs;
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ObChunkRowStore::Iterator it;
|
||||
const ObChunkRowStore::StoredRow* sr;
|
||||
const ObChunkRowStore::StoredRow *sr;
|
||||
LOG_WARN("starting mem_perf test: append rows", K(rows));
|
||||
int64_t begin = ObTimeUtil::current_time();
|
||||
ret = rs.init(0, tenant_id_, ctx_id_, label_);
|
||||
@ -688,21 +677,21 @@ TEST_F(TestChunkRowStore, time_cmp)
|
||||
ra.init(0, tenant_id_, ctx_id_, label_);
|
||||
int64_t begin = ObTimeUtil::current_time();
|
||||
for (int64_t i = 0; i < rows; i++) {
|
||||
ObNewRow& row = gen_row(i);
|
||||
ObNewRow &row = gen_row(i);
|
||||
ret = ra.add_row(row);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
if (i % 1000 == 0) {
|
||||
LOG_WARN("appended rows:", K(i));
|
||||
}
|
||||
}
|
||||
LOG_WARN(
|
||||
"ra write time:", K(rows), K(ra.get_mem_hold()), K(ra.get_file_size()), K(ObTimeUtil::current_time() - begin));
|
||||
LOG_WARN("ra write time:", K(rows), K(ra.get_mem_hold()), K(ra.get_file_size()),
|
||||
K(ObTimeUtil::current_time() - begin));
|
||||
begin = ObTimeUtil::current_time();
|
||||
i = 0;
|
||||
while (OB_SUCC(ra.get_row(i, row_verify_))) {
|
||||
row_verify_.get_cell(0).get_int(v);
|
||||
ASSERT_EQ(i, v);
|
||||
i++;
|
||||
row_verify_.get_cell(0).get_int(v);
|
||||
ASSERT_EQ(i, v);
|
||||
i++;
|
||||
}
|
||||
LOG_WARN("old rc scan time:", K(rows), K(ObTimeUtil::current_time() - begin));
|
||||
ra.reset();
|
||||
@ -715,19 +704,15 @@ TEST_F(TestChunkRowStore, time_cmp)
|
||||
|
||||
begin = ObTimeUtil::current_time();
|
||||
for (int64_t i = 0; i < rows; i++) {
|
||||
ObNewRow& row = gen_row(i);
|
||||
ObNewRow &row = gen_row(i);
|
||||
ret = rs_old.add_row(row);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
if (i % 1000 == 0) {
|
||||
LOG_WARN("appended rows:", K(i));
|
||||
}
|
||||
}
|
||||
LOG_WARN("old rs write time:",
|
||||
K(rows),
|
||||
K(rs_old.get_block_count()),
|
||||
K(rs_old.get_block_size()),
|
||||
K(rs_old.get_used_mem_size()),
|
||||
K(ObTimeUtil::current_time() - begin));
|
||||
LOG_WARN("old rs write time:", K(rows), K(rs_old.get_block_count()), K(rs_old.get_block_size()),
|
||||
K(rs_old.get_used_mem_size()), K(ObTimeUtil::current_time() - begin));
|
||||
|
||||
begin = ObTimeUtil::current_time();
|
||||
ObRowStore::Iterator it_old = rs_old.begin();
|
||||
@ -839,6 +824,7 @@ TEST_F(TestChunkRowStore, disk)
|
||||
rs.reset();
|
||||
}
|
||||
|
||||
|
||||
TEST_F(TestChunkRowStore, disk_with_chunk)
|
||||
{
|
||||
int64_t begin = 0;
|
||||
@ -853,9 +839,9 @@ TEST_F(TestChunkRowStore, disk_with_chunk)
|
||||
ASSERT_EQ(OB_SUCCESS, rs.init(0, tenant_id_, ctx_id_, label_));
|
||||
rs.set_mem_limit(1L << 20);
|
||||
for (int64_t i = 0; i < round; i++) {
|
||||
// if (i == round / 2) {
|
||||
// enable_big_row_ = true;
|
||||
// }
|
||||
//if (i == round / 2) {
|
||||
//enable_big_row_ = true;
|
||||
// }
|
||||
begin = common::ObTimeUtil::current_time();
|
||||
CALL(append_rows, rs, cnt);
|
||||
write_time += common::ObTimeUtil::current_time() - begin;
|
||||
@ -865,14 +851,7 @@ TEST_F(TestChunkRowStore, disk_with_chunk)
|
||||
LOG_WARN("disk write:", K(rows), K(write_time));
|
||||
|
||||
ASSERT_EQ(OB_SUCCESS, rs.finish_add_row());
|
||||
LOG_WARN("mem and disk after finish",
|
||||
K(rows),
|
||||
K(rs.get_mem_hold()),
|
||||
K(rs.get_mem_used()),
|
||||
K(rs.get_file_size()),
|
||||
K(rs.max_blk_size_),
|
||||
K(rs.min_blk_size_),
|
||||
K(rs.n_block_in_file_));
|
||||
LOG_WARN("mem and disk after finish", K(rows), K(rs.get_mem_hold()), K(rs.get_mem_used()), K(rs.get_file_size()), K(rs.max_blk_size_), K(rs.min_blk_size_), K(rs.n_block_in_file_));
|
||||
|
||||
it.reset();
|
||||
begin = ObTimeUtil::current_time();
|
||||
@ -918,8 +897,9 @@ TEST_F(TestChunkRowStore, disk_with_chunk)
|
||||
rs.reset();
|
||||
}
|
||||
|
||||
// test chunk row store
|
||||
// case from oarowstore
|
||||
|
||||
// test chunk row store根据内存比例进行dump,去掉了这层逻辑,是否dump由上层驱动
|
||||
//case from oarowstore
|
||||
// TEST_F(TestChunkRowStore, start_dump_by_total_mem_used)
|
||||
// {
|
||||
// LOG_WARN("starting dump mem test: append rows", K(500000));
|
||||
@ -944,6 +924,7 @@ TEST_F(TestChunkRowStore, disk_with_chunk)
|
||||
// ret = rs.init(0, tenant_id_, ctx_id_, mod_id_);
|
||||
// ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
|
||||
// // case1: trigger dump by memory mod usage (60% of limit, 30MB)
|
||||
// // write 28MB, all in memory
|
||||
// CALL(append_rows, rs, (28L << 20) / avg_row_size);
|
||||
@ -1007,20 +988,20 @@ TEST_F(TestChunkRowStore, disk_with_chunk)
|
||||
TEST_F(TestChunkRowStore, test_add_block)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
// send
|
||||
//send
|
||||
ObChunkRowStore rs;
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ObChunkRowStore::Block* block;
|
||||
ObChunkRowStore::Block *block;
|
||||
ObArenaAllocator alloc(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20);
|
||||
|
||||
ObNewRow& row = gen_row(1);
|
||||
ObNewRow &row = gen_row(1);
|
||||
int64_t row_size = ObChunkRowStore::Block::row_store_size(row);
|
||||
|
||||
ret = rs.init(0, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
int64_t min_size = rs.min_blk_size(row_size);
|
||||
void* mem = alloc.alloc(min_size);
|
||||
void *mem = alloc.alloc(min_size);
|
||||
|
||||
ret = rs.init_block_buffer(mem, min_size, block);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
@ -1036,13 +1017,13 @@ TEST_F(TestChunkRowStore, test_add_block)
|
||||
LOG_WARN("Molly size", K(block->get_buffer()->data_size()), K(block->get_buffer()->capacity()));
|
||||
|
||||
ObArenaAllocator alloc2(ObModIds::OB_MODULE_PAGE_ALLOCATOR, 2 << 20);
|
||||
void* mem2 = alloc2.alloc(block->get_buffer()->data_size());
|
||||
void *mem2 = alloc2.alloc(block->get_buffer()->data_size());
|
||||
memcpy(mem2, mem, block->get_buffer()->data_size());
|
||||
|
||||
// recv
|
||||
//recv
|
||||
ObChunkRowStore rs2;
|
||||
ASSERT_EQ(OB_SUCCESS, rs2.alloc_dir_id());
|
||||
ObChunkRowStore::Block* block2 = reinterpret_cast<ObChunkRowStore::Block*>(mem2);
|
||||
ObChunkRowStore::Block *block2 = reinterpret_cast<ObChunkRowStore::Block *>(mem2);
|
||||
ret = rs2.init(0, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = rs2.add_block(block2, true);
|
||||
@ -1053,7 +1034,7 @@ TEST_F(TestChunkRowStore, test_add_block)
|
||||
CALL(verify_n_rows, rs2, it2, 99, true);
|
||||
ASSERT_EQ(true, it2.has_next());
|
||||
CALL(verify_n_rows, rs2, it2, 100, true, 0, 99);
|
||||
ASSERT_EQ(false, it2.has_next());
|
||||
ASSERT_FALSE(it2.has_next());
|
||||
ASSERT_EQ(OB_ITER_END, it2.get_next_row(row));
|
||||
rs2.remove_added_blocks();
|
||||
it2.reset();
|
||||
@ -1070,14 +1051,15 @@ TEST_F(TestChunkRowStore, row_extend_row)
|
||||
ObChunkRowStore rs;
|
||||
ASSERT_EQ(OB_SUCCESS, rs.alloc_dir_id());
|
||||
ObChunkRowStore::Iterator it;
|
||||
ASSERT_EQ(OB_SUCCESS, rs.init(0, tenant_id_, ctx_id_, label_, true, ObChunkRowStore::WITHOUT_PROJECTOR, 8));
|
||||
ASSERT_EQ(OB_SUCCESS,
|
||||
rs.init(0, tenant_id_, ctx_id_, label_, true, ObChunkRowStore::WITHOUT_PROJECTOR, 8));
|
||||
LOG_WARN("starting basic test: append 3000 rows");
|
||||
int64_t ret = OB_SUCCESS;
|
||||
int64_t base = rs.get_row_cnt();
|
||||
ObChunkRowStore::StoredRow* sr = NULL;
|
||||
const ObChunkRowStore::StoredRow* v_sr = NULL;
|
||||
for (int64_t i = 0; i < 100; i++) {
|
||||
ObNewRow& row = gen_row(i);
|
||||
ObNewRow &row = gen_row(i);
|
||||
ret = rs.add_row(row, &sr);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
*static_cast<int64_t*>(sr->get_extra_payload()) = i;
|
||||
@ -1103,6 +1085,7 @@ TEST_F(TestChunkRowStore, row_extend_row)
|
||||
LOG_WARN("verify failed", K(i), K(v), K(extend_v));
|
||||
}
|
||||
ASSERT_EQ(i, v);
|
||||
|
||||
}
|
||||
it.reset();
|
||||
rs.reset();
|
||||
@ -1134,6 +1117,7 @@ TEST_F(TestChunkRowStore, test_both_disk_and_memory)
|
||||
rs.reset();
|
||||
}
|
||||
|
||||
|
||||
TEST_F(TestChunkRowStore, test_only_disk_data)
|
||||
{
|
||||
int64_t round = 2;
|
||||
@ -1157,22 +1141,23 @@ TEST_F(TestChunkRowStore, test_only_disk_data)
|
||||
rs.reset();
|
||||
}
|
||||
|
||||
} // end namespace sql
|
||||
} // end namespace oceanbase
|
||||
} // end namespace sql
|
||||
} // end namespace oceanbase
|
||||
|
||||
|
||||
void ignore_sig(int sig)
|
||||
{
|
||||
UNUSED(sig);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
signal(49, ignore_sig);
|
||||
oceanbase::sql::init_sql_factories();
|
||||
oceanbase::common::ObLogger::get_logger().set_file_name("test_chunk_row_store.log", true);
|
||||
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
auto* env = new (oceanbase::sql::TestEnv);
|
||||
auto *env = new (oceanbase::sql::TestEnv);
|
||||
testing::AddGlobalTestEnvironment(env);
|
||||
int ret = RUN_ALL_TESTS();
|
||||
OB_LOGGER.disable();
|
||||
|
||||
@ -20,18 +20,21 @@
|
||||
#include "sql/engine/basic/ob_ra_row_store.h"
|
||||
#include "share/config/ob_server_config.h"
|
||||
|
||||
namespace oceanbase {
|
||||
namespace sql {
|
||||
namespace oceanbase
|
||||
{
|
||||
namespace sql
|
||||
{
|
||||
using namespace common;
|
||||
|
||||
class TestEnv : public ::testing::Environment {
|
||||
class TestEnv : public ::testing::Environment
|
||||
{
|
||||
public:
|
||||
virtual void SetUp() override
|
||||
{
|
||||
GCONF.enable_sql_operator_dump.set_value("True");
|
||||
lib::AChunkMgr::instance().set_max_chunk_cache_cnt(0);
|
||||
int ret = OB_SUCCESS;
|
||||
lib::ObMallocAllocator* malloc_allocator = lib::ObMallocAllocator::get_instance();
|
||||
lib::ObMallocAllocator *malloc_allocator = lib::ObMallocAllocator::get_instance();
|
||||
ret = malloc_allocator->create_tenant_ctx_allocator(OB_SYS_TENANT_ID);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = malloc_allocator->create_tenant_ctx_allocator(OB_SYS_TENANT_ID, common::ObCtxIds::WORK_AREA);
|
||||
@ -42,24 +45,25 @@ public:
|
||||
}
|
||||
|
||||
virtual void TearDown() override
|
||||
{}
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
#define CALL(func, ...) \
|
||||
func(__VA_ARGS__); \
|
||||
ASSERT_FALSE(HasFatalFailure());
|
||||
#define CALL(func, ...) func(__VA_ARGS__); ASSERT_FALSE(HasFatalFailure());
|
||||
|
||||
class TestRARowStore : public blocksstable::TestDataFilePrepare {
|
||||
class TestRARowStore : public blocksstable::TestDataFilePrepare
|
||||
{
|
||||
public:
|
||||
TestRARowStore() : blocksstable::TestDataFilePrepare("TestDiskIR", 2 << 20, 1000)
|
||||
{}
|
||||
TestRARowStore() : blocksstable::TestDataFilePrepare("TestDiskIR", 2<<20, 1000)
|
||||
{
|
||||
}
|
||||
|
||||
virtual void SetUp() override
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
blocksstable::TestDataFilePrepare::SetUp();
|
||||
ret = blocksstable::ObMacroFileManager::get_instance().init();
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
ret = blocksstable::ObMacroFileManager::get_instance().init();
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
row_.count_ = COLS;
|
||||
row_.cells_ = cells_;
|
||||
@ -80,7 +84,7 @@ public:
|
||||
common::ObModItem mod;
|
||||
}
|
||||
|
||||
ObNewRow& gen_row(int64_t row_id)
|
||||
ObNewRow &gen_row(int64_t row_id)
|
||||
{
|
||||
cells_[0].set_int(row_id);
|
||||
int64_t max_size = 512;
|
||||
@ -93,9 +97,9 @@ public:
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void verify_row(T& reader, int64_t id, bool verify_all = false)
|
||||
void verify_row(T &reader, int64_t id, bool verify_all = false)
|
||||
{
|
||||
const ObNewRow* r = NULL;
|
||||
const ObNewRow *r = NULL;
|
||||
int ret = reader.get_row(id, r);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
@ -114,7 +118,8 @@ public:
|
||||
return verify_row(rs_, id, verify_all);
|
||||
}
|
||||
|
||||
void verify_rows(int64_t begin, int64_t end, const char* mode = "scan", int64_t cnt = 0, bool verify_all = false)
|
||||
void verify_rows(int64_t begin, int64_t end,
|
||||
const char *mode = "scan", int64_t cnt = 0, bool verify_all = false)
|
||||
{
|
||||
if (strcmp(mode, "scan") == 0) {
|
||||
for (int64_t i = begin; i < end; i++) {
|
||||
@ -143,6 +148,7 @@ public:
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
const static int64_t COLS = 3;
|
||||
bool enable_big_row_ = false;
|
||||
ObObj cells_[COLS];
|
||||
@ -151,7 +157,7 @@ protected:
|
||||
|
||||
int64_t tenant_id_ = OB_SYS_TENANT_ID;
|
||||
int64_t ctx_id_ = ObCtxIds::WORK_AREA;
|
||||
const char* label_ = ObModIds::OB_SQL_ROW_STORE;
|
||||
const char *label_ = ObModIds::OB_SQL_ROW_STORE;
|
||||
|
||||
const static int64_t BUF_SIZE = 2 << 20;
|
||||
char str_buf_[BUF_SIZE];
|
||||
@ -159,7 +165,7 @@ protected:
|
||||
|
||||
TEST_F(TestRARowStore, basic)
|
||||
{
|
||||
CALL(append_rows, 3000); // approximate 1MB, index block not needed
|
||||
CALL(append_rows, 3000); // approximate 1MB, index block not needed
|
||||
CALL(verify_rows, 0, rs_.get_row_cnt(), "scan", 0, true);
|
||||
CALL(verify_rows, 0, rs_.get_row_cnt(), "rscan", 0, true);
|
||||
CALL(verify_rows, 0, rs_.get_row_cnt(), "get", rs_.get_row_cnt(), true);
|
||||
@ -172,7 +178,7 @@ TEST_F(TestRARowStore, basic)
|
||||
}
|
||||
|
||||
int ret = OB_SUCCESS;
|
||||
const ObNewRow* row = NULL;
|
||||
const ObNewRow *row = NULL;
|
||||
ret = rs_.get_row(-1, row);
|
||||
ASSERT_NE(OB_SUCCESS, ret);
|
||||
ret = rs_.get_row(rs_.get_row_cnt(), row);
|
||||
@ -257,6 +263,7 @@ TEST_F(TestRARowStore, finish_add_row)
|
||||
CALL(verify_rows, 0, rs_.get_row_cnt(), "scan");
|
||||
CALL(verify_rows, 0, rs_.get_row_cnt(), "get", 1000);
|
||||
|
||||
|
||||
rs_.reset();
|
||||
ret = rs_.init(0, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
@ -289,7 +296,7 @@ TEST_F(TestRARowStore, multi_reader)
|
||||
ObRARowStore::Reader reader1(rs_);
|
||||
ObRARowStore::Reader reader2(rs_);
|
||||
|
||||
ObRARowStore::Reader* readers[] = {&reader0, &reader1, &reader2};
|
||||
ObRARowStore::Reader *readers[] = { &reader0, &reader1, &reader2 };
|
||||
for (int64_t i = 0; i < 10000; ++i) {
|
||||
int64_t row_id = random() % rs_.get_row_cnt();
|
||||
CALL(verify_row, rs_, row_id);
|
||||
@ -414,7 +421,7 @@ TEST_F(TestRARowStore, start_dump_by_total_mem_used)
|
||||
int64_t avg_row_size = rs_.get_mem_hold() / rs_.get_row_cnt();
|
||||
LOG_INFO("average row size", K(avg_row_size));
|
||||
|
||||
lib::ObMallocAllocator* malloc_allocator = lib::ObMallocAllocator::get_instance();
|
||||
lib::ObMallocAllocator *malloc_allocator = lib::ObMallocAllocator::get_instance();
|
||||
malloc_allocator->set_tenant_limit(OB_SYS_TENANT_ID, 1L << 30);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
// 50MB for work area
|
||||
@ -425,6 +432,7 @@ TEST_F(TestRARowStore, start_dump_by_total_mem_used)
|
||||
ret = rs_.init(0, tenant_id_, ctx_id_, label_);
|
||||
ASSERT_EQ(OB_SUCCESS, ret);
|
||||
|
||||
|
||||
// case1: trigger dump by memory mod usage (60% of limit, 30MB)
|
||||
// write 28MB, all in memory
|
||||
CALL(append_rows, (28L << 20) / avg_row_size);
|
||||
@ -443,7 +451,7 @@ TEST_F(TestRARowStore, start_dump_by_total_mem_used)
|
||||
attr.tenant_id_ = tenant_id_;
|
||||
attr.ctx_id_ = ctx_id_;
|
||||
// memory ctx hold 20MB
|
||||
void* mem = ob_malloc(20L << 20, attr);
|
||||
void *mem = ob_malloc(20L << 20, attr);
|
||||
|
||||
// write 15MB, in memory
|
||||
CALL(append_rows, (15L << 20) / avg_row_size);
|
||||
@ -482,20 +490,21 @@ TEST_F(TestRARowStore, start_dump_by_total_mem_used)
|
||||
ASSERT_EQ(rs_.get_file_size(), 0);
|
||||
}
|
||||
|
||||
} // end namespace sql
|
||||
} // end namespace oceanbase
|
||||
} // end namespace sql
|
||||
} // end namespace oceanbase
|
||||
|
||||
|
||||
void ignore_sig(int sig)
|
||||
{
|
||||
UNUSED(sig);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
signal(49, ignore_sig);
|
||||
oceanbase::common::ObLogger::get_logger().set_log_level("TRACE");
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
auto* env = new (oceanbase::sql::TestEnv);
|
||||
auto *env = new (oceanbase::sql::TestEnv);
|
||||
testing::AddGlobalTestEnvironment(env);
|
||||
int ret = RUN_ALL_TESTS();
|
||||
OB_LOGGER.disable();
|
||||
|
||||
@ -15,8 +15,10 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include "sql/engine/basic/ob_ra_row_store.h"
|
||||
|
||||
namespace oceanbase {
|
||||
namespace sql {
|
||||
namespace oceanbase
|
||||
{
|
||||
namespace sql
|
||||
{
|
||||
using namespace common;
|
||||
|
||||
TEST(RARowStore, keep_projector)
|
||||
@ -49,7 +51,7 @@ TEST(RARowStore, keep_projector)
|
||||
r.projector_size_++;
|
||||
ASSERT_EQ(OB_SUCCESS, rs.add_row(r));
|
||||
|
||||
const ObNewRow* rr = NULL;
|
||||
const ObNewRow *rr = NULL;
|
||||
ASSERT_EQ(OB_SUCCESS, rs.get_row(1, rr));
|
||||
|
||||
ASSERT_TRUE(NULL != rr->projector_);
|
||||
@ -67,19 +69,11 @@ TEST(RARowStore, keep_projector)
|
||||
ASSERT_EQ(2, r.get_cell(1).get_int());
|
||||
}
|
||||
|
||||
class ObEmptyAlloc : public ObIAllocator {
|
||||
void* alloc(const int64_t) override
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
void* alloc(const int64_t, const ObMemAttr&) override
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
void free(void *ptr) override
|
||||
{
|
||||
UNUSED(ptr);
|
||||
}
|
||||
class ObEmptyAlloc : public ObIAllocator
|
||||
{
|
||||
void *alloc(const int64_t) override { return NULL; }
|
||||
void *alloc(const int64_t, const ObMemAttr &) override { return NULL; }
|
||||
void free(void *ptr) override { UNUSED(ptr); }
|
||||
};
|
||||
|
||||
TEST(RARowStore, alloc_project_fail)
|
||||
@ -105,10 +99,10 @@ TEST(RARowStore, alloc_project_fail)
|
||||
ASSERT_EQ(OB_ALLOCATE_MEMORY_FAILED, rs.add_row(r));
|
||||
}
|
||||
|
||||
} // end namespace sql
|
||||
} // end namespace oceanbase
|
||||
} // end namespace sql
|
||||
} // end namespace oceanbase
|
||||
|
||||
int main(int argc, char** argv)
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
|
||||
Reference in New Issue
Block a user