hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE]: use parameter _ob_ddl_temp_file_compress_func to control compression in ddl.

Browse Source
This commit is contained in:
Monk-Liu
2024-03-07 04:45:49 +00:00
committed by ob-robot
parent 5279be2c40
commit 92fca7def1
32 changed files with 491 additions and 780 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
unittest/sql/parser/test_parser.result
View File

@ -5696,6 +5696,7 @@ question_mask_size: 0
}
]
},
{ },
{ }
]
}
@ -5793,6 +5794,7 @@ question_mask_size: 0
}
]
},
{ },
{ }
]
}
@ -5895,6 +5897,7 @@ question_mask_size: 0
}
]
},
{ },
{ }
]
}

341
unittest/storage/ddl/test_chunk_compact_store.cpp
View File

@ -206,88 +206,13 @@ void TestCompactChunk::TearDown()
TestDataFilePrepare::TearDown();
}
TEST_F(TestCompactChunk, test_read_writer)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true, share::SORT_DEFAULT_LEVEL);
StoredRow **sr;
ret = row_generate_.get_stored_row(sr);
ASSERT_EQ(ret, OB_SUCCESS);
char *buf = reinterpret_cast<char*>(sr);
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
StoredRow *tmp_sr = (StoredRow *)(buf + pos);
ret = cs_chunk.add_row(*tmp_sr);
ASSERT_EQ(ret, OB_SUCCESS);
pos += tmp_sr->row_size_;
}
ret = cs_chunk.finish_add_row();
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
int64_t result = 0;
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_next_row(cur_sr);
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
}
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
result += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(result == 64);
}
}
TEST_F(TestCompactChunk, test_batch)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true, share::SORT_DEFAULT_LEVEL);
StoredRow **sr;
ret = row_generate_.get_stored_row(sr);
ASSERT_EQ(ret, OB_SUCCESS);
char *buf = reinterpret_cast<char*>(sr);
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
StoredRow *tmp_sr = (StoredRow *)(buf + pos);
ret = cs_chunk.add_row(*tmp_sr);
ASSERT_EQ(ret, OB_SUCCESS);
pos += tmp_sr->row_size_;
}
ret = cs_chunk.finish_add_row();
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
int64_t result = 0;
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_next_row(cur_sr);
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
}
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
result += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(result == 64);
}
}
TEST_F(TestCompactChunk, test_read_writer_compact)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true, share::SORT_COMPACT_LEVEL);
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true);
ChunkRowMeta row_meta(allocator_);
row_meta.col_cnt_ = COLUMN_CNT;
row_meta.fixed_cnt_ = COLUMN_CNT;
@ -343,7 +268,7 @@ TEST_F(TestCompactChunk, test_read_writer_compact_vardata)
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true, share::SORT_COMPACT_LEVEL);
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true);
ChunkRowMeta row_meta(allocator_);
row_meta.col_cnt_ = COLUMN_CNT;
row_meta.fixed_cnt_ = 0;
@ -391,272 +316,12 @@ TEST_F(TestCompactChunk, test_read_writer_compact_vardata)
}
}
TEST_F(TestCompactChunk, test_read_writer_compression)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true, share::SORT_COMPRESSION_LEVEL, ZSTD_COMPRESSOR);
StoredRow **sr;
ret = row_generate_.get_stored_row(sr);
ASSERT_EQ(ret, OB_SUCCESS);
char *buf = reinterpret_cast<char*>(sr);
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
StoredRow *tmp_sr = (StoredRow *)(buf + pos);
ret = cs_chunk.add_row(*tmp_sr);
ASSERT_EQ(ret, OB_SUCCESS);
pos += tmp_sr->row_size_;
}
ret = cs_chunk.finish_add_row();
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
int64_t result = 0;
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_next_row(cur_sr);
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
}
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
result += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(result == 64);
}
}
TEST_F(TestCompactChunk, test_irregular)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/, share::SORT_DEFAULT_LEVEL);
StoredRow **sr;
ret = row_generate_.get_stored_row_irregular(sr);
ASSERT_EQ(ret, OB_SUCCESS);
char *buf = reinterpret_cast<char*>(sr);
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
StoredRow *tmp_sr = (StoredRow *)(buf + pos);
ret = cs_chunk.add_row(*tmp_sr);
ASSERT_EQ(ret, OB_SUCCESS);
pos += tmp_sr->row_size_;
}
ret = cs_chunk.finish_add_row();
ASSERT_EQ(ret, OB_SUCCESS);
for (int j = 0; OB_SUCC(ret) && j < 1; j++ ) {
int64_t total_res = 0;
//cs_chunk.rescan();
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
int64_t result = 0;
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_next_row(cur_sr);
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
}
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
result += *(int64_t *)(cur_cell.ptr_);
total_res += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(result == ((1024 * i * COLUMN_CNT) + ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
OB_ASSERT(total_res == ((1024 * (BATCH_SIZE-1) * BATCH_SIZE * COLUMN_CNT / 2) + BATCH_SIZE * ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
}
TEST_F(TestCompactChunk, test_rescan_irregular_1)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/, share::SORT_DEFAULT_LEVEL);
StoredRow **sr;
ret = row_generate_.get_stored_row_irregular(sr);
ASSERT_EQ(ret, OB_SUCCESS);
char *buf = reinterpret_cast<char*>(sr);
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
StoredRow *tmp_sr = (StoredRow *)(buf + pos);
ret = cs_chunk.add_row(*tmp_sr);
ASSERT_EQ(ret, OB_SUCCESS);
pos += tmp_sr->row_size_;
}
ret = cs_chunk.finish_add_row();
ASSERT_EQ(ret, OB_SUCCESS);
for (int j = 0; OB_SUCC(ret) && j < 1; j++ ) {
int64_t total_res = 0;
cs_chunk.rescan();
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
int64_t result = 0;
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_next_row(cur_sr);
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
}
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
result += *(int64_t *)(cur_cell.ptr_);
total_res += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(result == ((1024 * i * COLUMN_CNT) + ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
OB_ASSERT(total_res == ((1024 * (BATCH_SIZE-1) * BATCH_SIZE * COLUMN_CNT / 2) + BATCH_SIZE * ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
}
TEST_F(TestCompactChunk, test_rescan)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/, share::SORT_DEFAULT_LEVEL);
StoredRow **sr;
ret = row_generate_.get_stored_row(sr);
ASSERT_EQ(ret, OB_SUCCESS);
char *buf = reinterpret_cast<char*>(sr);
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
StoredRow *tmp_sr = (StoredRow *)(buf + pos);
ret = cs_chunk.add_row(*tmp_sr);
ASSERT_EQ(ret, OB_SUCCESS);
pos += tmp_sr->row_size_;
}
ret = cs_chunk.finish_add_row();
ASSERT_EQ(ret, OB_SUCCESS);
for (int j = 0; OB_SUCC(ret) && j < 10; j++ ) {
cs_chunk.rescan();
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
int64_t result = 0;
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_next_row(cur_sr);
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
}
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
result += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(result == 64);
}
}
}
TEST_F(TestCompactChunk, test_rescan_irregular)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/, share::SORT_DEFAULT_LEVEL);
StoredRow **sr;
ret = row_generate_.get_stored_row_irregular(sr);
ASSERT_EQ(ret, OB_SUCCESS);
char *buf = reinterpret_cast<char*>(sr);
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
StoredRow *tmp_sr = (StoredRow *)(buf + pos);
ret = cs_chunk.add_row(*tmp_sr);
ASSERT_EQ(ret, OB_SUCCESS);
pos += tmp_sr->row_size_;
}
ret = cs_chunk.finish_add_row();
ASSERT_EQ(ret, OB_SUCCESS);
for (int j = 0; OB_SUCC(ret) && j < 2; j++ ) {
int64_t total_res = 0;
cs_chunk.rescan();
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
int64_t result = 0;
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_next_row(cur_sr);
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
}
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
result += *(int64_t *)(cur_cell.ptr_);
total_res += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(result == ((1024 * i * COLUMN_CNT) + ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
OB_ASSERT(total_res == ((1024 * (BATCH_SIZE-1) * BATCH_SIZE * COLUMN_CNT / 2) + BATCH_SIZE * ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
}
TEST_F(TestCompactChunk, test_rescan_get_last_row)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/, share::SORT_DEFAULT_LEVEL);
StoredRow **sr;
ret = row_generate_.get_stored_row_irregular(sr);
ASSERT_EQ(ret, OB_SUCCESS);
char *buf = reinterpret_cast<char*>(sr);
int64_t pos = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
StoredRow *tmp_sr = (StoredRow *)(buf + pos);
ret = cs_chunk.add_row(*tmp_sr);
ASSERT_EQ(ret, OB_SUCCESS);
pos += tmp_sr->row_size_;
// get last row
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_last_stored_row(cur_sr);
ASSERT_EQ(ret, OB_SUCCESS);
int64_t res = 0;
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
res += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(res == ((1024 * i * COLUMN_CNT) + ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
ret = cs_chunk.finish_add_row();
ASSERT_EQ(ret, OB_SUCCESS);
for (int j = 0; OB_SUCC(ret) && j < 2; j++ ) {
int64_t total_res = 0;
cs_chunk.rescan();
for (int64_t i = 0; OB_SUCC(ret) && i < BATCH_SIZE; i++) {
int64_t result = 0;
const StoredRow *cur_sr = nullptr;
ret = cs_chunk.get_next_row(cur_sr);
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
}
ASSERT_EQ(ret, OB_SUCCESS);
for (int64_t k = 0; k < cur_sr->cnt_; k++) {
ObDatum cur_cell = cur_sr->cells()[k];
result += *(int64_t *)(cur_cell.ptr_);
total_res += *(int64_t *)(cur_cell.ptr_);
}
OB_ASSERT(result == ((1024 * i * COLUMN_CNT) + ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
OB_ASSERT(total_res == ((1024 * (BATCH_SIZE-1) * BATCH_SIZE * COLUMN_CNT / 2) + BATCH_SIZE * ((COLUMN_CNT - 1) * COLUMN_CNT / 2)));
}
}
TEST_F(TestCompactChunk, test_rescan_get_last_row_compact)
{
int ret = OB_SUCCESS;
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/, share::SORT_COMPACT_LEVEL);
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/);
ChunkRowMeta row_meta(allocator_);
row_meta.col_cnt_ = COLUMN_CNT;
row_meta.fixed_cnt_ = 0;