database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity
Files
7fb0913d35940563bd2b78cd14cf3e8a00a40bb9
doris/be/test/olap/rowid_conversion_test.cpp

535 lines
22 KiB
C++
Raw Blame History

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "olap/rowid_conversion.h"
#include <gtest/gtest.h>
#include "common/logging.h"
#include "olap/data_dir.h"
#include "olap/delete_handler.h"
#include "olap/merger.h"
#include "olap/row_cursor.h"
#include "olap/rowset/rowset.h"
#include "olap/rowset/rowset_factory.h"
#include "olap/rowset/rowset_reader.h"
#include "olap/rowset/rowset_reader_context.h"
#include "olap/rowset/rowset_writer.h"
#include "olap/rowset/rowset_writer_context.h"
#include "olap/schema.h"
#include "olap/tablet_schema.h"
#include "olap/tablet_schema_helper.h"
#include "util/file_utils.h"
namespace doris {
static const uint32_t MAX_PATH_LEN = 1024;
static StorageEngine* k_engine = nullptr;
class TestRowIdConversion : public testing::TestWithParam<std::tuple<KeysType, bool, bool>> {
protected:
void SetUp() override {
char buffer[MAX_PATH_LEN];
EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
absolute_dir = std::string(buffer) + kTestDir;
if (FileUtils::check_exist(absolute_dir)) {
EXPECT_TRUE(FileUtils::remove_all(absolute_dir).ok());
}
EXPECT_TRUE(FileUtils::create_dir(absolute_dir).ok());
EXPECT_TRUE(FileUtils::create_dir(absolute_dir + "/tablet_path").ok());
_data_dir = std::make_unique<DataDir>(absolute_dir);
_data_dir->update_capacity();
doris::EngineOptions options;
k_engine = new StorageEngine(options);
StorageEngine::_s_instance = k_engine;
}
void TearDown() override {
if (FileUtils::check_exist(absolute_dir)) {
EXPECT_TRUE(FileUtils::remove_all(absolute_dir).ok());
}
if (k_engine != nullptr) {
k_engine->stop();
delete k_engine;
k_engine = nullptr;
}
}
TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS) {
TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
TabletSchemaPB tablet_schema_pb;
tablet_schema_pb.set_keys_type(keys_type);
tablet_schema_pb.set_num_short_key_columns(2);
tablet_schema_pb.set_num_rows_per_row_block(1024);
tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
tablet_schema_pb.set_next_column_unique_id(4);
ColumnPB* column_1 = tablet_schema_pb.add_column();
column_1->set_unique_id(1);
column_1->set_name("c1");
column_1->set_type("INT");
column_1->set_is_key(true);
column_1->set_length(4);
column_1->set_index_length(4);
column_1->set_is_nullable(false);
column_1->set_is_bf_column(false);
ColumnPB* column_2 = tablet_schema_pb.add_column();
column_2->set_unique_id(2);
column_2->set_name("c2");
column_2->set_type("INT");
column_2->set_length(4);
column_2->set_index_length(4);
column_2->set_is_nullable(true);
column_2->set_is_key(false);
column_2->set_is_nullable(false);
column_2->set_is_bf_column(false);
tablet_schema->init_from_pb(tablet_schema_pb);
return tablet_schema;
}
void create_rowset_writer_context(TabletSchemaSPtr tablet_schema,
const SegmentsOverlapPB& overlap,
uint32_t max_rows_per_segment,
RowsetWriterContext* rowset_writer_context) {
static int64_t inc_id = 0;
RowsetId rowset_id;
rowset_id.init(inc_id);
rowset_writer_context->rowset_id = rowset_id;
rowset_writer_context->rowset_type = BETA_ROWSET;
rowset_writer_context->data_dir = _data_dir.get();
rowset_writer_context->rowset_state = VISIBLE;
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->tablet_path = "tablet_path";
rowset_writer_context->version = Version(inc_id, inc_id);
rowset_writer_context->segments_overlap = overlap;
rowset_writer_context->max_rows_per_segment = max_rows_per_segment;
inc_id++;
}
void create_and_init_rowset_reader(Rowset* rowset, RowsetReaderContext& context,
RowsetReaderSharedPtr* result) {
auto s = rowset->create_reader(result);
EXPECT_TRUE(s.ok());
EXPECT_TRUE(*result != nullptr);
s = (*result)->init(&context);
EXPECT_TRUE(s.ok());
}
RowsetSharedPtr create_rowset(
TabletSchemaSPtr tablet_schema, const SegmentsOverlapPB& overlap,
std::vector<std::vector<std::tuple<int64_t, int64_t>>> rowset_data) {
RowsetWriterContext writer_context;
if (overlap == NONOVERLAPPING) {
for (auto i = 1; i < rowset_data.size(); i++) {
auto& last_seg_data = rowset_data[i - 1];
auto& cur_seg_data = rowset_data[i];
int64_t last_seg_max = std::get<0>(last_seg_data[last_seg_data.size() - 1]);
int64_t cur_seg_min = std::get<0>(cur_seg_data[0]);
EXPECT_LT(last_seg_max, cur_seg_min);
}
}
create_rowset_writer_context(tablet_schema, overlap, UINT32_MAX, &writer_context);
std::unique_ptr<RowsetWriter> rowset_writer;
Status s = RowsetFactory::create_rowset_writer(writer_context, &rowset_writer);
EXPECT_TRUE(s.ok());
RowCursor input_row;
input_row.init(tablet_schema);
uint32_t num_rows = 0;
for (int i = 0; i < rowset_data.size(); ++i) {
MemPool mem_pool;
for (int rid = 0; rid < rowset_data[i].size(); ++rid) {
uint32_t c1 = std::get<0>(rowset_data[i][rid]);
uint32_t c2 = std::get<1>(rowset_data[i][rid]);
input_row.set_field_content(0, reinterpret_cast<char*>(&c1), &mem_pool);
input_row.set_field_content(1, reinterpret_cast<char*>(&c2), &mem_pool);
s = rowset_writer->add_row(input_row);
EXPECT_TRUE(s.ok());
num_rows++;
}
s = rowset_writer->flush();
EXPECT_TRUE(s.ok());
}
RowsetSharedPtr rowset;
rowset = rowset_writer->build();
EXPECT_TRUE(rowset != nullptr);
EXPECT_EQ(rowset_data.size(), rowset->rowset_meta()->num_segments());
EXPECT_EQ(num_rows, rowset->rowset_meta()->num_rows());
return rowset;
}
TabletSharedPtr create_tablet(const TabletSchema& tablet_schema,
bool enable_unique_key_merge_on_write, int64_t version,
bool has_delete_handler) {
std::vector<TColumn> cols;
std::unordered_map<uint32_t, uint32_t> col_ordinal_to_unique_id;
for (auto i = 0; i < tablet_schema.num_columns(); i++) {
const TabletColumn& column = tablet_schema.column(i);
TColumn col;
col.column_type.type = TPrimitiveType::INT;
col.__set_column_name(column.name());
col.__set_is_key(column.is_key());
cols.push_back(col);
col_ordinal_to_unique_id[i] = i;
}
TTabletSchema t_tablet_schema;
t_tablet_schema.__set_short_key_column_count(tablet_schema.num_short_key_columns());
t_tablet_schema.__set_schema_hash(3333);
if (tablet_schema.keys_type() == UNIQUE_KEYS) {
t_tablet_schema.__set_keys_type(TKeysType::UNIQUE_KEYS);
} else if (tablet_schema.keys_type() == DUP_KEYS) {
t_tablet_schema.__set_keys_type(TKeysType::DUP_KEYS);
}
t_tablet_schema.__set_storage_type(TStorageType::COLUMN);
t_tablet_schema.__set_columns(cols);
TabletMetaSharedPtr tablet_meta(
new TabletMeta(1, 1, 1, 1, 1, 1, t_tablet_schema, 1, col_ordinal_to_unique_id,
UniqueId(1, 2), TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F, "", enable_unique_key_merge_on_write));
if (has_delete_handler) {
// delete data with key < 1000
std::vector<TCondition> conditions;
TCondition condition;
condition.column_name = tablet_schema.column(0).name();
condition.condition_op = "<";
condition.condition_values.clear();
condition.condition_values.push_back("1000");
conditions.push_back(condition);
DeletePredicatePB del_pred;
Status st =
DeleteHandler::generate_delete_predicate(tablet_schema, conditions, &del_pred);
EXPECT_EQ(Status::OK(), st);
tablet_meta->add_delete_predicate(del_pred, version);
}
TabletSharedPtr tablet(new Tablet(tablet_meta, nullptr));
return tablet;
}
void block_create(TabletSchemaSPtr tablet_schema, vectorized::Block* block) {
block->clear();
Schema schema(tablet_schema);
const auto& column_ids = schema.column_ids();
for (size_t i = 0; i < schema.num_column_ids(); ++i) {
auto column_desc = schema.column(column_ids[i]);
auto data_type = Schema::get_data_type_ptr(*column_desc);
EXPECT_TRUE(data_type != nullptr);
auto column = data_type->create_column();
block->insert(vectorized::ColumnWithTypeAndName(std::move(column), data_type,
column_desc->name()));
}
}
void check_rowid_conversion(KeysType keys_type, bool enable_unique_key_merge_on_write,
uint32_t num_input_rowset, uint32_t num_segments,
uint32_t rows_per_segment, const SegmentsOverlapPB& overlap,
bool has_delete_handler) {
// generate input data
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
generate_input_data(num_input_rowset, num_segments, rows_per_segment, overlap, input_data);
TabletSchemaSPtr tablet_schema = create_schema(keys_type);
// create input rowset
vector<RowsetSharedPtr> input_rowsets;
SegmentsOverlapPB new_overlap = overlap;
for (auto i = 0; i < num_input_rowset; i++) {
if (overlap == OVERLAP_UNKNOWN) {
if (i == 0) {
new_overlap = NONOVERLAPPING;
} else {
new_overlap = OVERLAPPING;
}
}
RowsetSharedPtr rowset = create_rowset(tablet_schema, new_overlap, input_data[i]);
input_rowsets.push_back(rowset);
}
// create input rowset reader
vector<RowsetReaderSharedPtr> input_rs_readers;
for (auto& rowset : input_rowsets) {
RowsetReaderSharedPtr rs_reader;
EXPECT_TRUE(rowset->create_reader(&rs_reader).ok());
input_rs_readers.push_back(std::move(rs_reader));
}
// create output rowset writer
RowsetWriterContext writer_context;
create_rowset_writer_context(tablet_schema, NONOVERLAPPING, 3456, &writer_context);
std::unique_ptr<RowsetWriter> output_rs_writer;
Status s = RowsetFactory::create_rowset_writer(writer_context, &output_rs_writer);
EXPECT_TRUE(s.ok());
// merge input rowset
TabletSharedPtr tablet =
create_tablet(*tablet_schema, enable_unique_key_merge_on_write,
output_rs_writer->version().first - 1, has_delete_handler);
Merger::Statistics stats;
RowIdConversion rowid_conversion;
stats.rowid_conversion = &rowid_conversion;
s = Merger::vmerge_rowsets(tablet, READER_BASE_COMPACTION, tablet_schema, input_rs_readers,
output_rs_writer.get(), &stats);
EXPECT_TRUE(s.ok());
RowsetSharedPtr out_rowset = output_rs_writer->build();
// create output rowset reader
RowsetReaderContext reader_context;
reader_context.tablet_schema = tablet_schema;
reader_context.need_ordered_result = false;
std::vector<uint32_t> return_columns = {0, 1};
reader_context.return_columns = &return_columns;
reader_context.seek_columns = &return_columns;
reader_context.is_vec = true;
RowsetReaderSharedPtr output_rs_reader;
create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader);
// read output rowset data
vectorized::Block output_block;
std::vector<std::tuple<int64_t, int64_t>> output_data;
do {
block_create(tablet_schema, &output_block);
s = output_rs_reader->next_block(&output_block);
auto columns = output_block.get_columns_with_type_and_name();
EXPECT_EQ(columns.size(), 2);
for (auto i = 0; i < output_block.rows(); i++) {
output_data.emplace_back(columns[0].column->get_int(i),
columns[1].column->get_int(i));
}
} while (s == Status::OK());
EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_DATA_EOF), s);
EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
std::vector<uint32_t> segment_num_rows;
EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
if (has_delete_handler) {
// All keys less than 1000 are deleted by delete handler
for (auto& item : output_data) {
EXPECT_GE(std::get<0>(item), 1000);
}
}
// check rowid conversion
uint64_t count = 0;
for (auto rs_id = 0; rs_id < input_data.size(); rs_id++) {
for (auto s_id = 0; s_id < input_data[rs_id].size(); s_id++) {
for (auto row_id = 0; row_id < input_data[rs_id][s_id].size(); row_id++) {
RowLocation src(input_rowsets[rs_id]->rowset_id(), s_id, row_id);
RowLocation dst;
int res = rowid_conversion.get(src, &dst);
if (res < 0) {
continue;
}
size_t rowid_in_output_data = dst.row_id;
for (auto n = 1; n <= dst.segment_id; n++) {
rowid_in_output_data += segment_num_rows[n - 1];
}
EXPECT_EQ(std::get<0>(output_data[rowid_in_output_data]),
std::get<0>(input_data[rs_id][s_id][row_id]));
EXPECT_EQ(std::get<1>(output_data[rowid_in_output_data]),
std::get<1>(input_data[rs_id][s_id][row_id]));
count++;
}
}
}
EXPECT_EQ(count, output_data.size());
}
// if overlap == NONOVERLAPPING, all rowsets are non overlapping;
// if overlap == OVERLAPPING, all rowsets are overlapping;
// if overlap == OVERLAP_UNKNOWN, the first rowset is non overlapping, the
// others are overlaping.
void generate_input_data(
uint32_t num_input_rowset, uint32_t num_segments, uint32_t rows_per_segment,
const SegmentsOverlapPB& overlap,
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>& input_data) {
EXPECT_GE(rows_per_segment, 10);
EXPECT_GE(num_segments * rows_per_segment, 500);
bool is_overlap = false;
for (auto i = 0; i < num_input_rowset; i++) {
if (overlap == OVERLAPPING) {
is_overlap = true;
} else if (overlap == NONOVERLAPPING) {
is_overlap = false;
} else {
if (i == 0) {
is_overlap = false;
} else {
is_overlap = true;
}
}
std::vector<std::vector<std::tuple<int64_t, int64_t>>> rowset_data;
for (auto j = 0; j < num_segments; j++) {
std::vector<std::tuple<int64_t, int64_t>> segment_data;
for (auto n = 0; n < rows_per_segment; n++) {
int64_t c1 = j * rows_per_segment + n;
// There are 500 rows of data overlap between rowsets
if (i > 0) {
c1 += i * num_segments * rows_per_segment - 500;
}
if (is_overlap && j > 0) {
// There are 10 rows of data overlap between segments
c1 += j * rows_per_segment - 10;
}
int64_t c2 = c1 + 1;
segment_data.emplace_back(c1, c2);
}
rowset_data.emplace_back(segment_data);
}
input_data.emplace_back(rowset_data);
}
}
private:
const std::string kTestDir = "/ut_dir/rowid_conversion_test";
string absolute_dir;
std::unique_ptr<DataDir> _data_dir;
};
TEST_F(TestRowIdConversion, Basic) {
// rowset_id, segment_id, row_id
int input_data[11][3] = {{0, 0, 0}, {0, 0, 1}, {0, 0, 2}, {0, 0, 3}, {0, 1, 0}, {0, 1, 1},
{0, 1, 2}, {1, 0, 0}, {1, 0, 1}, {1, 0, 2}, {1, 0, 3}};
RowsetId src_rowset;
RowsetId dst_rowset;
dst_rowset.init(3);
std::vector<RowLocation> rss_row_ids;
for (auto i = 0; i < 11; i++) {
src_rowset.init(input_data[i][0]);
RowLocation rss_row_id(src_rowset, input_data[i][1], input_data[i][2]);
rss_row_ids.push_back(rss_row_id);
}
RowIdConversion rowid_conversion;
src_rowset.init(0);
std::vector<uint32_t> rs0_segment_num_rows = {4, 3};
rowid_conversion.init_segment_map(src_rowset, rs0_segment_num_rows);
src_rowset.init(1);
std::vector<uint32_t> rs1_segment_num_rows = {4};
rowid_conversion.init_segment_map(src_rowset, rs1_segment_num_rows);
rowid_conversion.set_dst_rowset_id(dst_rowset);
std::vector<uint32_t> dst_segment_num_rows = {4, 3, 4};
rowid_conversion.add(rss_row_ids, dst_segment_num_rows);
int res = 0;
src_rowset.init(0);
RowLocation src0(src_rowset, 0, 0);
RowLocation dst0;
res = rowid_conversion.get(src0, &dst0);
EXPECT_EQ(dst0.rowset_id, dst_rowset);
EXPECT_EQ(dst0.segment_id, 0);
EXPECT_EQ(dst0.row_id, 0);
EXPECT_EQ(res, 0);
src_rowset.init(0);
RowLocation src1(src_rowset, 1, 2);
RowLocation dst1;
res = rowid_conversion.get(src1, &dst1);
EXPECT_EQ(dst1.rowset_id, dst_rowset);
EXPECT_EQ(dst1.segment_id, 1);
EXPECT_EQ(dst1.row_id, 2);
EXPECT_EQ(res, 0);
src_rowset.init(1);
RowLocation src2(src_rowset, 0, 3);
RowLocation dst2;
res = rowid_conversion.get(src2, &dst2);
EXPECT_EQ(dst2.rowset_id, dst_rowset);
EXPECT_EQ(dst2.segment_id, 2);
EXPECT_EQ(dst2.row_id, 3);
EXPECT_EQ(res, 0);
src_rowset.init(1);
RowLocation src3(src_rowset, 0, 4);
RowLocation dst3;
res = rowid_conversion.get(src3, &dst3);
EXPECT_EQ(res, -1);
src_rowset.init(100);
RowLocation src4(src_rowset, 5, 4);
RowLocation dst4;
res = rowid_conversion.get(src4, &dst4);
EXPECT_EQ(res, -1);
}
INSTANTIATE_TEST_SUITE_P(
Parameters, TestRowIdConversion,
::testing::ValuesIn(std::vector<std::tuple<KeysType, bool, bool>> {
// Parameters: data_type, enable_unique_key_merge_on_write, has_delete_handler
{DUP_KEYS, false, false},
{UNIQUE_KEYS, false, false},
{UNIQUE_KEYS, true, false},
{DUP_KEYS, false, true},
{UNIQUE_KEYS, false, true},
{UNIQUE_KEYS, true, true}}));
TEST_P(TestRowIdConversion, Conversion) {
KeysType keys_type = std::get<0>(GetParam());
bool enable_unique_key_merge_on_write = std::get<1>(GetParam());
bool has_delete_handler = std::get<2>(GetParam());
// if num_input_rowset = 2, VCollectIterator::Level1Iterator::_merge = flase
// if num_input_rowset = 3, VCollectIterator::Level1Iterator::_merge = true
for (auto num_input_rowset = 2; num_input_rowset <= 3; num_input_rowset++) {
uint32_t rows_per_segment = 4567;
// RowsetReader: SegmentIterator
{
uint32_t num_segments = 1;
SegmentsOverlapPB overlap = NONOVERLAPPING;
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
check_rowid_conversion(keys_type, enable_unique_key_merge_on_write, num_input_rowset,
num_segments, rows_per_segment, overlap, has_delete_handler);
}
// RowsetReader: VMergeIterator
{
uint32_t num_segments = 2;
SegmentsOverlapPB overlap = OVERLAPPING;
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
check_rowid_conversion(keys_type, enable_unique_key_merge_on_write, num_input_rowset,
num_segments, rows_per_segment, overlap, has_delete_handler);
}
// RowsetReader: VUnionIterator
{
uint32_t num_segments = 2;
SegmentsOverlapPB overlap = NONOVERLAPPING;
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
check_rowid_conversion(keys_type, enable_unique_key_merge_on_write, num_input_rowset,
num_segments, rows_per_segment, overlap, has_delete_handler);
}
// RowsetReader: VUnionIterator + VMergeIterator
{
uint32_t num_segments = 2;
SegmentsOverlapPB overlap = OVERLAP_UNKNOWN;
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
check_rowid_conversion(keys_type, enable_unique_key_merge_on_write, num_input_rowset,
num_segments, rows_per_segment, overlap, has_delete_handler);
}
}
}
} // namespace doris
Reference in New Issue View Git Blame Copy Permalink