database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity
Files
4ac38ca67adce44ba2e8a574d052dcd6ea342a11
doris/be/test/olap/tablet_meta_test.cpp
Adonis Ling 16a394da0e [chore](build) Use include-what-you-use to optimize includes (PART III) (#18958) 
Currently, there are some useless includes in the codebase. We can use a tool named include-what-you-use to optimize these includes. By using a strict include-what-you-use policy, we can get lots of benefits from it.
2023-04-24 14:51:51 +08:00

249 lines
9.5 KiB
C++
Raw Blame History

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "olap/tablet_meta.h"
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
#include <memory>
#include <string>
#include <utility>
#include "gtest/gtest_pred_impl.h"
#include "olap/rowset/rowset.h"
#include "olap/tablet_schema.h"
#include "testutil/mock_rowset.h"
namespace doris {
TEST(TabletMetaTest, SaveAndParse) {
std::string meta_path = "./be/test/olap/test_data/tablet_meta_test.hdr";
TabletMeta old_tablet_meta(1, 2, 3, 3, 4, 5, TTabletSchema(), 6, {{7, 8}}, UniqueId(9, 10),
TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F);
EXPECT_EQ(Status::OK(), old_tablet_meta.save(meta_path));
{
// Just to make stack space dirty
TabletMeta new_tablet_meta;
new_tablet_meta._preferred_rowset_type = BETA_ROWSET;
}
TabletMeta new_tablet_meta;
new_tablet_meta.create_from_file(meta_path);
EXPECT_EQ(old_tablet_meta, new_tablet_meta);
}
TEST(TabletMetaTest, TestReviseMeta) {
TabletMeta tablet_meta;
std::vector<RowsetSharedPtr> src_rowsets;
std::vector<RowsetId> rsids;
// src rowsets
for (int i = 0; i < 4; i++) {
RowsetMetaPB rs_meta_pb;
RowsetId rowset_id;
rowset_id.init(i);
rsids.push_back(rowset_id);
rs_meta_pb.set_rowset_id_v2(rowset_id.to_string());
rs_meta_pb.set_num_segments(2);
rs_meta_pb.set_start_version(i);
rs_meta_pb.set_end_version(i);
RowsetMetaSharedPtr meta_ptr = std::make_shared<RowsetMeta>();
meta_ptr->init_from_pb(rs_meta_pb);
RowsetSharedPtr rowset_ptr;
TabletSchemaSPtr schema = std::make_shared<TabletSchema>();
MockRowset::create_rowset(schema, "", meta_ptr, &rowset_ptr, false);
src_rowsets.push_back(rowset_ptr);
tablet_meta.add_rs_meta(rowset_ptr->rowset_meta());
}
ASSERT_EQ(4, tablet_meta.all_rs_metas().size());
tablet_meta.delete_bitmap().add({rsids[0], 1, 1}, 1);
tablet_meta.delete_bitmap().add({rsids[1], 0, 2}, 2);
tablet_meta.delete_bitmap().add({rsids[2], 1, 1}, 1);
tablet_meta.delete_bitmap().add({rsids[3], 0, 2}, 3);
tablet_meta.delete_bitmap().add({rsids[3], 0, 4}, 4);
ASSERT_EQ(5, tablet_meta.delete_bitmap().delete_bitmap.size());
std::vector<RowsetMetaSharedPtr> new_rowsets;
new_rowsets.push_back(src_rowsets[2]->rowset_meta());
new_rowsets.push_back(src_rowsets[3]->rowset_meta());
tablet_meta.revise_rs_metas(std::move(new_rowsets));
// Take a snapshot with max_version=3.
DeleteBitmap snap = tablet_meta.delete_bitmap().snapshot(3);
tablet_meta.revise_delete_bitmap_unlocked(snap);
ASSERT_EQ(2, tablet_meta.all_rs_metas().size());
ASSERT_EQ(2, tablet_meta.delete_bitmap().delete_bitmap.size());
for (auto entry : tablet_meta.delete_bitmap().delete_bitmap) {
RowsetId rsid = std::get<0>(entry.first);
ASSERT_TRUE(rsid == rsids[2] || rsid == rsids[3]);
int64_t version = std::get<2>(entry.first);
ASSERT_TRUE(version <= 3); // should not contain versions greater than 3.
}
}
TEST(TabletMetaTest, TestDeleteBitmap) {
std::unique_ptr<DeleteBitmap> dbmp(new DeleteBitmap(10086));
auto gen1 = [&dbmp](int64_t max_rst_id, uint32_t max_seg_id, uint32_t max_row) {
for (int64_t i = 0; i < max_rst_id; ++i) {
for (uint32_t j = 0; j < max_seg_id; ++j) {
for (uint32_t k = 0; k < max_row; ++k) {
dbmp->add({RowsetId {2, 0, 1, i}, j, 0}, k);
}
}
}
};
gen1(10, 20, 1000);
dbmp->add({RowsetId {2, 0, 1, 2}, 2, 0}, 2); // redundant
{
roaring::Roaring d;
dbmp->get({RowsetId {2, 0, 1, 2}, 0, 0}, &d);
EXPECT_EQ(d.cardinality(), 1000);
d -= *dbmp->get({RowsetId {2, 0, 1, 2}, 0, 0});
EXPECT_EQ(d.cardinality(), 0);
}
// Add version 1 and 2
dbmp->add({RowsetId {2, 0, 1, 1}, 1, 1}, 1100);
dbmp->add({RowsetId {2, 0, 1, 1}, 1, 1}, 1101);
dbmp->add({RowsetId {2, 0, 1, 1}, 1, 1}, 1102);
dbmp->add({RowsetId {2, 0, 1, 1}, 1, 1}, 1103);
dbmp->add({RowsetId {2, 0, 1, 1}, 1, 2}, 1104);
ASSERT_EQ(dbmp->delete_bitmap.size(), 10 * 20 + 2);
{
auto snap = dbmp->snapshot(1);
auto it = snap.delete_bitmap.begin();
while (it != snap.delete_bitmap.end()) {
ASSERT_TRUE(std::get<2>(it->first) <= 1);
it++;
}
ASSERT_EQ(snap.delete_bitmap.size(), 10 * 20 + 1);
}
{
auto snap = dbmp->snapshot(0);
auto it = snap.delete_bitmap.begin();
while (it != snap.delete_bitmap.end()) {
ASSERT_TRUE(std::get<2>(it->first) <= 0);
it++;
}
ASSERT_EQ(snap.delete_bitmap.size(), 10 * 20);
}
{ // Bitmap of certain verisons only get their own row ids
auto bm = dbmp->get({RowsetId {2, 0, 1, 1}, 1, 2});
ASSERT_EQ(bm->cardinality(), 1);
ASSERT_FALSE(bm->contains(999));
ASSERT_FALSE(bm->contains(1100));
ASSERT_TRUE(bm->contains(1104));
}
{
// test remove
// Nothing removed
dbmp->remove({RowsetId {2, 0, 1, 1}, 0, 0}, {RowsetId {2, 0, 1, 1}, 0, 0});
ASSERT_EQ(dbmp->delete_bitmap.size(), 10 * 20 + 2);
dbmp->remove({RowsetId {2, 0, 1, 100}, 0, 0}, {RowsetId {2, 0, 1, 100}, 50000, 0});
ASSERT_EQ(dbmp->delete_bitmap.size(), 10 * 20 + 2);
// Remove all seg of rowset {2,0,1,0}
dbmp->remove({RowsetId {2, 0, 1, 0}, 0, 0}, {RowsetId {2, 0, 1, 0}, 5000, 0});
ASSERT_EQ(dbmp->delete_bitmap.size(), 9 * 20 + 2);
// Remove all rowset {2,0,1,7} to {2,0,1,9}
dbmp->remove({RowsetId {2, 0, 1, 8}, 0, 0}, {RowsetId {2, 0, 1, 9}, 5000, 0});
ASSERT_EQ(dbmp->delete_bitmap.size(), 7 * 20 + 2);
}
{
DeleteBitmap db_upper(10086);
dbmp->subset({RowsetId {2, 0, 1, 1}, 1, 0}, {RowsetId {2, 0, 1, 1}, 1000000, 0}, &db_upper);
roaring::Roaring d;
ASSERT_EQ(db_upper.get({RowsetId {2, 0, 1, 1}, 1, 1}, &d), 0);
ASSERT_EQ(d.cardinality(), 4);
ASSERT_EQ(db_upper.get({RowsetId {2, 0, 1, 1}, 1, 2}, &d), 0);
ASSERT_EQ(d.cardinality(), 1);
ASSERT_EQ(db_upper.delete_bitmap.size(), 20);
}
{
auto old_size = dbmp->delete_bitmap.size();
// test merge
DeleteBitmap other(10086);
other.add({RowsetId {2, 0, 1, 1}, 1, 1}, 1100);
dbmp->merge(other);
ASSERT_EQ(dbmp->delete_bitmap.size(), old_size);
other.add({RowsetId {2, 0, 1, 1}, 1001, 1}, 1100);
other.add({RowsetId {2, 0, 1, 1}, 1002, 1}, 1100);
dbmp->merge(other);
ASSERT_EQ(dbmp->delete_bitmap.size(), old_size + 2);
}
////////////////////////////////////////////////////////////////////////////
// Cache test
////////////////////////////////////////////////////////////////////////////
// Aggregation bitmap contains all row ids that are in versions smaller or
{
// equal to the given version, boundary test
auto bm = dbmp->get_agg({RowsetId {2, 0, 1, 1}, 1, 2});
ASSERT_EQ(bm->cardinality(), 1005);
ASSERT_TRUE(bm->contains(999));
ASSERT_TRUE(bm->contains(1100));
ASSERT_TRUE(bm->contains(1101));
ASSERT_TRUE(bm->contains(1102));
ASSERT_TRUE(bm->contains(1103));
ASSERT_TRUE(bm->contains(1104));
bm = dbmp->get_agg({RowsetId {2, 0, 1, 1}, 1, 2});
ASSERT_EQ(bm->cardinality(), 1005);
}
// Aggregation bitmap contains all row ids that are in versions smaller or
// equal to the given version, normal test
{
auto bm = dbmp->get_agg({RowsetId {2, 0, 1, 1}, 1, 1000});
ASSERT_EQ(bm->cardinality(), 1005);
ASSERT_TRUE(bm->contains(999));
ASSERT_TRUE(bm->contains(1100));
ASSERT_TRUE(bm->contains(1101));
ASSERT_TRUE(bm->contains(1102));
ASSERT_TRUE(bm->contains(1103));
ASSERT_TRUE(bm->contains(1104));
bm = dbmp->get_agg({RowsetId {2, 0, 1, 1}, 1, 1000});
ASSERT_EQ(bm->cardinality(), 1005);
}
// Check data is not messed-up
ASSERT_TRUE(dbmp->contains({RowsetId {2, 0, 1, 1}, 1, 2}, 1104));
ASSERT_FALSE(dbmp->contains({RowsetId {2, 0, 1, 1}, 1, 2}, 1103));
ASSERT_TRUE(dbmp->contains_agg({RowsetId {2, 0, 1, 1}, 1, 2}, 1104));
ASSERT_TRUE(dbmp->contains_agg({RowsetId {2, 0, 1, 1}, 1, 2}, 1103));
// Test c-tor of agg cache with global LRU
int cached_cardinality = dbmp->get_agg({RowsetId {2, 0, 1, 1}, 1, 2})->cardinality();
{
// New delete bitmap with old agg cache
std::unique_ptr<DeleteBitmap> dbmp(new DeleteBitmap(10086));
auto bm = dbmp->get_agg({RowsetId {2, 0, 1, 1}, 1, 2});
ASSERT_TRUE(bm->contains(1104));
ASSERT_EQ(bm->cardinality(), cached_cardinality);
}
}
} // namespace doris
Reference in New Issue View Git Blame Copy Permalink