hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix ha create sstable do not fill tx scn bug.

Browse Source
This commit is contained in:
godyangfight 2024-09-03 14:45:33 +00:00 committed by ob-robot
parent a97021df02
commit 0cd88a4196
9 changed files with 117 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h
View File

@ -596,6 +596,7 @@ void TestIndexBlockDataPrepare::close_builder_and_prepare_sstable(const int64_t
param.master_key_id_ = res.master_key_id_;
param.table_backup_flag_.reset();
param.table_shared_flag_.reset();
param.filled_tx_scn_ = table_key.get_end_scn();
MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH);
if (merge_type_ == MAJOR_MERGE) {
OK(ObSSTableMergeRes::fill_column_checksum_for_empty_major(param.column_cnt_, param.column_checksums_));

32
mittest/mtlenv/storage/test_ls_tablet_service.cpp
View File

@ -1149,6 +1149,38 @@ TEST_F(TestLSTabletService, test_empty_shell_mds_compat)
ASSERT_EQ(OB_SUCCESS, ret);
}
TEST_F(TestLSTabletService, test_serialize_sstable_with_min_filled_tx_scn)
{
ObTabletID tablet_id(99999);
blocksstable::ObSSTable sstable;
share::schema::ObTableSchema schema;
TestSchemaUtils::prepare_data_schema(schema);
ObTabletCreateSSTableParam param;
TestTabletHelper::prepare_sstable_param(tablet_id, schema, param);
//update sstable param table key
param.table_key_.table_type_ = ObITable::MINOR_SSTABLE;
param.filled_tx_scn_ = param.table_key_.get_end_scn();
ASSERT_EQ(OB_SUCCESS, sstable.init(param, &allocator_));
//modified sstable filled tx scn as min
sstable.meta_->basic_meta_.filled_tx_scn_.set_min();
sstable.meta_cache_.filled_tx_scn_.set_min();
const int64_t size = sstable.get_serialize_size();
char *full_buf = static_cast<char *>(allocator_.alloc(size));
int64_t pos = 0;
int ret = sstable.serialize(full_buf, size, pos);
ASSERT_EQ(common::OB_SUCCESS, ret);
pos = 0;
ret = sstable.deserialize(allocator_, full_buf, size, pos);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(sstable.meta_->basic_meta_.filled_tx_scn_, sstable.get_key().get_end_scn());
}
} // end storage
} // end oceanbase

5
src/storage/blocksstable/ob_sstable.cpp
View File

@ -1296,6 +1296,8 @@ int ObSSTable::deserialize(common::ObArenaAllocator &allocator,
} else if (OB_UNLIKELY(!meta_->is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sstable meta is not valid", K(ret), K_(meta));
} else if (OB_FAIL(ObSSTableMetaCompactUtil::fix_filled_tx_scn_value_for_compact(key_, meta_->get_basic_meta().filled_tx_scn_))) {
LOG_WARN("failed to fix filled tx scn value for compact", K(ret), K_(meta));
} else if (OB_FAIL(meta_->transform_root_block_extra_buf(allocator))) {
LOG_WARN("fail to transform root block data", K(ret));
} else if (OB_FAIL(check_valid_for_reading())) {
@ -1412,6 +1414,9 @@ int ObSSTable::deserialize_fixed_struct(const char *buf, const int64_t data_len,
if (OB_SUCC(ret)) {
valid_for_reading_ = key_.is_valid();
if (OB_FAIL(ObSSTableMetaCompactUtil::fix_filled_tx_scn_value_for_compact(key_, meta_cache_.filled_tx_scn_))) {
LOG_WARN("failed to fix filled tx scn value for compact", K(ret), K_(meta));
}
}
}
return ret;

23
src/storage/blocksstable/ob_sstable_meta.cpp
View File

@ -1215,6 +1215,12 @@ int ObMigrationSSTableParam::deserialize_(const char *buf, const int64_t data_le
}
LST_DO_CODE(OB_UNIS_DECODE, is_meta_root_);
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObSSTableMetaCompactUtil::fix_filled_tx_scn_value_for_compact(table_key_, basic_meta_.filled_tx_scn_))) {
LOG_WARN("failed to fix filled tx scn value for compact", K(ret), K(table_key_), K(basic_meta_));
}
return ret;
}
@ -1439,5 +1445,22 @@ int ObSSTableMetaChecker::check_sstable_column_checksum_(
return ret;
}
int ObSSTableMetaCompactUtil::fix_filled_tx_scn_value_for_compact(
const ObITable::TableKey &table_key,
share::SCN &filled_tx_scn)
{
int ret = OB_SUCCESS;
if (table_key.tablet_id_.is_ls_inner_tablet()) {
//do nothing
} else if (table_key.is_major_sstable()) {
//do nothing
} else if (filled_tx_scn.is_min()) {
filled_tx_scn = table_key.get_end_scn();
LOG_WARN("fix filled tx scn value for compact", K(table_key), K(filled_tx_scn));
}
return ret;
}
} // namespace blocksstable
} // namespace oceanbase

8
src/storage/blocksstable/ob_sstable_meta.h
View File

@ -31,6 +31,7 @@ struct ObTabletCreateSSTableParam;
}
namespace blocksstable
{
class ObTxContext final
{
public:
@ -425,6 +426,13 @@ private:
const int64_t new_column_count);
};
class ObSSTableMetaCompactUtil
{
public:
static int fix_filled_tx_scn_value_for_compact(
const ObITable::TableKey &table_key,
share::SCN &filled_tx_scn);
};
} // namespace blocksstable
} // namespace oceanbase

1
src/storage/ddl/ob_tablet_ddl_kv.cpp
View File

@ -195,6 +195,7 @@ int ObDDLMemtable::init_sstable_param(
sstable_param.use_old_macro_block_count_ = 0; // all new, no reuse
sstable_param.index_blocks_cnt_ = 0; // index macro block count, the index is in memory, so be 0.
sstable_param.other_block_ids_.reset(); // other blocks contains only index macro blocks now, so empty.
sstable_param.filled_tx_scn_ = table_key.get_end_scn();
}
if (OB_SUCC(ret)) {

48
src/storage/tablet/ob_tablet_create_sstable_param.cpp
View File

@ -146,6 +146,9 @@ bool ObTabletCreateSSTableParam::is_valid() const
} else if (table_shared_flag_.is_shared_sstable() && (data_blocks_cnt_ + index_blocks_cnt_) > 0 && 0 == root_macro_seq_) {
ret = false;
LOG_ERROR("invalid root macro seq", K(data_blocks_cnt_), K(data_blocks_cnt_), K(root_macro_seq_));
} else if (!table_key_.get_tablet_id().is_ls_inner_tablet() && table_key_.is_minor_sstable() && filled_tx_scn_ < table_key_.get_end_scn()) {
ret = false;
LOG_WARN("filled tx scn is invalid", K(filled_tx_scn_), K(table_key_));
}
return ret;
}
@ -228,7 +231,13 @@ int ObTabletCreateSSTableParam::init_for_small_sstable(const blocksstable::ObSST
LOG_WARN("fail to fill column checksum", K(ret), K(res.data_column_checksums_));
}
}
if (OB_SUCC(ret)) {
if (!is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("init for small sstable get invalid argument", K(ret), K(res), K(table_key), KPC(this),
K(sstable_meta), K(block_info));
}
}
return ret;
}
@ -311,6 +320,14 @@ int ObTabletCreateSSTableParam::init_for_merge(const compaction::ObBasicTabletMe
table_shared_flag_.set_shared_sstable();
}
}
if (OB_SUCC(ret)) {
if (!is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("init for merge sstable get invalid argument", K(ret), K(table_key), KPC(this),
K(res), K(ctx));
}
}
}
return ret;
}
@ -462,6 +479,14 @@ int ObTabletCreateSSTableParam::init_for_ddl(blocksstable::ObSSTableIndexBuilder
LOG_WARN("unexpected column checksums", K(ret), K(column_count), KPC(this));
}
}
if (OB_SUCC(ret)) {
if (!is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("init for ddl sstable get invalid argument", K(ret), K(ddl_param), KPC(this),
K(res));
}
}
}
}
return ret;
@ -586,6 +611,7 @@ int ObTabletCreateSSTableParam::init_for_ha(
rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_;
ddl_scn_ = sstable_param.basic_meta_.ddl_scn_;
table_shared_flag_ = sstable_param.basic_meta_.table_shared_flag_;
filled_tx_scn_ = sstable_param.basic_meta_.filled_tx_scn_;
if (table_key_.is_co_sstable()) {
column_group_cnt_ = sstable_param.column_group_cnt_;
full_column_cnt_ = sstable_param.full_column_cnt_;
@ -601,8 +627,12 @@ int ObTabletCreateSSTableParam::init_for_ha(
#ifdef OB_BUILD_SHARED_STORAGE
root_macro_seq_ += oceanbase::compaction::MACRO_STEP_SIZE;
#endif
if (!is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("init for ha sstable get invalid argument", K(ret), K(sstable_param), KPC(this),
K(res));
}
}
return ret;
}
@ -651,8 +681,10 @@ int ObTabletCreateSSTableParam::init_for_ha(const blocksstable::ObMigrationSSTab
}
if (OB_FAIL(column_checksums_.assign(sstable_param.column_checksums_))) {
LOG_WARN("fail to assign column checksums", K(ret), K(sstable_param));
} else if (!is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("init for ha sstable get invalid argument", K(ret), K(sstable_param), KPC(this));
}
return ret;
}
@ -698,6 +730,7 @@ int ObTabletCreateSSTableParam::init_for_remote(const blocksstable::ObMigrationS
ddl_scn_ = sstable_param.basic_meta_.ddl_scn_;
table_backup_flag_ = sstable_param.basic_meta_.table_backup_flag_;
table_shared_flag_ = sstable_param.basic_meta_.table_shared_flag_;
filled_tx_scn_ = sstable_param.basic_meta_.filled_tx_scn_;
if (table_key_.is_co_sstable()) {
column_group_cnt_ = sstable_param.column_group_cnt_;
full_column_cnt_ = sstable_param.full_column_cnt_;
@ -707,6 +740,9 @@ int ObTabletCreateSSTableParam::init_for_remote(const blocksstable::ObMigrationS
MEMCPY(encrypt_key_, sstable_param.basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH);
if (OB_FAIL(column_checksums_.assign(sstable_param.column_checksums_))) {
LOG_WARN("fail to fill column checksum", K(ret), K(sstable_param));
} else if (!is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("init for remote sstable get invalid argument", K(ret), K(sstable_param), KPC(this));
}
return ret;
}
@ -764,6 +800,12 @@ int ObTabletCreateSSTableParam::init_for_mds(
}
}
if (OB_SUCC(ret)) {
if (!is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("init for mds sstable get invalid argument", K(ret), K(res), K(ctx), KPC(this));
}
}
return ret;
}

1
unittest/storage/blocksstable/ob_multi_version_sstable_test.h
View File

@ -542,6 +542,7 @@ void ObMultiVersionSSTableTest::prepare_data_end(
param.ddl_scn_.set_min();
param.table_backup_flag_.reset();
param.table_shared_flag_.reset();
param.filled_tx_scn_ = table_key_.get_end_scn();
if (table_type == ObITable::MAJOR_SSTABLE) {
ASSERT_EQ(OB_SUCCESS, ObSSTableMergeRes::fill_column_checksum_for_empty_major(param.column_cnt_, param.column_checksums_));
}

1
unittest/storage/test_compaction_policy.cpp
View File

@ -288,6 +288,7 @@ int TestCompactionPolicy::mock_sstable(
} else {
param.table_key_ = table_key;
param.max_merged_trans_version_ = max_merged_trans_version;
param.filled_tx_scn_ = table_key.get_end_scn();
}
void *buf = nullptr;