From 6502da89170751cedbe54b80ac2c3c613ce9b1cc Mon Sep 17 00:00:00 2001 From: DeadlineFen Date: Mon, 25 Sep 2023 20:07:24 +0800 Subject: [PATCH] [bugfix](restore) add partition id into convert_rowset_ids() (#24834) --- be/src/olap/single_replica_compaction.cpp | 3 ++- be/src/olap/snapshot_manager.cpp | 6 +++++- be/src/olap/snapshot_manager.h | 2 +- be/src/olap/task/engine_clone_task.cpp | 2 +- be/src/olap/task/engine_storage_migration_task.cpp | 4 ++-- be/src/runtime/snapshot_loader.cpp | 2 +- .../src/main/java/org/apache/doris/task/CloneTask.java | 1 + gensrc/thrift/AgentService.thrift | 1 + 8 files changed, 14 insertions(+), 7 deletions(-) diff --git a/be/src/olap/single_replica_compaction.cpp b/be/src/olap/single_replica_compaction.cpp index a480717633..5bb605cb00 100644 --- a/be/src/olap/single_replica_compaction.cpp +++ b/be/src/olap/single_replica_compaction.cpp @@ -348,7 +348,8 @@ Status SingleReplicaCompaction::_fetch_rowset(const TReplicaInfo& addr, const st if (status.ok()) { // change all rowset ids because they maybe its id same with local rowset auto olap_st = SnapshotManager::instance()->convert_rowset_ids( - local_path, _tablet->tablet_id(), _tablet->replica_id(), _tablet->schema_hash()); + local_path, _tablet->tablet_id(), _tablet->replica_id(), _tablet->partition_id(), + _tablet->schema_hash()); if (!olap_st.ok()) { LOG(WARNING) << "fail to convert rowset ids, path=" << local_path << ", tablet_id=" << _tablet->tablet_id() << ", error=" << olap_st; diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index e5b1aa8bca..7495d09e43 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -125,7 +125,8 @@ Status SnapshotManager::release_snapshot(const string& snapshot_path) { } Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t tablet_id, - int64_t replica_id, const int32_t& schema_hash) { + int64_t replica_id, int64_t partition_id, + const int32_t& schema_hash) { SCOPED_CONSUME_MEM_TRACKER(_mem_tracker); Status res = Status::OK(); // check clone dir existed @@ -160,6 +161,9 @@ Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t new_tablet_meta_pb.set_tablet_id(tablet_id); *new_tablet_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto(); new_tablet_meta_pb.set_replica_id(replica_id); + if (partition_id != -1) { + new_tablet_meta_pb.set_partition_id(partition_id); + } new_tablet_meta_pb.set_schema_hash(schema_hash); TabletSchemaSPtr tablet_schema; tablet_schema = diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index 3536cebd1a..3548fd65e2 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -55,7 +55,7 @@ public: static SnapshotManager* instance(); Status convert_rowset_ids(const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, - const int32_t& schema_hash); + int64_t partition_id, const int32_t& schema_hash); private: SnapshotManager() : _snapshot_base_id(0) { diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 75e17e433e..4eb43864be 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -323,7 +323,7 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, // change all rowset ids because they maybe its id same with local rowset status = SnapshotManager::instance()->convert_rowset_ids( local_data_path, _clone_req.tablet_id, _clone_req.replica_id, - _clone_req.schema_hash); + _clone_req.partition_id, _clone_req.schema_hash); } else { LOG_WARNING("failed to download snapshot from remote BE") .tag("url", remote_url_prefix) diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index 5f12be804f..f76c3ee208 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -151,8 +151,8 @@ Status EngineStorageMigrationTask::_gen_and_write_header_to_hdr_file( // it will change rowset id and its create time // rowset create time is useful when load tablet from meta to check which tablet is the tablet to load - return SnapshotManager::instance()->convert_rowset_ids(full_path, tablet_id, - _tablet->replica_id(), schema_hash); + return SnapshotManager::instance()->convert_rowset_ids( + full_path, tablet_id, _tablet->replica_id(), _tablet->partition_id(), schema_hash); } Status EngineStorageMigrationTask::_reload_tablet(const std::string& full_path) { diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index aff4dd7f2a..b5752710ae 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -697,7 +697,7 @@ Status SnapshotLoader::move(const std::string& snapshot_path, TabletSharedPtr ta // rename the rowset ids and tabletid info in rowset meta Status convert_status = SnapshotManager::instance()->convert_rowset_ids( - snapshot_path, tablet_id, tablet->replica_id(), schema_hash); + snapshot_path, tablet_id, tablet->replica_id(), tablet->partition_id(), schema_hash); if (!convert_status.ok()) { std::stringstream ss; ss << "failed to convert rowsetids in snapshot: " << snapshot_path diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java index 60531ced30..4c06460cee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java @@ -85,6 +85,7 @@ public class CloneTask extends AgentTask { request.setStorageMedium(storageMedium); request.setCommittedVersion(visibleVersion); request.setTaskVersion(taskVersion); + request.setPartitionId(partitionId); if (taskVersion == VERSION_2) { request.setSrcPathHash(srcPathHash); request.setDestPathHash(destPathHash); diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 9f0755c735..a61c951256 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -262,6 +262,7 @@ struct TCloneReq { 9: optional i64 dest_path_hash; 10: optional i32 timeout_s; 11: optional Types.TReplicaId replica_id = 0 + 12: optional i64 partition_id } struct TCompactionReq {