From ab467f53dbf0cac53ab3637991f6934ac3c476ec Mon Sep 17 00:00:00 2001 From: deardeng <565620795@qq.com> Date: Fri, 22 Mar 2024 15:38:58 +0800 Subject: [PATCH] [fix](partition) Fix be tablet partition id eq 0 By report tablet (#32179) (#32667) --- be/src/agent/task_worker_pool.cpp | 17 ++++++++++++++++ be/src/olap/tablet_manager.cpp | 20 +++++++++++++++++++ be/src/olap/tablet_manager.h | 3 +++ be/src/olap/tablet_meta.cpp | 4 ++-- .../java/org/apache/doris/common/Config.java | 4 ++++ .../doris/catalog/TabletInvertedIndex.java | 9 +++++++++ 6 files changed, 55 insertions(+), 2 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 34918a5b0a..4921f4585f 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -677,6 +677,23 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest& continue; } bool need_to_save = false; + if (tablet_meta_info.__isset.partition_id) { + // for fix partition_id = 0 + LOG(WARNING) << "change be tablet id: " << tablet->tablet_meta()->tablet_id() + << "partition id from : " << tablet->tablet_meta()->partition_id() + << " to : " << tablet_meta_info.partition_id; + auto succ = engine.tablet_manager()->update_tablet_partition_id( + tablet_meta_info.partition_id, tablet->tablet_meta()->tablet_id()); + if (!succ) { + std::string err_msg = fmt::format( + "change be tablet id : {} partition_id : {} failed", + tablet->tablet_meta()->tablet_id(), tablet_meta_info.partition_id); + LOG(WARNING) << err_msg; + status = Status::InvalidArgument(err_msg); + continue; + } + need_to_save = true; + } if (tablet_meta_info.__isset.storage_policy_id) { tablet->tablet_meta()->set_storage_policy_id(tablet_meta_info.storage_policy_id); need_to_save = true; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 5293a13bf8..1f5411b9de 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1536,4 +1536,24 @@ std::set TabletManager::check_all_tablet_segment(bool repair) { return bad_tablets; } +bool TabletManager::update_tablet_partition_id(::doris::TPartitionId partition_id, + ::doris::TTabletId tablet_id) { + std::shared_lock rdlock(_get_tablets_shard_lock(tablet_id)); + TabletSharedPtr tablet = _get_tablet_unlocked(tablet_id); + if (tablet == nullptr) { + LOG(WARNING) << "get tablet err partition_id: " << partition_id + << " tablet_id:" << tablet_id; + return false; + } + _remove_tablet_from_partition(tablet); + auto st = tablet->tablet_meta()->set_partition_id(partition_id); + if (!st.ok()) { + LOG(WARNING) << "set partition id err partition_id: " << partition_id + << " tablet_id:" << tablet_id; + return false; + } + _add_tablet_to_partition(tablet); + return true; +} + } // end namespace doris diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index 178ba50d9e..9f8164b853 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -163,6 +163,9 @@ public: std::set check_all_tablet_segment(bool repair); + bool update_tablet_partition_id(::doris::TPartitionId partition_id, + ::doris::TTabletId tablet_id); + private: // Add a tablet pointer to StorageEngine // If force, drop the existing tablet add this new one diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index cc9b9dc0b9..d8235191b9 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -875,8 +875,8 @@ RowsetMetaSharedPtr TabletMeta::acquire_stale_rs_meta_by_version(const Version& Status TabletMeta::set_partition_id(int64_t partition_id) { if ((_partition_id > 0 && _partition_id != partition_id) || partition_id < 1) { - LOG(FATAL) << "cur partition id=" << _partition_id << " new partition id=" << partition_id - << " not equal"; + LOG(WARNING) << "cur partition id=" << _partition_id << " new partition id=" << partition_id + << " not equal"; } _partition_id = partition_id; return Status::OK(); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 7f4f579cdd..5fd9a969bc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2505,6 +2505,9 @@ public class Config extends ConfigBase { options = {"default", "ranger-doris"}) public static String access_controller_type = "default"; + @ConfField(mutable = true) + public static boolean fix_tablet_partition_id_eq_0 = false; + @ConfField(mutable = true, masterOnly = true, description = { "倒排索引默认存储格式", "Default storage format of inverted index, the default value is V1." @@ -2517,6 +2520,7 @@ public class Config extends ConfigBase { }) public static boolean enable_proxy_protocol = false; + //========================================================================== // begin of cloud config //========================================================================== diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java index b5b8cc6166..c4650cb5e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java @@ -167,6 +167,15 @@ public class TabletInvertedIndex { tabletMetaInfo.setIsInMemory(!backendTabletInfo.isIsInMemory()); } } + if (Config.fix_tablet_partition_id_eq_0 + && tabletMeta.getPartitionId() > 0 + && backendTabletInfo.getPartitionId() == 0) { + LOG.warn("be report tablet partition id not eq fe, in be {} but in fe {}", + backendTabletInfo, tabletMeta); + // Need to update partition id in BE + tabletMetaInfo = new TTabletMetaInfo(); + tabletMetaInfo.setPartitionId(tabletMeta.getPartitionId()); + } // 1. (intersection) if (needSync(replica, backendTabletInfo)) { // need sync