From 5962dceddbbc83e7706a66453da889c0fcfd8380 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:19:06 +0800 Subject: [PATCH] branch-2.1: [fix](hms)fix hive catalog follower not get event. #46485 (#46490) Cherry-picked from #46485 Co-authored-by: daidai --- .../datasource/hive/HMSExternalCatalog.java | 28 +++++++++---------- .../hive/event/MetastoreEventsProcessor.java | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 2d6c66af44..2c80901da4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -117,20 +117,6 @@ public class HMSExternalCatalog extends ExternalCatalog { throw new DdlException( "The parameter " + FILE_META_CACHE_TTL_SECOND + " is wrong, value is " + fileMetaCacheTtlSecond); } - Map properties = catalogProperty.getProperties(); - if (properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) { - enableHmsEventsIncrementalSync = - properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true"); - } else { - enableHmsEventsIncrementalSync = Config.enable_hms_events_incremental_sync; - } - - if (properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) { - hmsEventsBatchSizePerRpc = Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)); - } else { - hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc; - } - // check the dfs.ha properties // 'dfs.nameservices'='your-nameservice', // 'dfs.ha.namenodes.your-nameservice'='nn1,nn2', @@ -290,6 +276,20 @@ public class HMSExternalCatalog extends ExternalCatalog { // always allow fallback to simple auth, so to support both kerberos and simple auth catalogProperty.addProperty(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "true"); } + + Map properties = catalogProperty.getProperties(); + if (properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) { + enableHmsEventsIncrementalSync = + properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true"); + } else { + enableHmsEventsIncrementalSync = Config.enable_hms_events_incremental_sync; + } + + if (properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) { + hmsEventsBatchSizePerRpc = Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)); + } else { + hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc; + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java index 7305477340..23bf324360 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java @@ -269,7 +269,7 @@ public class MetastoreEventsProcessor extends MasterDaemon { // For slave FE nodes, only fetch events which id is lower than masterLastSyncedEventId int maxEventSize = Math.min((int) (masterLastSyncedEventId - lastSyncedEventId), - Config.hms_events_batch_size_per_rpc); + hmsExternalCatalog.getHmsEventsBatchSizePerRpc()); try { return hmsExternalCatalog.getClient().getNextNotification(lastSyncedEventId, maxEventSize, null); } catch (MetastoreNotificationFetchException e) {