database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1: [fix](hms)fix hive catalog follower not get event. #46485 (#46490)

Browse Source 
Cherry-picked from #46485

Co-authored-by: daidai <changyuwei@selectdb.com>
This commit is contained in:
github-actions[bot]
2025-01-07 12:19:06 +08:00
committed by GitHub
parent 8f4b76a5f3
commit 5962dceddb
2 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
View File

@ -117,20 +117,6 @@ public class HMSExternalCatalog extends ExternalCatalog {
throw new DdlException(
"The parameter " + FILE_META_CACHE_TTL_SECOND + " is wrong, value is " + fileMetaCacheTtlSecond);
}
Map<String, String> properties = catalogProperty.getProperties();
if (properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) {
enableHmsEventsIncrementalSync =
properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true");
} else {
enableHmsEventsIncrementalSync = Config.enable_hms_events_incremental_sync;
}
if (properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) {
hmsEventsBatchSizePerRpc = Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC));
} else {
hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc;
}
// check the dfs.ha properties
// 'dfs.nameservices'='your-nameservice',
// 'dfs.ha.namenodes.your-nameservice'='nn1,nn2',
@ -290,6 +276,20 @@ public class HMSExternalCatalog extends ExternalCatalog {
// always allow fallback to simple auth, so to support both kerberos and simple auth
catalogProperty.addProperty(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "true");
}
Map<String, String> properties = catalogProperty.getProperties();
if (properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) {
enableHmsEventsIncrementalSync =
properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true");
} else {
enableHmsEventsIncrementalSync = Config.enable_hms_events_incremental_sync;
}
if (properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) {
hmsEventsBatchSizePerRpc = Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC));
} else {
hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc;
}
}
@Override

2
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
View File

@ -269,7 +269,7 @@ public class MetastoreEventsProcessor extends MasterDaemon {
// For slave FE nodes, only fetch events which id is lower than masterLastSyncedEventId
int maxEventSize = Math.min((int) (masterLastSyncedEventId - lastSyncedEventId),
Config.hms_events_batch_size_per_rpc);
hmsExternalCatalog.getHmsEventsBatchSizePerRpc());
try {
return hmsExternalCatalog.getClient().getNextNotification(lastSyncedEventId, maxEventSize, null);
} catch (MetastoreNotificationFetchException e) {