database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](hudi) disable fs.impl.cache to avoid FE OOM (#36402) (#36403)

Browse Source 
bp #36402
This commit is contained in:
Mingyu Chen
2024-06-17 22:20:23 +08:00
committed by GitHub
parent 3810861bb1
commit 4ae8607b2e
1 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
View File

@ -66,6 +66,7 @@ import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.time.LocalDateTime;
import java.time.ZoneId;
@ -819,6 +820,15 @@ public class HiveMetaStoreClientHelper {
public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
String hudiBasePath = table.getRemoteTable().getSd().getLocation();
Configuration conf = getConfiguration(table);
if (LOG.isDebugEnabled()) {
LOG.debug("try setting 'fs.xxx.impl.disable.cache' to true for hudi's base path: {}", hudiBasePath);
}
URI hudiBasePathUri = URI.create(hudiBasePath);
String scheme = hudiBasePathUri.getScheme();
if (!Strings.isNullOrEmpty(scheme)) {
// Avoid using Cache in Hadoop FileSystem, which may cause FE OOM.
conf.set("fs." + scheme + ".impl.disable.cache", "true");
}
return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf),
() -> HoodieTableMetaClient.builder().setConf(conf).setBasePath(hudiBasePath).build());
}