diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 9f49c1b7f2..919379b220 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -1048,34 +1048,24 @@ public class HiveMetaStoreCache { return false; } String pathStr = path.toUri().toString(); - if (containsHiddenPath(pathStr) || path.getName().startsWith("_")) { + if (containsHiddenPath(pathStr)) { return false; } - for (String name : pathStr.split("/")) { - if (isGeneratedPath(name)) { - return false; - } - } return true; } private static boolean containsHiddenPath(String path) { - if (path.startsWith(".")) { + // Hive ignores files starting with _ and . + if (path.startsWith(".") || path.startsWith("_")) { return true; } for (int i = 0; i < path.length() - 1; i++) { - if (path.charAt(i) == '/' && path.charAt(i + 1) == '.') { + if (path.charAt(i) == '/' && (path.charAt(i + 1) == '.' || path.charAt(i + 1) == '_')) { return true; } } return false; } - - private static boolean isGeneratedPath(String name) { - return "_temporary".equals(name) // generated by spark - || "_imapala_insert_staging".equals(name) // generated by impala - || name.startsWith("."); // generated by hive or hidden file - } } @Data diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java index 0937bbc3cc..6610a5f252 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java @@ -31,17 +31,20 @@ public class PathVisibleTest { Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/.hidden/path"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible/path/.file"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_temporary_xx"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_imapala_insert_staging"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_impala_insert_staging"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible//.hidden/path"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("s3://visible/.hidden/path"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("///visible/path/.file"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path///_temporary_xx"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible//path/_imapala_insert_staging"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible//path/_impala_insert_staging"))); + Assert.assertFalse(FileCacheValue.isFileVisible( + new Path("hdfs://hacluster/user/hive/warehouse/db1.db/tbl1/_spark_metadata/"))); Assert.assertTrue(FileCacheValue.isFileVisible(new Path("s3://visible/path"))); Assert.assertTrue(FileCacheValue.isFileVisible(new Path("path"))); Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://visible/path./1.txt"))); Assert.assertTrue(FileCacheValue.isFileVisible(new Path("/1.txt"))); + Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://vis_ible_/pa.th./1_.txt__"))); } }