From 8480691871783a9b4dee49a2addee840c7e5d4f8 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Sat, 28 Jun 2025 14:23:27 +0800
Subject: [PATCH] branch-2.1: [fix](hive) ignore all hidden dir and files
 #52286 (#52324)

Cherry-picked from #52286

Co-authored-by: Mingyu Chen (Rayner) <morningman@163.com>
---
 .../datasource/hive/HiveMetaStoreCache.java    | 18 ++++--------------
 .../doris/datasource/PathVisibleTest.java      |  7 +++++--
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 9f49c1b7f2..919379b220 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -1048,34 +1048,24 @@ public class HiveMetaStoreCache {
                 return false;
             }
             String pathStr = path.toUri().toString();
-            if (containsHiddenPath(pathStr) || path.getName().startsWith("_")) {
+            if (containsHiddenPath(pathStr)) {
                 return false;
             }
-            for (String name : pathStr.split("/")) {
-                if (isGeneratedPath(name)) {
-                    return false;
-                }
-            }
             return true;
         }
 
         private static boolean containsHiddenPath(String path) {
-            if (path.startsWith(".")) {
+            // Hive ignores files starting with _ and .
+            if (path.startsWith(".") || path.startsWith("_")) {
                 return true;
             }
             for (int i = 0; i < path.length() - 1; i++) {
-                if (path.charAt(i) == '/' && path.charAt(i + 1) == '.') {
+                if (path.charAt(i) == '/' && (path.charAt(i + 1) == '.' || path.charAt(i + 1) == '_')) {
                     return true;
                 }
             }
             return false;
         }
-
-        private static boolean isGeneratedPath(String name) {
-            return "_temporary".equals(name) // generated by spark
-                    || "_imapala_insert_staging".equals(name) // generated by impala
-                    || name.startsWith("."); // generated by hive or hidden file
-        }
     }
 
     @Data
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java
index 0937bbc3cc..6610a5f252 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java
@@ -31,17 +31,20 @@ public class PathVisibleTest {
         Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/.hidden/path")));
         Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible/path/.file")));
         Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_temporary_xx")));
-        Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_imapala_insert_staging")));
+        Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_impala_insert_staging")));
 
         Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible//.hidden/path")));
         Assert.assertFalse(FileCacheValue.isFileVisible(new Path("s3://visible/.hidden/path")));
         Assert.assertFalse(FileCacheValue.isFileVisible(new Path("///visible/path/.file")));
         Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path///_temporary_xx")));
-        Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible//path/_imapala_insert_staging")));
+        Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible//path/_impala_insert_staging")));
+        Assert.assertFalse(FileCacheValue.isFileVisible(
+                new Path("hdfs://hacluster/user/hive/warehouse/db1.db/tbl1/_spark_metadata/")));
 
         Assert.assertTrue(FileCacheValue.isFileVisible(new Path("s3://visible/path")));
         Assert.assertTrue(FileCacheValue.isFileVisible(new Path("path")));
         Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://visible/path./1.txt")));
         Assert.assertTrue(FileCacheValue.isFileVisible(new Path("/1.txt")));
+        Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://vis_ible_/pa.th./1_.txt__")));
     }
 }