From bf07415ff0c8e1f56a3340775c8e54a2749ba9ff Mon Sep 17 00:00:00 2001
From: qiye <jianliang5669@gmail.com>
Date: Sat, 23 Sep 2023 10:58:55 +0800
Subject: [PATCH] [fix](es catalog) remove _default_ field when parsing
 mapping. (#24781)

---
 .../doris/external/elasticsearch/EsUtil.java  |   4 +
 .../external/elasticsearch/EsUtilTest.java    |   7 +
 .../resources/data/es/default_mappings.json   | 149 ++++++++++++++++++
 3 files changed, 160 insertions(+)
 create mode 100644 fe/fe-core/src/test/resources/data/es/default_mappings.json

diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 31f8840432..2def928cce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -154,6 +154,10 @@ public class EsUtil {
         mappings.remove("dynamic_templates");
         // remove `dynamic` field
         mappings.remove("dynamic");
+        // remove `_default` field, we do not parse `_default_` mapping, only explicit mapping.
+        // `_default` _mapping type is deprecated in 7.0 and removed in 8.0
+        // https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html
+        mappings.remove("_default_");
         // check explicit mapping
         if (mappings.isEmpty()) {
             throw new DorisEsException("Do not support index without explicit mapping.");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
index 4fce17cc1a..1cdf1cb94d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
@@ -278,5 +278,12 @@ public class EsUtilTest extends EsTestCase {
 
     }
 
+    @Test
+    public void testDefaultMapping() throws IOException, URISyntaxException {
+        ObjectNode testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/default_mappings.json"),
+                null);
+        Assertions.assertEquals("{\"@timestamp\":{\"type\":\"date\"},\"@version\":{\"type\":\"keyword\"},\"act\":{\"type\":\"text\",\"norms\":false,\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"app\":{\"type\":\"text\",\"norms\":false,\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"tags\":{\"type\":\"text\",\"norms\":false,\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"type\":{\"type\":\"text\",\"norms\":false,\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"uid\":{\"type\":\"text\",\"norms\":false,\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}",
+                testAliases.toString());
+    }
 
 }
diff --git a/fe/fe-core/src/test/resources/data/es/default_mappings.json b/fe/fe-core/src/test/resources/data/es/default_mappings.json
new file mode 100644
index 0000000000..fd28dcffe5
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/default_mappings.json
@@ -0,0 +1,149 @@
+{
+  "test" : {
+    "mappings" : {
+      "_default_" : {
+        "dynamic_templates" : [
+          {
+            "message_field" : {
+              "path_match" : "message",
+              "match_mapping_type" : "string",
+              "mapping" : {
+                "norms" : false,
+                "type" : "text"
+              }
+            }
+          },
+          {
+            "string_fields" : {
+              "match" : "*",
+              "match_mapping_type" : "string",
+              "mapping" : {
+                "fields" : {
+                  "keyword" : {
+                    "ignore_above" : 256,
+                    "type" : "keyword"
+                  }
+                },
+                "norms" : false,
+                "type" : "text"
+              }
+            }
+          }
+        ],
+        "properties" : {
+          "@timestamp" : {
+            "type" : "date"
+          },
+          "@version" : {
+            "type" : "keyword"
+          },
+          "geoip" : {
+            "dynamic" : "true",
+            "properties" : {
+              "ip" : {
+                "type" : "ip"
+              },
+              "latitude" : {
+                "type" : "half_float"
+              },
+              "location" : {
+                "type" : "geo_point"
+              },
+              "longitude" : {
+                "type" : "half_float"
+              }
+            }
+          }
+        }
+      },
+      "doc" : {
+        "dynamic_templates" : [
+          {
+            "message_field" : {
+              "path_match" : "message",
+              "match_mapping_type" : "string",
+              "mapping" : {
+                "norms" : false,
+                "type" : "text"
+              }
+            }
+          },
+          {
+            "string_fields" : {
+              "match" : "*",
+              "match_mapping_type" : "string",
+              "mapping" : {
+                "fields" : {
+                  "keyword" : {
+                    "ignore_above" : 256,
+                    "type" : "keyword"
+                  }
+                },
+                "norms" : false,
+                "type" : "text"
+              }
+            }
+          }
+        ],
+        "properties" : {
+          "@timestamp" : {
+            "type" : "date"
+          },
+          "@version" : {
+            "type" : "keyword"
+          },
+          "act" : {
+            "type" : "text",
+            "norms" : false,
+            "fields" : {
+              "keyword" : {
+                "type" : "keyword",
+                "ignore_above" : 256
+              }
+            }
+          },
+          "app" : {
+            "type" : "text",
+            "norms" : false,
+            "fields" : {
+              "keyword" : {
+                "type" : "keyword",
+                "ignore_above" : 256
+              }
+            }
+          },
+          "tags" : {
+            "type" : "text",
+            "norms" : false,
+            "fields" : {
+              "keyword" : {
+                "type" : "keyword",
+                "ignore_above" : 256
+              }
+            }
+          },
+          "type" : {
+            "type" : "text",
+            "norms" : false,
+            "fields" : {
+              "keyword" : {
+                "type" : "keyword",
+                "ignore_above" : 256
+              }
+            }
+          },
+          "uid" : {
+            "type" : "text",
+            "norms" : false,
+            "fields" : {
+              "keyword" : {
+                "type" : "keyword",
+                "ignore_above" : 256
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}