From 87439e227e02ad1903b9c926e5eb5d2ff3e42eed Mon Sep 17 00:00:00 2001
From: Stalary <stalary@163.com>
Date: Tue, 13 Sep 2022 09:59:48 +0800
Subject: [PATCH] [Enhancement](DOE): Doe support object/nested use string
 (#12401)

* MOD: doe support object/nested use string
---
 be/src/http/http_client.cpp                   |  4 +-
 .../doris/external/elasticsearch/EsUtil.java  | 37 ++++++++++---------
 .../external/elasticsearch/MappingPhase.java  |  3 ++
 .../external/elasticsearch/EsUtilTest.java    |  8 ++++
 .../resources/data/es/test_index_mapping.json | 30 +++++++++++++++
 .../data/es/test_index_mapping_after_7x.json  | 30 +++++++++++++++
 ...est_index_mapping_field_mult_analyzer.json | 30 +++++++++++++++
 7 files changed, 123 insertions(+), 19 deletions(-)

diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp
index 96d9c447e4..0cb5b97ec4 100644
--- a/be/src/http/http_client.cpp
+++ b/be/src/http/http_client.cpp
@@ -18,6 +18,7 @@
 #include "http/http_client.h"
 
 #include "common/config.h"
+#include "util/stack_util.h"
 
 namespace doris {
 
@@ -162,7 +163,8 @@ Status HttpClient::execute(const std::function<bool(const void* data, size_t len
     _callback = &callback;
     auto code = curl_easy_perform(_curl);
     if (code != CURLE_OK) {
-        LOG(WARNING) << "fail to execute HTTP client, errmsg=" << _to_errmsg(code);
+        LOG(WARNING) << "fail to execute HTTP client, errmsg=" << _to_errmsg(code)
+                     << ", trace=" << get_stack_trace();
         return Status::InternalError(_to_errmsg(code));
     }
     return Status::OK();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 33dd1ce97d..c9b89e8f5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -37,6 +37,7 @@ import org.apache.doris.analysis.RangePartitionDesc;
 import org.apache.doris.analysis.SlotRef;
 import org.apache.doris.catalog.ArrayType;
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.DdlException;
@@ -358,22 +359,23 @@ public class EsUtil {
         List<Column> columns = new ArrayList<>();
         for (String key : keys) {
             JSONObject field = (JSONObject) mappingProps.get(key);
-            // Complex types are not currently supported.
+            Type type;
+            // Complex types are treating as String types for now.
             if (field.containsKey("type")) {
-                Type type = toDorisType(field.get("type").toString());
-                if (!type.isInvalid()) {
-                    Column column = new Column();
-                    column.setName(key);
-                    column.setIsKey(true);
-                    column.setIsAllowNull(true);
-                    if (arrayFields.contains(key)) {
-                        column.setType(ArrayType.create(type, true));
-                    } else {
-                        column.setType(type);
-                    }
-                    columns.add(column);
-                }
+                type = toDorisType(field.get("type").toString());
+            } else {
+                type = Type.STRING;
             }
+            Column column = new Column();
+            column.setName(key);
+            column.setIsKey(true);
+            column.setIsAllowNull(true);
+            if (arrayFields.contains(key)) {
+                column.setType(ArrayType.create(type, true));
+            } else {
+                column.setType(type);
+            }
+            columns.add(column);
         }
         return columns;
     }
@@ -403,16 +405,15 @@ public class EsUtil {
             case "double":
             case "scaled_float":
                 return Type.DOUBLE;
+            case "date":
+                return ScalarType.getDefaultDateType(Type.DATE);
             case "keyword":
             case "text":
             case "ip":
             case "nested":
             case "object":
-                return Type.STRING;
-            case "date":
-                return Type.DATE;
             default:
-                return Type.INVALID;
+                return Type.STRING;
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
index d0082c45e8..493fa413a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
@@ -119,6 +119,9 @@ public class MappingPhase implements SearchPhase {
                 if (!docValue) {
                     return;
                 }
+            } else if (fieldType == null || "nested".equals(fieldType)) {
+                // The object field has no type, and nested not support doc value.
+                return;
             }
             docValueField = colName;
         }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
index 9a00b58b4d..2e53e5f1ae 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
@@ -74,9 +74,15 @@ public class EsUtilTest extends EsTestCase {
         Column k1 = new Column("k1", PrimitiveType.BIGINT);
         Column k2 = new Column("k2", PrimitiveType.VARCHAR);
         Column k3 = new Column("k3", PrimitiveType.VARCHAR);
+        Column k4 = new Column("k4", PrimitiveType.VARCHAR);
+        Column k5 = new Column("k5", PrimitiveType.VARCHAR);
+        Column k6 = new Column("k6", PrimitiveType.DATE);
         columns.add(k1);
         columns.add(k2);
         columns.add(k3);
+        columns.add(k4);
+        columns.add(k5);
+        columns.add(k6);
     }
 
     @Test
@@ -120,6 +126,8 @@ public class EsUtilTest extends EsTestCase {
         Assertions.assertEquals("k3.keyword", searchContext1.docValueFieldsContext().get("k3"));
         Assertions.assertEquals("k1", searchContext1.docValueFieldsContext().get("k1"));
         Assertions.assertEquals("k2", searchContext1.docValueFieldsContext().get("k2"));
+        Assertions.assertNull(searchContext1.docValueFieldsContext().get("k4"));
+        Assertions.assertNull(searchContext1.docValueFieldsContext().get("k5"));
 
     }
 
diff --git a/fe/fe-core/src/test/resources/data/es/test_index_mapping.json b/fe/fe-core/src/test/resources/data/es/test_index_mapping.json
index 297a7fbb9d..d5e12bc155 100644
--- a/fe/fe-core/src/test/resources/data/es/test_index_mapping.json
+++ b/fe/fe-core/src/test/resources/data/es/test_index_mapping.json
@@ -16,6 +16,36 @@
                 "type": "keyword"
               }
             }
+          },
+          "k4": {
+            "properties": {
+              "child1": {
+                "type": "text",
+                "fields": {
+                  "keyword": {
+                    "type": "keyword",
+                    "ignore_above": 256
+                  }
+                }
+              },
+              "child2": {
+                "type": "long"
+              }
+            }
+          },
+          "k5": {
+            "type": "nested",
+            "properties": {
+              "child3": {
+                "type": "keyword"
+              },
+              "child4": {
+                "type": "keyword"
+              }
+            }
+          },
+          "k6": {
+            "type": "date"
           }
         }
       }
diff --git a/fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json b/fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json
index d018360c48..c4a8e8528b 100644
--- a/fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json
+++ b/fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json
@@ -15,6 +15,36 @@
               "type": "keyword"
             }
           }
+        },
+        "k4": {
+          "properties": {
+            "child1": {
+              "type": "text",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "child2": {
+              "type": "long"
+            }
+          }
+        },
+        "k5": {
+          "type": "nested",
+          "properties": {
+            "child3": {
+              "type": "keyword"
+            },
+            "child4": {
+              "type": "keyword"
+            }
+          }
+        },
+        "k6": {
+          "type": "date"
         }
       }
     }
diff --git a/fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json b/fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json
index 1a002bd2db..5f3d785b39 100644
--- a/fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json
+++ b/fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json
@@ -16,6 +16,36 @@
               "analyzer": "ik_max_word"
             }
           }
+        },
+        "k4": {
+          "properties": {
+            "child1": {
+              "type": "text",
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            },
+            "child2": {
+              "type": "long"
+            }
+          }
+        },
+        "k5": {
+          "type": "nested",
+          "properties": {
+            "child3": {
+              "type": "keyword"
+            },
+            "child4": {
+              "type": "keyword"
+            }
+          }
+        },
+        "k6": {
+          "type": "date"
         }
       }
     }