database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhancement](DOE): Doe support object/nested use string (#12401)

Browse Source 
* MOD: doe support object/nested use string
This commit is contained in:
Stalary
2022-09-13 09:59:48 +08:00
committed by GitHub
parent 97cb095010
commit 87439e227e
7 changed files with 123 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java vendored
View File

@ -37,6 +37,7 @@ import org.apache.doris.analysis.RangePartitionDesc;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
@ -358,22 +359,23 @@ public class EsUtil {
List<Column> columns = new ArrayList<>();
for (String key : keys) {
JSONObject field = (JSONObject) mappingProps.get(key);
// Complex types are not currently supported.
Type type;
// Complex types are treating as String types for now.
if (field.containsKey("type")) {
Type type = toDorisType(field.get("type").toString());
if (!type.isInvalid()) {
Column column = new Column();
column.setName(key);
column.setIsKey(true);
column.setIsAllowNull(true);
if (arrayFields.contains(key)) {
column.setType(ArrayType.create(type, true));
} else {
column.setType(type);
}
columns.add(column);
}
type = toDorisType(field.get("type").toString());
} else {
type = Type.STRING;
}
Column column = new Column();
column.setName(key);
column.setIsKey(true);
column.setIsAllowNull(true);
if (arrayFields.contains(key)) {
column.setType(ArrayType.create(type, true));
} else {
column.setType(type);
}
columns.add(column);
}
return columns;
}
@ -403,16 +405,15 @@ public class EsUtil {
case "double":
case "scaled_float":
return Type.DOUBLE;
case "date":
return ScalarType.getDefaultDateType(Type.DATE);
case "keyword":
case "text":
case "ip":
case "nested":
case "object":
return Type.STRING;
case "date":
return Type.DATE;
default:
return Type.INVALID;
return Type.STRING;
}
}

3
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java vendored
View File

@ -119,6 +119,9 @@ public class MappingPhase implements SearchPhase {
if (!docValue) {
return;
}
} else if (fieldType == null || "nested".equals(fieldType)) {
// The object field has no type, and nested not support doc value.
return;
}
docValueField = colName;
}

8
fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java vendored
View File

@ -74,9 +74,15 @@ public class EsUtilTest extends EsTestCase {
Column k1 = new Column("k1", PrimitiveType.BIGINT);
Column k2 = new Column("k2", PrimitiveType.VARCHAR);
Column k3 = new Column("k3", PrimitiveType.VARCHAR);
Column k4 = new Column("k4", PrimitiveType.VARCHAR);
Column k5 = new Column("k5", PrimitiveType.VARCHAR);
Column k6 = new Column("k6", PrimitiveType.DATE);
columns.add(k1);
columns.add(k2);
columns.add(k3);
columns.add(k4);
columns.add(k5);
columns.add(k6);
}
@Test
@ -120,6 +126,8 @@ public class EsUtilTest extends EsTestCase {
Assertions.assertEquals("k3.keyword", searchContext1.docValueFieldsContext().get("k3"));
Assertions.assertEquals("k1", searchContext1.docValueFieldsContext().get("k1"));
Assertions.assertEquals("k2", searchContext1.docValueFieldsContext().get("k2"));
Assertions.assertNull(searchContext1.docValueFieldsContext().get("k4"));
Assertions.assertNull(searchContext1.docValueFieldsContext().get("k5"));
}

30
fe/fe-core/src/test/resources/data/es/test_index_mapping.json
View File

@ -16,6 +16,36 @@
"type": "keyword"
}
}
},
"k4": {
"properties": {
"child1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"child2": {
"type": "long"
}
}
},
"k5": {
"type": "nested",
"properties": {
"child3": {
"type": "keyword"
},
"child4": {
"type": "keyword"
}
}
},
"k6": {
"type": "date"
}
}
}

30
fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json
View File

@ -15,6 +15,36 @@
"type": "keyword"
}
}
},
"k4": {
"properties": {
"child1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"child2": {
"type": "long"
}
}
},
"k5": {
"type": "nested",
"properties": {
"child3": {
"type": "keyword"
},
"child4": {
"type": "keyword"
}
}
},
"k6": {
"type": "date"
}
}
}

30
fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json
View File

@ -16,6 +16,36 @@
"analyzer": "ik_max_word"
}
}
},
"k4": {
"properties": {
"child1": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"child2": {
"type": "long"
}
}
},
"k5": {
"type": "nested",
"properties": {
"child3": {
"type": "keyword"
},
"child4": {
"type": "keyword"
}
}
},
"k6": {
"type": "date"
}
}
}