diff --git a/docs/en/docs/lakehouse/multi-catalog/es.md b/docs/en/docs/lakehouse/multi-catalog/es.md index 580882f69b..6edbb95556 100644 --- a/docs/en/docs/lakehouse/multi-catalog/es.md +++ b/docs/en/docs/lakehouse/multi-catalog/es.md @@ -68,27 +68,28 @@ After switching to the ES Catalog, you will be in the `dafault_db` so you don't ## Column Type Mapping -| ES Type | Doris Type | Comment | -| ------------- | ----------- | ------- | -| null | null | | -| boolean | boolean | | -| byte | tinyint | | -| short | smallint | | -| integer | int | | -| long | bigint | | -| unsigned_long | largeint | | -| float | float | | -| half_float | float | | -| double | double | | -| scaled_float | double | | -| date | date | | -| keyword | string | | -| text | string | | -| ip | string | | -| nested | string | | -| object | string | | -| other | unsupported | | +| ES Type | Doris Type | Comment | +| ------------- | ----------- |-------------------------------------------------------------------------| +| null | null | | +| boolean | boolean | | +| byte | tinyint | | +| short | smallint | | +| integer | int | | +| long | bigint | | +| unsigned_long | largeint | | +| float | float | | +| half_float | float | | +| double | double | | +| scaled_float | double | | +| date | date | Only support default/yyyy-MM-dd HH:mm:ss/yyyy-MM-dd/epoch_millis format | +| keyword | string | | +| text | string | | +| ip | string | | +| nested | string | | +| object | string | | +| other | unsupported | | + ### Array Type Elasticsearch does not have an explicit array type, but one of its fields can contain @@ -153,6 +154,7 @@ curl -X PUT "localhost:9200/doc/_mapping?pretty" -H 'Content-Type: application/j `array_fields`：Used to indicate a field that is an array type. + ## Best Practice ### Predicate Pushdown diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/es.md b/docs/zh-CN/docs/lakehouse/multi-catalog/es.md index b745a36d93..ec74f1207d 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/es.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/es.md @@ -68,27 +68,28 @@ CREATE CATALOG es PROPERTIES ( ## 列类型映射 -| ES Type | Doris Type | Comment | -|---|---|---| +| ES Type | Doris Type | Comment | +|---|---|------------------------------------------------------------| |null| null|| -| boolean | boolean | | -| byte| tinyint| | -| short| smallint| | -| integer| int| | -| long| bigint| | -| unsigned_long| largeint | | -| float| float| | -| half_float| float| | -| double | double | | -| scaled_float| double | | -| date | date | | -| keyword | string | | -| text |string | | -| ip |string | | -| nested |string | | -| object |string | | +| boolean | boolean | | +| byte| tinyint| | +| short| smallint| | +| integer| int| | +| long| bigint| | +| unsigned_long| largeint | | +| float| float| | +| half_float| float| | +| double | double | | +| scaled_float| double | | +| date | date | 仅支持 default/yyyy-MM-dd HH:mm:ss/yyyy-MM-dd/epoch_millis 格式 | +| keyword | string | | +| text |string | | +| ip |string | | +| nested |string | | +| object |string | | |other| unsupported || + ### Array 类型 Elasticsearch 没有明确的数组类型，但是它的某个字段可以含有[0个或多个值](https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html)。 @@ -148,6 +149,7 @@ curl -X PUT "localhost:9200/doc/_mapping?pretty" -H 'Content-Type: application/j `array_fields`：用来表示是数组类型的字段。 + ## 最佳实践 ### 过滤条件下推 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java index 81000c69d1..5d77163857 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java @@ -145,6 +145,10 @@ public class EsTable extends Table { return esMetaStateTracker.searchContext().docValueFieldsContext(); } + public List needCompatDateFields() { + return esMetaStateTracker.searchContext().needCompatDateFields(); + } + private void validate(Map properties) throws DdlException { EsResource.valid(properties, false); hosts = properties.get(EsResource.HOSTS).trim(); @@ -270,10 +274,10 @@ public class EsTable extends Table { indexName = tableContext.get("indexName"); mappingType = tableContext.get("mappingType"); - enableDocValueScan = Boolean.parseBoolean(tableContext.getOrDefault("enableDocValueScan", - EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE)); - enableKeywordSniff = Boolean.parseBoolean(tableContext.getOrDefault("enableKeywordSniff", - EsResource.KEYWORD_SNIFF_DEFAULT_VALUE)); + enableDocValueScan = Boolean.parseBoolean( + tableContext.getOrDefault("enableDocValueScan", EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE)); + enableKeywordSniff = Boolean.parseBoolean( + tableContext.getOrDefault("enableKeywordSniff", EsResource.KEYWORD_SNIFF_DEFAULT_VALUE)); if (tableContext.containsKey("maxDocValueFields")) { try { maxDocValueFields = Integer.parseInt(tableContext.get("maxDocValueFields")); @@ -281,12 +285,12 @@ public class EsTable extends Table { maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS; } } - nodesDiscovery = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.NODES_DISCOVERY, - EsResource.NODES_DISCOVERY_DEFAULT_VALUE)); - httpSslEnabled = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.HTTP_SSL_ENABLED, - EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE)); - likePushDown = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.LIKE_PUSH_DOWN, - EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE)); + nodesDiscovery = Boolean.parseBoolean( + tableContext.getOrDefault(EsResource.NODES_DISCOVERY, EsResource.NODES_DISCOVERY_DEFAULT_VALUE)); + httpSslEnabled = Boolean.parseBoolean( + tableContext.getOrDefault(EsResource.HTTP_SSL_ENABLED, EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE)); + likePushDown = Boolean.parseBoolean( + tableContext.getOrDefault(EsResource.LIKE_PUSH_DOWN, EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE)); PartitionType partType = PartitionType.valueOf(Text.readString(in)); if (partType == PartitionType.UNPARTITIONED) { partitionInfo = SinglePartitionInfo.read(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/JsonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/JsonUtil.java index 1cd52b5067..d1eccaeb4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/JsonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/JsonUtil.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -63,6 +64,17 @@ public class JsonUtil { } } + /** + * Parse json text to ObjectNode + **/ + public static ObjectNode parseObject(String text) { + try { + return (ObjectNode) objectMapper.readTree(text); + } catch (Exception e) { + throw new RuntimeException("parseObject exception.", e); + } + } + public static T readValue(String text, Class clazz) throws JsonProcessingException { return objectMapper.readValue(text, clazz); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java index c2447c8428..1357608eb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java @@ -26,17 +26,19 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.common.util.JsonUtil; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; /** * Util for ES, some static method. @@ -78,28 +80,6 @@ public class EsUtil { } } - - /** - * Get the json object from specified jsonObject - */ - public static JSONObject getJsonObject(JSONObject jsonObject, String key, int fromIndex) { - int firstOccr = key.indexOf('.', fromIndex); - if (firstOccr == -1) { - String token = key.substring(key.lastIndexOf('.') + 1); - if (jsonObject.containsKey(token)) { - return (JSONObject) jsonObject.get(token); - } else { - return null; - } - } - String fieldName = key.substring(fromIndex, firstOccr); - if (jsonObject.containsKey(fieldName)) { - return getJsonObject((JSONObject) jsonObject.get(fieldName), key, firstOccr + 1); - } else { - return null; - } - } - /** * Get boolean throw DdlException when parse error **/ @@ -113,46 +93,15 @@ public class EsUtil { } } - /** - * Get Array fields. - **/ - public static List getArrayFields(String indexMapping) { - JSONObject mappings = getMapping(indexMapping); - JSONObject meta; - if (!mappings.containsKey("_meta")) { - // For ES 6.x and 7.x - String firstType = (String) mappings.keySet().iterator().next(); - if (!"properties".equals(firstType)) { - // If type is not passed in takes the first type. - JSONObject firstData = (JSONObject) mappings.get(firstType); - if (!firstData.containsKey("_meta")) { - return new ArrayList<>(); - } else { - meta = (JSONObject) firstData.get("_meta"); - } - } else { - return new ArrayList<>(); - } - } else { - meta = (JSONObject) mappings.get("_meta"); - } - if (!meta.containsKey("doris")) { - return new ArrayList<>(); - } - JSONObject dorisMeta = (JSONObject) meta.get("doris"); - return (List) dorisMeta.get("array_fields"); - } - - private static JSONObject getMapping(String indexMapping) { - JSONObject jsonObject = (JSONObject) JSONValue.parse(indexMapping); + @VisibleForTesting + public static ObjectNode getMapping(String indexMapping) { + ObjectNode jsonNodes = JsonUtil.parseObject(indexMapping); // If the indexName use alias takes the first mapping - Iterator keys = jsonObject.keySet().iterator(); - String docKey = keys.next(); - JSONObject docData = (JSONObject) jsonObject.get(docKey); - return (JSONObject) docData.get("mappings"); + return (ObjectNode) jsonNodes.iterator().next().get("mappings"); } - private static JSONObject getRootSchema(JSONObject mappings, String mappingType) { + @VisibleForTesting + public static ObjectNode getRootSchema(ObjectNode mappings, String mappingType) { // Type is null in the following three cases // 1. Equal 6.8.x and after // 2. Multi-catalog auto infer @@ -160,10 +109,10 @@ public class EsUtil { if (mappingType == null) { // remove dynamic templates, for ES 7.x and 8.x checkNonPropertiesFields(mappings); - String firstType = (String) mappings.keySet().iterator().next(); + String firstType = mappings.fieldNames().next(); if (!"properties".equals(firstType)) { // If type is not passed in takes the first type. - JSONObject firstData = (JSONObject) mappings.get(firstType); + ObjectNode firstData = (ObjectNode) mappings.get(firstType); // check for ES 6.x and before checkNonPropertiesFields(firstData); return firstData; @@ -171,8 +120,8 @@ public class EsUtil { // Equal 7.x and after return mappings; } else { - if (mappings.containsKey(mappingType)) { - JSONObject jsonData = (JSONObject) mappings.get(mappingType); + if (mappings.has(mappingType)) { + ObjectNode jsonData = (ObjectNode) mappings.get(mappingType); // check for ES 6.x and before checkNonPropertiesFields(jsonData); return jsonData; @@ -187,7 +136,7 @@ public class EsUtil { * * @param mappings */ - private static void checkNonPropertiesFields(JSONObject mappings) { + private static void checkNonPropertiesFields(ObjectNode mappings) { // remove `_meta` field mappings.remove("_meta"); // remove `dynamic_templates` field @@ -199,12 +148,12 @@ public class EsUtil { } /** - * Get mapping properties JSONObject. + * Get mapping properties transform to ObjectNode. **/ - public static JSONObject getMappingProps(String sourceIndex, String indexMapping, String mappingType) { - JSONObject mappings = getMapping(indexMapping); - JSONObject rootSchema = getRootSchema(mappings, mappingType); - JSONObject properties = (JSONObject) rootSchema.get("properties"); + public static ObjectNode getMappingProps(String sourceIndex, String indexMapping, String mappingType) { + ObjectNode mappings = getMapping(indexMapping); + ObjectNode rootSchema = getRootSchema(mappings, mappingType); + ObjectNode properties = (ObjectNode) rootSchema.get("properties"); if (properties == null) { throw new DorisEsException( "index[" + sourceIndex + "] type[" + mappingType + "] mapping not found for the ES Cluster"); @@ -212,7 +161,6 @@ public class EsUtil { return properties; } - /** * Generate columns from ES Cluster. * Add mappingEsId config in es external catalog. @@ -220,9 +168,14 @@ public class EsUtil { public static List genColumnsFromEs(EsRestClient client, String indexName, String mappingType, boolean mappingEsId) { String mapping = client.getMapping(indexName); - JSONObject mappingProps = getMappingProps(indexName, mapping, mappingType); - List arrayFields = getArrayFields(mapping); - Set keys = (Set) mappingProps.keySet(); + ObjectNode mappings = getMapping(mapping); + ObjectNode rootSchema = getRootSchema(mappings, mappingType); + return genColumnsFromEs(indexName, mappingType, rootSchema, mappingEsId); + } + + @VisibleForTesting + public static List genColumnsFromEs(String indexName, String mappingType, ObjectNode rootSchema, + boolean mappingEsId) { List columns = new ArrayList<>(); if (mappingEsId) { Column column = new Column(); @@ -233,68 +186,143 @@ public class EsUtil { column.setUniqueId(-1); columns.add(column); } - for (String key : keys) { - JSONObject field = (JSONObject) mappingProps.get(key); - Type type; - // Complex types are treating as String types for now. - if (field.containsKey("type")) { - type = toDorisType(field.get("type").toString()); - } else { - type = Type.STRING; - } - Column column = new Column(); - column.setName(key); - column.setIsKey(true); - column.setIsAllowNull(true); - column.setUniqueId(-1); - if (arrayFields.contains(key)) { - column.setType(ArrayType.create(type, true)); - } else { - column.setType(type); + ObjectNode mappingProps = (ObjectNode) rootSchema.get("properties"); + if (mappingProps == null) { + throw new DorisEsException( + "index[" + indexName + "] type[" + mappingType + "] mapping not found for the ES Cluster"); + } + List arrayFields = new ArrayList<>(); + JsonNode meta = mappingProps.get("_meta"); + if (meta != null) { + JsonNode dorisMeta = meta.get("doris"); + if (dorisMeta != null) { + arrayFields = dorisMeta.findValuesAsText("array_fields"); } + } + Iterator iterator = mappingProps.fieldNames(); + while (iterator.hasNext()) { + String fieldName = iterator.next(); + ObjectNode fieldValue = (ObjectNode) mappingProps.get(fieldName); + Column column = parseEsField(fieldName, fieldValue, arrayFields); columns.add(column); } return columns; } - /** - * Transfer es type to doris type. - **/ - public static Type toDorisType(String esType) { - // reference https://www.elastic.co/guide/en/elasticsearch/reference/8.3/sql-data-types.html - switch (esType) { - case "null": - return Type.NULL; - case "boolean": - return Type.BOOLEAN; - case "byte": - return Type.TINYINT; - case "short": - return Type.SMALLINT; - case "integer": - return Type.INT; - case "long": - return Type.BIGINT; - case "unsigned_long": - return Type.LARGEINT; - case "float": - case "half_float": - return Type.FLOAT; - case "double": - case "scaled_float": - return Type.DOUBLE; - case "date": - return ScalarType.createDateV2Type(); - case "keyword": - case "text": - case "ip": - case "nested": - case "object": - return ScalarType.createStringType(); - default: - return Type.UNSUPPORTED; + private static Column parseEsField(String fieldName, ObjectNode fieldValue, List arrayFields) { + Column column = new Column(); + column.setName(fieldName); + column.setIsKey(true); + column.setIsAllowNull(true); + column.setUniqueId(-1); + Type type; + // Complex types are treating as String types for now. + if (fieldValue.has("type")) { + String typeStr = fieldValue.get("type").asText(); + column.setComment("Elasticsearch type is " + typeStr); + // reference https://www.elastic.co/guide/en/elasticsearch/reference/8.3/sql-data-types.html + switch (typeStr) { + case "null": + type = Type.NULL; + break; + case "boolean": + type = Type.BOOLEAN; + break; + case "byte": + type = Type.TINYINT; + break; + case "short": + type = Type.SMALLINT; + break; + case "integer": + type = Type.INT; + break; + case "long": + type = Type.BIGINT; + break; + case "unsigned_long": + type = Type.LARGEINT; + break; + case "float": + case "half_float": + type = Type.FLOAT; + break; + case "double": + case "scaled_float": + type = Type.DOUBLE; + break; + case "date": + type = parseEsDateType(column, fieldValue); + break; + case "keyword": + case "text": + case "ip": + case "nested": + case "object": + type = ScalarType.createStringType(); + break; + default: + type = Type.UNSUPPORTED; + } + } else { + type = Type.STRING; + column.setComment("Elasticsearch no type"); } + if (arrayFields.contains(fieldName)) { + column.setType(ArrayType.create(type, true)); + } else { + column.setType(type); + } + return column; } -} + private static final List ALLOW_DATE_FORMATS = Lists.newArrayList("yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd", + "epoch_millis"); + /** + * Parse es date to doris type by format + **/ + private static Type parseEsDateType(Column column, ObjectNode field) { + if (!field.has("format")) { + // default format + column.setComment("Elasticsearch type is date, no format"); + return ScalarType.createDatetimeV2Type(0); + } else { + String originFormat = field.get("format").asText(); + String[] formats = originFormat.split("\\|\\|"); + boolean dateTimeFlag = false; + boolean dateFlag = false; + boolean bigIntFlag = false; + for (String format : formats) { + // pre-check format + if (!ALLOW_DATE_FORMATS.contains(format)) { + column.setComment( + "Elasticsearch type is date, format is " + format + " not support, use String type"); + return ScalarType.createStringType(); + } + switch (format) { + case "yyyy-MM-dd HH:mm:ss": + dateTimeFlag = true; + break; + case "yyyy-MM-dd": + dateFlag = true; + break; + case "epoch_millis": + default: + bigIntFlag = true; + } + } + column.setComment("Elasticsearch type is date, format is " + originFormat); + if (dateTimeFlag) { + return ScalarType.createDatetimeV2Type(0); + } + if (dateFlag) { + return ScalarType.createDateV2Type(); + } + if (bigIntFlag) { + return Type.BIGINT; + } + return ScalarType.createStringType(); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java index 42500d7f96..26d1bc1dcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java @@ -20,8 +20,11 @@ package org.apache.doris.external.elasticsearch; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang3.StringUtils; -import org.json.simple.JSONObject; + +import java.util.Iterator; /** * Get index mapping from remote ES Cluster, and resolved `keyword` and `doc_values` field @@ -55,69 +58,88 @@ public class MappingPhase implements SearchPhase { * @param indexMapping the return value of _mapping */ public static void resolveFields(SearchContext searchContext, String indexMapping) throws DorisEsException { - JSONObject properties = EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping, searchContext.type()); + ObjectNode properties = EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping, searchContext.type()); for (Column col : searchContext.columns()) { String colName = col.getName(); // _id not exist mapping, but be can query it. if (!"_id".equals(colName)) { - if (!properties.containsKey(colName)) { + if (!properties.has(colName)) { throw new DorisEsException( "index[" + searchContext.sourceIndex() + "] mapping[" + indexMapping + "] not found " + "column " + colName + " for the ES Cluster"); } - JSONObject fieldObject = (JSONObject) properties.get(colName); - resolveKeywordFields(searchContext, fieldObject, colName); - resolveDocValuesFields(searchContext, fieldObject, colName); + ObjectNode fieldObject = (ObjectNode) properties.get(colName); + if (!fieldObject.has("type")) { + continue; + } + String fieldType = fieldObject.get("type").asText(); + resolveDateFields(searchContext, fieldObject, colName, fieldType); + resolveKeywordFields(searchContext, fieldObject, colName, fieldType); + resolveDocValuesFields(searchContext, fieldObject, colName, fieldType); + } + } + } + + private static void resolveDateFields(SearchContext searchContext, ObjectNode fieldObject, String colName, + String fieldType) { + // Compat use default/strict_date_optional_time format date type, need transform datetime to + if ("date".equals(fieldType)) { + if (!fieldObject.has("format") || "strict_date_optional_time".equals(fieldObject.get("format").asText())) { + searchContext.needCompatDateFields().add(colName); } } } // get a field of keyword type in the fields - private static void resolveKeywordFields(SearchContext searchContext, JSONObject fieldObject, String colName) { - String fieldType = (String) fieldObject.get("type"); + private static void resolveKeywordFields(SearchContext searchContext, ObjectNode fieldObject, String colName, + String fieldType) { // string-type field used keyword type to generate predicate // if text field type seen, we should use the `field` keyword type? if ("text".equals(fieldType)) { - JSONObject fieldsObject = (JSONObject) fieldObject.get("fields"); + JsonNode fieldsObject = fieldObject.get("fields"); if (fieldsObject != null) { - for (Object key : fieldsObject.keySet()) { - JSONObject innerTypeObject = (JSONObject) fieldsObject.get(key); + Iterator fieldNames = fieldsObject.fieldNames(); + while (fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + ObjectNode innerTypeObject = (ObjectNode) fieldsObject.get(fieldName); // just for text type - if ("keyword".equals(innerTypeObject.get("type"))) { - searchContext.fetchFieldsContext().put(colName, colName + "." + key); + if ("keyword".equals(innerTypeObject.get("type").asText())) { + searchContext.fetchFieldsContext().put(colName, colName + "." + fieldName); } } } } } - private static void resolveDocValuesFields(SearchContext searchContext, JSONObject fieldObject, String colName) { - String fieldType = (String) fieldObject.get("type"); + private static void resolveDocValuesFields(SearchContext searchContext, ObjectNode fieldObject, String colName, + String fieldType) { String docValueField = null; if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { - JSONObject fieldsObject = (JSONObject) fieldObject.get("fields"); + JsonNode fieldsObject = fieldObject.get("fields"); if (fieldsObject != null) { - for (Object key : fieldsObject.keySet()) { - JSONObject innerTypeObject = (JSONObject) fieldsObject.get((String) key); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains((String) innerTypeObject.get("type"))) { + Iterator fieldNames = fieldsObject.fieldNames(); + while (fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + ObjectNode innerTypeObject = (ObjectNode) fieldsObject.get(fieldName); + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(innerTypeObject.get("type").asText())) { continue; } - if (innerTypeObject.containsKey("doc_values")) { - boolean docValue = (Boolean) innerTypeObject.get("doc_values"); + if (innerTypeObject.has("doc_values")) { + boolean docValue = innerTypeObject.get("doc_values").asBoolean(); if (docValue) { docValueField = colName; } } else { // a : {c : {}} -> a -> a.c - docValueField = colName + "." + key; + docValueField = colName + "." + fieldName; } } } } else { // set doc_value = false manually - if (fieldObject.containsKey("doc_values")) { - Boolean docValue = (Boolean) fieldObject.get("doc_values"); + if (fieldObject.has("doc_values")) { + boolean docValue = fieldObject.get("doc_values").asBoolean(); if (!docValue) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java index cc269a452a..a0a805747a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java @@ -42,6 +42,9 @@ import lombok.Builder; import lombok.Data; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; import java.io.IOException; import java.io.StringWriter; @@ -123,8 +126,12 @@ public final class QueryBuilders { .build()); } - private static QueryBuilder parseBinaryPredicate(Expr expr, TExprOpcode opCode, String column) { + private static QueryBuilder parseBinaryPredicate(Expr expr, TExprOpcode opCode, String column, + boolean needDateCompat) { Object value = toDorisLiteral(expr.getChild(1)); + if (needDateCompat) { + value = compatDefaultDate(value); + } switch (opCode) { case EQ: case EQ_FOR_NULL: @@ -180,10 +187,14 @@ public final class QueryBuilders { } } - private static QueryBuilder parseInPredicate(Expr expr, String column) { + private static QueryBuilder parseInPredicate(Expr expr, String column, boolean needDateCompat) { InPredicate inPredicate = (InPredicate) expr; - List values = inPredicate.getListChildren().stream().map(QueryBuilders::toDorisLiteral) - .collect(Collectors.toList()); + List values = inPredicate.getListChildren().stream().map(v -> { + if (needDateCompat) { + return compatDefaultDate(v); + } + return toDorisLiteral(v); + }).collect(Collectors.toList()); if (inPredicate.isNotIn()) { return QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(column, values)); } @@ -237,10 +248,13 @@ public final class QueryBuilders { notPushDownList.add(expr); return null; } + // Check whether the date type need compat, it must before keyword replace. + List needCompatDateFields = builderOptions.getNeedCompatDateFields(); + boolean needDateCompat = needCompatDateFields != null && needCompatDateFields.contains(column); // Replace col with col.keyword if mapping exist. column = fieldsContext.getOrDefault(column, column); if (expr instanceof BinaryPredicate) { - return parseBinaryPredicate(expr, opCode, column); + return parseBinaryPredicate(expr, opCode, column, needDateCompat); } if (expr instanceof IsNullPredicate) { return parseIsNullPredicate(expr, column); @@ -254,7 +268,7 @@ public final class QueryBuilders { } } if (expr instanceof InPredicate) { - return parseInPredicate(expr, column); + return parseInPredicate(expr, column, needDateCompat); } if (expr instanceof FunctionCallExpr) { return parseFunctionCallExpr(expr); @@ -262,6 +276,16 @@ public final class QueryBuilders { return null; } + private static final DateTimeFormatter dorisFmt = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); + private static final DateTimeFormatter esFmt = ISODateTimeFormat.dateTime(); + + private static Object compatDefaultDate(Object value) { + if (value == null) { + return null; + } + return dorisFmt.parseDateTime(value.toString()).toString(esFmt); + } + /** * Expr trans to doris literal. **/ @@ -424,6 +448,8 @@ public final class QueryBuilders { public static class BuilderOptions { private boolean likePushDown; + + private List needCompatDateFields; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchContext.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchContext.java index 6d55856209..bb842ea567 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchContext.java @@ -20,6 +20,7 @@ package org.apache.doris.external.elasticsearch; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -61,6 +62,8 @@ public class SearchContext { // fetch field value from doc_values, this is activated when `enable_docvalue_scan= true` private Map docValueFieldsContext = Maps.newHashMap(); + private List needCompatDateFields = Lists.newArrayList(); + // sourceIndex is the name of index when creating ES external table private final String sourceIndex; @@ -125,6 +128,10 @@ public class SearchContext { return docValueFieldsContext; } + public List needCompatDateFields() { + return needCompatDateFields; + } + public void version(EsMajorVersion version) { this.version = version; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java index b8358494ad..0e62f93657 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -363,7 +363,8 @@ public class EsScanNode extends ScanNode { List notPushDownList = new ArrayList<>(); for (Expr expr : conjuncts) { QueryBuilder queryBuilder = QueryBuilders.toEsDsl(expr, notPushDownList, fieldsContext, - BuilderOptions.builder().likePushDown(table.isLikePushDown()).build()); + BuilderOptions.builder().likePushDown(table.isLikePushDown()) + .needCompatDateFields(table.needCompatDateFields()).build()); if (queryBuilder != null) { hasFilter = true; boolQueryBuilder.must(queryBuilder); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java index a744b0fba0..8816935ba5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java @@ -17,31 +17,14 @@ package org.apache.doris.external.elasticsearch; -import org.apache.doris.analysis.BinaryPredicate; -import org.apache.doris.analysis.BinaryPredicate.Operator; -import org.apache.doris.analysis.CastExpr; -import org.apache.doris.analysis.CompoundPredicate; -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.FloatLiteral; -import org.apache.doris.analysis.FunctionCallExpr; -import org.apache.doris.analysis.InPredicate; -import org.apache.doris.analysis.IntLiteral; -import org.apache.doris.analysis.IsNullPredicate; -import org.apache.doris.analysis.LikePredicate; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.EsResource; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.Type; import org.apache.doris.common.ExceptionChecker; -import org.apache.doris.external.elasticsearch.QueryBuilders.BuilderOptions; +import com.fasterxml.jackson.databind.node.ObjectNode; import mockit.Expectations; import mockit.Injectable; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -51,9 +34,7 @@ import org.junit.rules.ExpectedException; import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; /** * Test for es util. @@ -62,16 +43,6 @@ public class EsUtilTest extends EsTestCase { private List columns = new ArrayList<>(); - private String jsonStr = "{\"settings\": {\n" + " \"index\": {\n" + " \"bpack\": {\n" - + " \"partition\": {\n" + " \"upperbound\": \"12\"\n" - + " }\n" + " },\n" + " \"number_of_shards\": \"5\",\n" - + " \"provided_name\": \"indexa\",\n" - + " \"creation_date\": \"1539328532060\",\n" - + " \"number_of_replicas\": \"1\",\n" - + " \"uuid\": \"plNNtKiiQ9-n6NpNskFzhQ\",\n" + " \"version\": {\n" - + " \"created\": \"5050099\"\n" + " }\n" + " }\n" - + " }}"; - @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -149,189 +120,31 @@ public class EsUtilTest extends EsTestCase { Assertions.assertFalse(searchContext.docValueFieldsContext().containsKey("k3")); } - @Test - public void testGetJsonObject() { - JSONObject json = (JSONObject) JSONValue.parse(jsonStr); - JSONObject upperBoundSetting = EsUtil.getJsonObject(json, "settings.index.bpack.partition", 0); - Assertions.assertTrue(upperBoundSetting.containsKey("upperbound")); - Assertions.assertEquals("12", (String) upperBoundSetting.get("upperbound")); - - JSONObject unExistKey = EsUtil.getJsonObject(json, "set", 0); - Assertions.assertNull(unExistKey); - - JSONObject singleKey = EsUtil.getJsonObject(json, "settings", 0); - Assertions.assertTrue(singleKey.containsKey("index")); - } - - @Test(expected = ClassCastException.class) - public void testGetJsonObjectWithException() { - JSONObject json = (JSONObject) JSONValue.parse(jsonStr); - // only support json object could not get string value directly from this api, exception will be threw - EsUtil.getJsonObject(json, "settings.index.bpack.partition.upperbound", 0); - } - - @Test - public void testBinaryPredicateConvertEsDsl() { - SlotRef k1 = new SlotRef(null, "k1"); - IntLiteral intLiteral = new IntLiteral(3); - Expr eqExpr = new BinaryPredicate(Operator.EQ, k1, intLiteral); - Expr neExpr = new BinaryPredicate(Operator.NE, k1, intLiteral); - Expr leExpr = new BinaryPredicate(Operator.LE, k1, intLiteral); - Expr geExpr = new BinaryPredicate(Operator.GE, k1, intLiteral); - Expr ltExpr = new BinaryPredicate(Operator.LT, k1, intLiteral); - Expr gtExpr = new BinaryPredicate(Operator.GT, k1, intLiteral); - Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "k1"), new IntLiteral(3)); - Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr).toJson()); - Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", - QueryBuilders.toEsDsl(neExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", QueryBuilders.toEsDsl(ltExpr).toJson()); - Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", QueryBuilders.toEsDsl(gtExpr).toJson()); - Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(efnExpr).toJson()); - } - - @Test - public void testCompoundPredicateConvertEsDsl() { - SlotRef k1 = new SlotRef(null, "k1"); - IntLiteral intLiteral1 = new IntLiteral(3); - SlotRef k2 = new SlotRef(null, "k2"); - IntLiteral intLiteral2 = new IntLiteral(5); - BinaryPredicate binaryPredicate1 = new BinaryPredicate(Operator.EQ, k1, intLiteral1); - BinaryPredicate binaryPredicate2 = new BinaryPredicate(Operator.GT, k2, intLiteral2); - CompoundPredicate andPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate1, - binaryPredicate2); - CompoundPredicate orPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, binaryPredicate1, - binaryPredicate2); - CompoundPredicate notPredicate = new CompoundPredicate(CompoundPredicate.Operator.NOT, binaryPredicate1, null); - Assertions.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", - QueryBuilders.toEsDsl(andPredicate).toJson()); - Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", - QueryBuilders.toEsDsl(orPredicate).toJson()); - Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", - QueryBuilders.toEsDsl(notPredicate).toJson()); - } - - @Test - public void testIsNullPredicateConvertEsDsl() { - SlotRef k1 = new SlotRef(null, "k1"); - IsNullPredicate isNullPredicate = new IsNullPredicate(k1, false); - IsNullPredicate isNotNullPredicate = new IsNullPredicate(k1, true); - Assertions.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}", - QueryBuilders.toEsDsl(isNullPredicate).toJson()); - Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", QueryBuilders.toEsDsl(isNotNullPredicate).toJson()); - } - - @Test - public void testLikePredicateConvertEsDsl() { - SlotRef k1 = new SlotRef(null, "k1"); - StringLiteral stringLiteral1 = new StringLiteral("%1%"); - StringLiteral stringLiteral2 = new StringLiteral("*1*"); - StringLiteral stringLiteral3 = new StringLiteral("1_2"); - LikePredicate likePredicate1 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral1); - LikePredicate regexPredicate = new LikePredicate(LikePredicate.Operator.REGEXP, k1, stringLiteral2); - LikePredicate likePredicate2 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral3); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(likePredicate1).toJson()); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(regexPredicate).toJson()); - Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", QueryBuilders.toEsDsl(likePredicate2).toJson()); - List notPushDownList = new ArrayList<>(); - Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate2, notPushDownList, new HashMap<>(), BuilderOptions.builder().likePushDown(false).build())); - Assertions.assertFalse(notPushDownList.isEmpty()); - } - - @Test - public void testInPredicateConvertEsDsl() { - SlotRef k1 = new SlotRef(null, "k1"); - IntLiteral intLiteral1 = new IntLiteral(3); - IntLiteral intLiteral2 = new IntLiteral(5); - List intLiterals = new ArrayList<>(); - intLiterals.add(intLiteral1); - intLiterals.add(intLiteral2); - InPredicate isInPredicate = new InPredicate(k1, intLiterals, false); - InPredicate isNotInPredicate = new InPredicate(k1, intLiterals, true); - Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", QueryBuilders.toEsDsl(isInPredicate).toJson()); - Assertions.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}", - QueryBuilders.toEsDsl(isNotInPredicate).toJson()); - } - - @Test - public void testFunctionCallConvertEsDsl() { - SlotRef k1 = new SlotRef(null, "k1"); - String str = "{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}"; - StringLiteral stringLiteral = new StringLiteral(str); - List exprs = new ArrayList<>(); - exprs.add(k1); - exprs.add(stringLiteral); - FunctionCallExpr functionCallExpr = new FunctionCallExpr("esquery", exprs); - Assertions.assertEquals(str, QueryBuilders.toEsDsl(functionCallExpr).toJson()); - - SlotRef k2 = new SlotRef(null, "k2"); - IntLiteral intLiteral = new IntLiteral(5); - BinaryPredicate binaryPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral); - CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate, - functionCallExpr); - Assertions.assertEquals( - "{\"bool\":{\"must\":[{\"term\":{\"k2\":5}},{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}]}}", - QueryBuilders.toEsDsl(compoundPredicate).toJson()); - } - - @Test - public void testCastConvertEsDsl() { - FloatLiteral floatLiteral = new FloatLiteral(3.14); - CastExpr castExpr = new CastExpr(Type.INT, floatLiteral); - BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, castExpr, new IntLiteral(3)); - List notPushDownList = new ArrayList<>(); - Map fieldsContext = new HashMap<>(); - BuilderOptions builderOptions = BuilderOptions.builder() - .likePushDown(Boolean.parseBoolean(EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE)).build(); - Assertions.assertNull(QueryBuilders.toEsDsl(castPredicate, notPushDownList, fieldsContext, builderOptions)); - Assertions.assertEquals(1, notPushDownList.size()); - - SlotRef k2 = new SlotRef(null, "k2"); - IntLiteral intLiteral = new IntLiteral(5); - BinaryPredicate eqPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral); - CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, castPredicate, - eqPredicate); - - QueryBuilders.toEsDsl(compoundPredicate, notPushDownList, fieldsContext, builderOptions); - Assertions.assertEquals(3, notPushDownList.size()); - - SlotRef k3 = new SlotRef(null, "k3"); - k3.setType(Type.FLOAT); - CastExpr castDoubleExpr = new CastExpr(Type.DOUBLE, k3); - BinaryPredicate castDoublePredicate = new BinaryPredicate(Operator.GE, castDoubleExpr, - new FloatLiteral(3.0, Type.DOUBLE)); - QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions); - Assertions.assertEquals(3, notPushDownList.size()); - } - @Test public void testEs6Mapping() throws IOException, URISyntaxException { - JSONObject testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_aliases_mapping.json"), + ObjectNode testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_aliases_mapping.json"), "doc"); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testAliases.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testAliases.toString()); - JSONObject testAliasesNoType = EsUtil.getMappingProps("test", + ObjectNode testAliasesNoType = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_aliases_mapping.json"), null); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", - testAliasesNoType.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testAliasesNoType.toString()); - JSONObject testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_index_mapping.json"), + ObjectNode testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_index_mapping.json"), "doc"); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testIndex.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testIndex.toString()); - JSONObject testDynamicTemplates = EsUtil.getMappingProps("test", + ObjectNode testDynamicTemplates = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_dynamic_templates_mapping.json"), "doc"); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", - testDynamicTemplates.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testDynamicTemplates.toString()); expectedEx.expect(DorisEsException.class); expectedEx.expectMessage("Do not support index without explicit mapping."); @@ -344,31 +157,29 @@ public class EsUtilTest extends EsTestCase { @Test public void testEs7Mapping() throws IOException, URISyntaxException { - JSONObject testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_aliases_mapping.json"), + ObjectNode testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_aliases_mapping.json"), null); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testAliases.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testAliases.toString()); - JSONObject testAliasesErrorType = EsUtil.getMappingProps("test", + ObjectNode testAliasesErrorType = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_aliases_mapping.json"), "doc"); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", - testAliasesErrorType.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testAliasesErrorType.toString()); - JSONObject testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_index_mapping.json"), + ObjectNode testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_index_mapping.json"), "doc"); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testIndex.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testIndex.toString()); - JSONObject testDynamicTemplates = EsUtil.getMappingProps("test", + ObjectNode testDynamicTemplates = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_dynamic_templates_mapping.json"), null); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", - testDynamicTemplates.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testDynamicTemplates.toString()); expectedEx.expect(DorisEsException.class); expectedEx.expectMessage("Do not support index without explicit mapping."); @@ -377,35 +188,66 @@ public class EsUtilTest extends EsTestCase { @Test public void testEs8Mapping() throws IOException, URISyntaxException { - JSONObject testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_aliases_mapping.json"), + ObjectNode testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_aliases_mapping.json"), null); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testAliases.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testAliases.toString()); - JSONObject testAliasesErrorType = EsUtil.getMappingProps("test", + ObjectNode testAliasesErrorType = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_aliases_mapping.json"), "doc"); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", - testAliasesErrorType.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testAliasesErrorType.toString()); - JSONObject testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_index_mapping.json"), + ObjectNode testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_index_mapping.json"), "doc"); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testIndex.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testIndex.toString()); - JSONObject testDynamicTemplates = EsUtil.getMappingProps("test", + ObjectNode testDynamicTemplates = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_dynamic_templates_mapping.json"), "doc"); - Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\"," - + "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}}," - + "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", - testDynamicTemplates.toJSONString()); + Assertions.assertEquals( + "{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}", + testDynamicTemplates.toString()); expectedEx.expect(DorisEsException.class); expectedEx.expectMessage("Do not support index without explicit mapping."); EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_only_dynamic_templates_mapping.json"), "doc"); } + @Test + public void testDateType() throws IOException, URISyntaxException { + ObjectNode testDateFormat = EsUtil.getRootSchema( + EsUtil.getMapping(loadJsonFromFile("data/es/test_date_format.json")), null); + List parseColumns = EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false); + Assertions.assertEquals(8, parseColumns.size()); + for (Column column : parseColumns) { + String name = column.getName(); + String type = column.getType().toSql(); + if ("test2".equals(name)) { + Assertions.assertEquals("datetimev2(0)", type); + } + if ("test3".equals(name)) { + Assertions.assertEquals("datetimev2(0)", type); + } + if ("test4".equals(name)) { + Assertions.assertEquals("datev2", type); + } + if ("test5".equals(name)) { + Assertions.assertEquals("datetimev2(0)", type); + } + if ("test6".equals(name)) { + Assertions.assertEquals("datev2", type); + } + if ("test7".equals(name)) { + Assertions.assertEquals("datetimev2(0)", type); + } + if ("test8".equals(name)) { + Assertions.assertEquals("bigint(20)", type); + } + } + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java index c1511bd206..d0cc111bef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java @@ -17,17 +17,39 @@ package org.apache.doris.external.elasticsearch; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.BinaryPredicate.Operator; +import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.CompoundPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FloatLiteral; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.InPredicate; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.IsNullPredicate; +import org.apache.doris.analysis.LikePredicate; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.Type; +import org.apache.doris.external.elasticsearch.QueryBuilders.BuilderOptions; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; import org.junit.Assert; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import java.io.IOException; import java.io.StringWriter; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -37,41 +59,170 @@ public class QueryBuildersTest { private final ObjectMapper mapper = new ObjectMapper(); + @Test + public void testBinaryPredicateConvertEsDsl() { + SlotRef k1 = new SlotRef(null, "k1"); + IntLiteral intLiteral = new IntLiteral(3); + Expr eqExpr = new BinaryPredicate(Operator.EQ, k1, intLiteral); + Expr neExpr = new BinaryPredicate(Operator.NE, k1, intLiteral); + Expr leExpr = new BinaryPredicate(Operator.LE, k1, intLiteral); + Expr geExpr = new BinaryPredicate(Operator.GE, k1, intLiteral); + Expr ltExpr = new BinaryPredicate(Operator.LT, k1, intLiteral); + Expr gtExpr = new BinaryPredicate(Operator.GT, k1, intLiteral); + Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "k1"), new IntLiteral(3)); + Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr).toJson()); + Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", + QueryBuilders.toEsDsl(neExpr).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", QueryBuilders.toEsDsl(ltExpr).toJson()); + Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", QueryBuilders.toEsDsl(gtExpr).toJson()); + Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(efnExpr).toJson()); + + SlotRef k2 = new SlotRef(null, "k2"); + Expr dateTimeLiteral = new StringLiteral("2023-02-19 22:00:00"); + Expr dateTimeEqExpr = new BinaryPredicate(Operator.EQ, k2, dateTimeLiteral); + Assertions.assertEquals("{\"term\":{\"k2\":\"2023-02-19T22:00:00.000+08:00\"}}", + QueryBuilders.toEsDsl(dateTimeEqExpr, new ArrayList<>(), new HashMap<>(), + BuilderOptions.builder().needCompatDateFields(Lists.newArrayList("k2")).build()).toJson()); + } + + @Test + public void testCompoundPredicateConvertEsDsl() { + SlotRef k1 = new SlotRef(null, "k1"); + IntLiteral intLiteral1 = new IntLiteral(3); + SlotRef k2 = new SlotRef(null, "k2"); + IntLiteral intLiteral2 = new IntLiteral(5); + BinaryPredicate binaryPredicate1 = new BinaryPredicate(Operator.EQ, k1, intLiteral1); + BinaryPredicate binaryPredicate2 = new BinaryPredicate(Operator.GT, k2, intLiteral2); + CompoundPredicate andPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate1, + binaryPredicate2); + CompoundPredicate orPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, binaryPredicate1, + binaryPredicate2); + CompoundPredicate notPredicate = new CompoundPredicate(CompoundPredicate.Operator.NOT, binaryPredicate1, null); + Assertions.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", + QueryBuilders.toEsDsl(andPredicate).toJson()); + Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}", + QueryBuilders.toEsDsl(orPredicate).toJson()); + Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", + QueryBuilders.toEsDsl(notPredicate).toJson()); + } + + @Test + public void testIsNullPredicateConvertEsDsl() { + SlotRef k1 = new SlotRef(null, "k1"); + IsNullPredicate isNullPredicate = new IsNullPredicate(k1, false); + IsNullPredicate isNotNullPredicate = new IsNullPredicate(k1, true); + Assertions.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}", + QueryBuilders.toEsDsl(isNullPredicate).toJson()); + Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", QueryBuilders.toEsDsl(isNotNullPredicate).toJson()); + } + + @Test + public void testLikePredicateConvertEsDsl() { + SlotRef k1 = new SlotRef(null, "k1"); + StringLiteral stringLiteral1 = new StringLiteral("%1%"); + StringLiteral stringLiteral2 = new StringLiteral("*1*"); + StringLiteral stringLiteral3 = new StringLiteral("1_2"); + LikePredicate likePredicate1 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral1); + LikePredicate regexPredicate = new LikePredicate(LikePredicate.Operator.REGEXP, k1, stringLiteral2); + LikePredicate likePredicate2 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral3); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(likePredicate1).toJson()); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(regexPredicate).toJson()); + Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", QueryBuilders.toEsDsl(likePredicate2).toJson()); + List notPushDownList = new ArrayList<>(); + Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate2, notPushDownList, new HashMap<>(), + BuilderOptions.builder().likePushDown(false).build())); + Assertions.assertFalse(notPushDownList.isEmpty()); + } + + @Test + public void testInPredicateConvertEsDsl() { + SlotRef k1 = new SlotRef(null, "k1"); + IntLiteral intLiteral1 = new IntLiteral(3); + IntLiteral intLiteral2 = new IntLiteral(5); + List intLiterals = new ArrayList<>(); + intLiterals.add(intLiteral1); + intLiterals.add(intLiteral2); + InPredicate isInPredicate = new InPredicate(k1, intLiterals, false); + InPredicate isNotInPredicate = new InPredicate(k1, intLiterals, true); + Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", QueryBuilders.toEsDsl(isInPredicate).toJson()); + Assertions.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}", + QueryBuilders.toEsDsl(isNotInPredicate).toJson()); + } + + @Test + public void testFunctionCallConvertEsDsl() { + SlotRef k1 = new SlotRef(null, "k1"); + String str = "{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}"; + StringLiteral stringLiteral = new StringLiteral(str); + List exprs = new ArrayList<>(); + exprs.add(k1); + exprs.add(stringLiteral); + FunctionCallExpr functionCallExpr = new FunctionCallExpr("esquery", exprs); + Assertions.assertEquals(str, QueryBuilders.toEsDsl(functionCallExpr).toJson()); + + SlotRef k2 = new SlotRef(null, "k2"); + IntLiteral intLiteral = new IntLiteral(5); + BinaryPredicate binaryPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral); + CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate, + functionCallExpr); + Assertions.assertEquals( + "{\"bool\":{\"must\":[{\"term\":{\"k2\":5}},{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}]}}", + QueryBuilders.toEsDsl(compoundPredicate).toJson()); + } + + @Test + public void testCastConvertEsDsl() { + FloatLiteral floatLiteral = new FloatLiteral(3.14); + CastExpr castExpr = new CastExpr(Type.INT, floatLiteral); + BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, castExpr, new IntLiteral(3)); + List notPushDownList = new ArrayList<>(); + Map fieldsContext = new HashMap<>(); + BuilderOptions builderOptions = BuilderOptions.builder().likePushDown(true).build(); + Assertions.assertNull(QueryBuilders.toEsDsl(castPredicate, notPushDownList, fieldsContext, builderOptions)); + Assertions.assertEquals(1, notPushDownList.size()); + + SlotRef k2 = new SlotRef(null, "k2"); + IntLiteral intLiteral = new IntLiteral(5); + BinaryPredicate eqPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral); + CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, castPredicate, + eqPredicate); + + QueryBuilders.toEsDsl(compoundPredicate, notPushDownList, fieldsContext, builderOptions); + Assertions.assertEquals(3, notPushDownList.size()); + + SlotRef k3 = new SlotRef(null, "k3"); + k3.setType(Type.FLOAT); + CastExpr castDoubleExpr = new CastExpr(Type.DOUBLE, k3); + BinaryPredicate castDoublePredicate = new BinaryPredicate(Operator.GE, castDoubleExpr, + new FloatLiteral(3.0, Type.DOUBLE)); + QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions); + Assertions.assertEquals(3, notPushDownList.size()); + } + @Test public void testTermQuery() throws Exception { - Assert.assertEquals("{\"term\":{\"k\":\"aaaa\"}}", - toJson(QueryBuilders.termQuery("k", "aaaa"))); - Assert.assertEquals("{\"term\":{\"aaaa\":\"k\"}}", - toJson(QueryBuilders.termQuery("aaaa", "k"))); - Assert.assertEquals("{\"term\":{\"k\":0}}", - toJson(QueryBuilders.termQuery("k", (byte) 0))); - Assert.assertEquals("{\"term\":{\"k\":123}}", - toJson(QueryBuilders.termQuery("k", (long) 123))); - Assert.assertEquals("{\"term\":{\"k\":41}}", - toJson(QueryBuilders.termQuery("k", (short) 41))); - Assert.assertEquals("{\"term\":{\"k\":128}}", - toJson(QueryBuilders.termQuery("k", 128))); - Assert.assertEquals("{\"term\":{\"k\":42.42}}", - toJson(QueryBuilders.termQuery("k", 42.42D))); - Assert.assertEquals("{\"term\":{\"k\":1.1}}", - toJson(QueryBuilders.termQuery("k", 1.1F))); - Assert.assertEquals("{\"term\":{\"k\":1}}", - toJson(QueryBuilders.termQuery("k", new BigDecimal(1)))); - Assert.assertEquals("{\"term\":{\"k\":121}}", - toJson(QueryBuilders.termQuery("k", new BigInteger("121")))); - Assert.assertEquals("{\"term\":{\"k\":true}}", - toJson(QueryBuilders.termQuery("k", new AtomicBoolean(true)))); + Assert.assertEquals("{\"term\":{\"k\":\"aaaa\"}}", toJson(QueryBuilders.termQuery("k", "aaaa"))); + Assert.assertEquals("{\"term\":{\"aaaa\":\"k\"}}", toJson(QueryBuilders.termQuery("aaaa", "k"))); + Assert.assertEquals("{\"term\":{\"k\":0}}", toJson(QueryBuilders.termQuery("k", (byte) 0))); + Assert.assertEquals("{\"term\":{\"k\":123}}", toJson(QueryBuilders.termQuery("k", (long) 123))); + Assert.assertEquals("{\"term\":{\"k\":41}}", toJson(QueryBuilders.termQuery("k", (short) 41))); + Assert.assertEquals("{\"term\":{\"k\":128}}", toJson(QueryBuilders.termQuery("k", 128))); + Assert.assertEquals("{\"term\":{\"k\":42.42}}", toJson(QueryBuilders.termQuery("k", 42.42D))); + Assert.assertEquals("{\"term\":{\"k\":1.1}}", toJson(QueryBuilders.termQuery("k", 1.1F))); + Assert.assertEquals("{\"term\":{\"k\":1}}", toJson(QueryBuilders.termQuery("k", new BigDecimal(1)))); + Assert.assertEquals("{\"term\":{\"k\":121}}", toJson(QueryBuilders.termQuery("k", new BigInteger("121")))); + Assert.assertEquals("{\"term\":{\"k\":true}}", toJson(QueryBuilders.termQuery("k", new AtomicBoolean(true)))); } @Test public void testTermsQuery() throws Exception { - Assert.assertEquals("{\"terms\":{\"k\":[]}}", - toJson(QueryBuilders.termsQuery("k", Collections.emptySet()))); + Assert.assertEquals("{\"terms\":{\"k\":[]}}", toJson(QueryBuilders.termsQuery("k", Collections.emptySet()))); - Assert.assertEquals("{\"terms\":{\"k\":[0]}}", - toJson(QueryBuilders.termsQuery("k", Collections.singleton(0)))); + Assert.assertEquals("{\"terms\":{\"k\":[0]}}", toJson(QueryBuilders.termsQuery("k", Collections.singleton(0)))); Assert.assertEquals("{\"terms\":{\"k\":[\"aaa\"]}}", toJson(QueryBuilders.termsQuery("k", Collections.singleton("aaa")))); @@ -91,45 +242,44 @@ public class QueryBuildersTest { @Test public void testBoolQuery() throws Exception { - QueryBuilders.QueryBuilder q1 = QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("k", "aaa")); + QueryBuilders.QueryBuilder q1 = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("k", "aaa")); - Assert.assertEquals("{\"bool\":{\"must\":{\"term\":{\"k\":\"aaa\"}}}}", - toJson(q1)); + Assert.assertEquals("{\"bool\":{\"must\":{\"term\":{\"k\":\"aaa\"}}}}", toJson(q1)); - QueryBuilders.QueryBuilder q2 = QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("k1", "aaa")).must(QueryBuilders.termQuery("k2", "bbb")); + QueryBuilders.QueryBuilder q2 = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("k1", "aaa")) + .must(QueryBuilders.termQuery("k2", "bbb")); Assert.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":\"aaa\"}},{\"term\":{\"k2\":\"bbb\"}}]}}", toJson(q2)); - QueryBuilders.QueryBuilder q3 = QueryBuilders.boolQuery() - .mustNot(QueryBuilders.termQuery("k", "fff")); + QueryBuilders.QueryBuilder q3 = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("k", "fff")); - Assert.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k\":\"fff\"}}}}", - toJson(q3)); + Assert.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k\":\"fff\"}}}}", toJson(q3)); QueryBuilders.QueryBuilder q4 = QueryBuilders.rangeQuery("k1").lt(200).gt(-200); QueryBuilders.QueryBuilder q5 = QueryBuilders.termsQuery("k2", Arrays.asList("aaa", "bbb", "ccc")); QueryBuilders.QueryBuilder q6 = QueryBuilders.boolQuery().must(q4).should(q5); - Assert.assertEquals("{\"bool\":{\"must\":{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},\"should\":{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}}}", toJson(q6)); - Assert.assertEquals("{\"bool\":{\"filter\":[{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}]}}", toJson(QueryBuilders.boolQuery().filter(q4).filter(q5))); - Assert.assertEquals("{\"bool\":{\"filter\":{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},\"must_not\":{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}}}", toJson(QueryBuilders.boolQuery().filter(q4).mustNot(q5))); + Assert.assertEquals( + "{\"bool\":{\"must\":{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},\"should\":{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}}}", + toJson(q6)); + Assert.assertEquals( + "{\"bool\":{\"filter\":[{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}]}}", + toJson(QueryBuilders.boolQuery().filter(q4).filter(q5))); + Assert.assertEquals( + "{\"bool\":{\"filter\":{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},\"must_not\":{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}}}", + toJson(QueryBuilders.boolQuery().filter(q4).mustNot(q5))); } @Test public void testExistsQuery() throws Exception { - Assert.assertEquals("{\"exists\":{\"field\":\"k\"}}", - toJson(QueryBuilders.existsQuery("k"))); + Assert.assertEquals("{\"exists\":{\"field\":\"k\"}}", toJson(QueryBuilders.existsQuery("k"))); } @Test public void testRangeQuery() throws Exception { - Assert.assertEquals("{\"range\":{\"k\":{\"lt\":123}}}", - toJson(QueryBuilders.rangeQuery("k").lt(123))); - Assert.assertEquals("{\"range\":{\"k\":{\"gt\":123}}}", - toJson(QueryBuilders.rangeQuery("k").gt(123))); + Assert.assertEquals("{\"range\":{\"k\":{\"lt\":123}}}", toJson(QueryBuilders.rangeQuery("k").lt(123))); + Assert.assertEquals("{\"range\":{\"k\":{\"gt\":123}}}", toJson(QueryBuilders.rangeQuery("k").gt(123))); Assert.assertEquals("{\"range\":{\"k\":{\"gte\":12345678}}}", toJson(QueryBuilders.rangeQuery("k").gte(12345678))); Assert.assertEquals("{\"range\":{\"k\":{\"lte\":12345678}}}", @@ -142,22 +292,19 @@ public class QueryBuildersTest { toJson(QueryBuilders.rangeQuery("k").gt(6789.33f).lte(9999.99f))); Assert.assertEquals("{\"range\":{\"k\":{\"gte\":1,\"lte\":\"zzz\"}}}", toJson(QueryBuilders.rangeQuery("k").gte(1).lte("zzz"))); - Assert.assertEquals("{\"range\":{\"k\":{\"gte\":\"zzz\"}}}", - toJson(QueryBuilders.rangeQuery("k").gte("zzz"))); + Assert.assertEquals("{\"range\":{\"k\":{\"gte\":\"zzz\"}}}", toJson(QueryBuilders.rangeQuery("k").gte("zzz"))); Assert.assertEquals("{\"range\":{\"k\":{\"gt\":\"aaa\",\"lt\":\"zzz\"}}}", toJson(QueryBuilders.rangeQuery("k").gt("aaa").lt("zzz"))); } @Test public void testMatchAllQuery() throws IOException { - Assert.assertEquals("{\"match_all\":{}}", - toJson(QueryBuilders.matchAllQuery())); + Assert.assertEquals("{\"match_all\":{}}", toJson(QueryBuilders.matchAllQuery())); } @Test public void testWildCardQuery() throws IOException { - Assert.assertEquals("{\"wildcard\":{\"k1\":\"?aa*\"}}", - toJson(QueryBuilders.wildcardQuery("k1", "?aa*"))); + Assert.assertEquals("{\"wildcard\":{\"k1\":\"?aa*\"}}", toJson(QueryBuilders.wildcardQuery("k1", "?aa*"))); } private String toJson(QueryBuilders.QueryBuilder builder) throws IOException { diff --git a/fe/fe-core/src/test/resources/data/es/test_date_format.json b/fe/fe-core/src/test/resources/data/es/test_date_format.json new file mode 100644 index 0000000000..98637c8502 --- /dev/null +++ b/fe/fe-core/src/test/resources/data/es/test_date_format.json @@ -0,0 +1,38 @@ +{ + "test_date_format": { + "mappings": { + "properties": { + "test1": { + "type": "keyword" + }, + "test2": { + "type": "date" + }, + "test3": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis" + }, + "test4": { + "type": "date", + "format": "yyyy-MM-dd||epoch_millis" + }, + "test5": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||epoch_millis" + }, + "test6": { + "type": "date", + "format": "yyyy-MM-dd" + }, + "test7": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss" + }, + "test8": { + "type": "date", + "format": "epoch_millis" + } + } + } + } +} \ No newline at end of file