[Enhancement](ES): Support mapping es date format and replace simple json with jackson (#16806)
* Support mapping es date format, default/yyyy-MM-dd HH:mm:ss/yyyy-MM-dd/epoch_millis * Replace simple json with jackson, resolve column order random problem * Add es array doc version
This commit is contained in:
Stalary
@ -145,6 +145,10 @@ public class EsTable extends Table {
|
||||
return esMetaStateTracker.searchContext().docValueFieldsContext();
|
||||
}
|
||||
|
||||
public List<String> needCompatDateFields() {
|
||||
return esMetaStateTracker.searchContext().needCompatDateFields();
|
||||
}
|
||||
|
||||
private void validate(Map<String, String> properties) throws DdlException {
|
||||
EsResource.valid(properties, false);
|
||||
hosts = properties.get(EsResource.HOSTS).trim();
|
||||
@ -270,10 +274,10 @@ public class EsTable extends Table {
|
||||
indexName = tableContext.get("indexName");
|
||||
mappingType = tableContext.get("mappingType");
|
||||
|
||||
enableDocValueScan = Boolean.parseBoolean(tableContext.getOrDefault("enableDocValueScan",
|
||||
EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE));
|
||||
enableKeywordSniff = Boolean.parseBoolean(tableContext.getOrDefault("enableKeywordSniff",
|
||||
EsResource.KEYWORD_SNIFF_DEFAULT_VALUE));
|
||||
enableDocValueScan = Boolean.parseBoolean(
|
||||
tableContext.getOrDefault("enableDocValueScan", EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE));
|
||||
enableKeywordSniff = Boolean.parseBoolean(
|
||||
tableContext.getOrDefault("enableKeywordSniff", EsResource.KEYWORD_SNIFF_DEFAULT_VALUE));
|
||||
if (tableContext.containsKey("maxDocValueFields")) {
|
||||
try {
|
||||
maxDocValueFields = Integer.parseInt(tableContext.get("maxDocValueFields"));
|
||||
@ -281,12 +285,12 @@ public class EsTable extends Table {
|
||||
maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS;
|
||||
}
|
||||
}
|
||||
nodesDiscovery = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.NODES_DISCOVERY,
|
||||
EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
|
||||
httpSslEnabled = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.HTTP_SSL_ENABLED,
|
||||
EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
|
||||
likePushDown = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.LIKE_PUSH_DOWN,
|
||||
EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE));
|
||||
nodesDiscovery = Boolean.parseBoolean(
|
||||
tableContext.getOrDefault(EsResource.NODES_DISCOVERY, EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
|
||||
httpSslEnabled = Boolean.parseBoolean(
|
||||
tableContext.getOrDefault(EsResource.HTTP_SSL_ENABLED, EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
|
||||
likePushDown = Boolean.parseBoolean(
|
||||
tableContext.getOrDefault(EsResource.LIKE_PUSH_DOWN, EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE));
|
||||
PartitionType partType = PartitionType.valueOf(Text.readString(in));
|
||||
if (partType == PartitionType.UNPARTITIONED) {
|
||||
partitionInfo = SinglePartitionInfo.read(in);
|
||||
|
||||
@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.MapperFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
@ -63,6 +64,17 @@ public class JsonUtil {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse json text to ObjectNode
|
||||
**/
|
||||
public static ObjectNode parseObject(String text) {
|
||||
try {
|
||||
return (ObjectNode) objectMapper.readTree(text);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("parseObject exception.", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static <T> T readValue(String text, Class<T> clazz) throws JsonProcessingException {
|
||||
return objectMapper.readValue(text, clazz);
|
||||
}
|
||||
|
||||
@ -26,17 +26,19 @@ import org.apache.doris.catalog.ScalarType;
|
||||
import org.apache.doris.catalog.Type;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.util.JsonUtil;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.json.simple.JSONObject;
|
||||
import org.json.simple.JSONValue;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Util for ES, some static method.
|
||||
@ -78,28 +80,6 @@ public class EsUtil {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the json object from specified jsonObject
|
||||
*/
|
||||
public static JSONObject getJsonObject(JSONObject jsonObject, String key, int fromIndex) {
|
||||
int firstOccr = key.indexOf('.', fromIndex);
|
||||
if (firstOccr == -1) {
|
||||
String token = key.substring(key.lastIndexOf('.') + 1);
|
||||
if (jsonObject.containsKey(token)) {
|
||||
return (JSONObject) jsonObject.get(token);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
String fieldName = key.substring(fromIndex, firstOccr);
|
||||
if (jsonObject.containsKey(fieldName)) {
|
||||
return getJsonObject((JSONObject) jsonObject.get(fieldName), key, firstOccr + 1);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get boolean throw DdlException when parse error
|
||||
**/
|
||||
@ -113,46 +93,15 @@ public class EsUtil {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Array fields.
|
||||
**/
|
||||
public static List<String> getArrayFields(String indexMapping) {
|
||||
JSONObject mappings = getMapping(indexMapping);
|
||||
JSONObject meta;
|
||||
if (!mappings.containsKey("_meta")) {
|
||||
// For ES 6.x and 7.x
|
||||
String firstType = (String) mappings.keySet().iterator().next();
|
||||
if (!"properties".equals(firstType)) {
|
||||
// If type is not passed in takes the first type.
|
||||
JSONObject firstData = (JSONObject) mappings.get(firstType);
|
||||
if (!firstData.containsKey("_meta")) {
|
||||
return new ArrayList<>();
|
||||
} else {
|
||||
meta = (JSONObject) firstData.get("_meta");
|
||||
}
|
||||
} else {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
} else {
|
||||
meta = (JSONObject) mappings.get("_meta");
|
||||
}
|
||||
if (!meta.containsKey("doris")) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
JSONObject dorisMeta = (JSONObject) meta.get("doris");
|
||||
return (List<String>) dorisMeta.get("array_fields");
|
||||
}
|
||||
|
||||
private static JSONObject getMapping(String indexMapping) {
|
||||
JSONObject jsonObject = (JSONObject) JSONValue.parse(indexMapping);
|
||||
@VisibleForTesting
|
||||
public static ObjectNode getMapping(String indexMapping) {
|
||||
ObjectNode jsonNodes = JsonUtil.parseObject(indexMapping);
|
||||
// If the indexName use alias takes the first mapping
|
||||
Iterator<String> keys = jsonObject.keySet().iterator();
|
||||
String docKey = keys.next();
|
||||
JSONObject docData = (JSONObject) jsonObject.get(docKey);
|
||||
return (JSONObject) docData.get("mappings");
|
||||
return (ObjectNode) jsonNodes.iterator().next().get("mappings");
|
||||
}
|
||||
|
||||
private static JSONObject getRootSchema(JSONObject mappings, String mappingType) {
|
||||
@VisibleForTesting
|
||||
public static ObjectNode getRootSchema(ObjectNode mappings, String mappingType) {
|
||||
// Type is null in the following three cases
|
||||
// 1. Equal 6.8.x and after
|
||||
// 2. Multi-catalog auto infer
|
||||
@ -160,10 +109,10 @@ public class EsUtil {
|
||||
if (mappingType == null) {
|
||||
// remove dynamic templates, for ES 7.x and 8.x
|
||||
checkNonPropertiesFields(mappings);
|
||||
String firstType = (String) mappings.keySet().iterator().next();
|
||||
String firstType = mappings.fieldNames().next();
|
||||
if (!"properties".equals(firstType)) {
|
||||
// If type is not passed in takes the first type.
|
||||
JSONObject firstData = (JSONObject) mappings.get(firstType);
|
||||
ObjectNode firstData = (ObjectNode) mappings.get(firstType);
|
||||
// check for ES 6.x and before
|
||||
checkNonPropertiesFields(firstData);
|
||||
return firstData;
|
||||
@ -171,8 +120,8 @@ public class EsUtil {
|
||||
// Equal 7.x and after
|
||||
return mappings;
|
||||
} else {
|
||||
if (mappings.containsKey(mappingType)) {
|
||||
JSONObject jsonData = (JSONObject) mappings.get(mappingType);
|
||||
if (mappings.has(mappingType)) {
|
||||
ObjectNode jsonData = (ObjectNode) mappings.get(mappingType);
|
||||
// check for ES 6.x and before
|
||||
checkNonPropertiesFields(jsonData);
|
||||
return jsonData;
|
||||
@ -187,7 +136,7 @@ public class EsUtil {
|
||||
*
|
||||
* @param mappings
|
||||
*/
|
||||
private static void checkNonPropertiesFields(JSONObject mappings) {
|
||||
private static void checkNonPropertiesFields(ObjectNode mappings) {
|
||||
// remove `_meta` field
|
||||
mappings.remove("_meta");
|
||||
// remove `dynamic_templates` field
|
||||
@ -199,12 +148,12 @@ public class EsUtil {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get mapping properties JSONObject.
|
||||
* Get mapping properties transform to ObjectNode.
|
||||
**/
|
||||
public static JSONObject getMappingProps(String sourceIndex, String indexMapping, String mappingType) {
|
||||
JSONObject mappings = getMapping(indexMapping);
|
||||
JSONObject rootSchema = getRootSchema(mappings, mappingType);
|
||||
JSONObject properties = (JSONObject) rootSchema.get("properties");
|
||||
public static ObjectNode getMappingProps(String sourceIndex, String indexMapping, String mappingType) {
|
||||
ObjectNode mappings = getMapping(indexMapping);
|
||||
ObjectNode rootSchema = getRootSchema(mappings, mappingType);
|
||||
ObjectNode properties = (ObjectNode) rootSchema.get("properties");
|
||||
if (properties == null) {
|
||||
throw new DorisEsException(
|
||||
"index[" + sourceIndex + "] type[" + mappingType + "] mapping not found for the ES Cluster");
|
||||
@ -212,7 +161,6 @@ public class EsUtil {
|
||||
return properties;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generate columns from ES Cluster.
|
||||
* Add mappingEsId config in es external catalog.
|
||||
@ -220,9 +168,14 @@ public class EsUtil {
|
||||
public static List<Column> genColumnsFromEs(EsRestClient client, String indexName, String mappingType,
|
||||
boolean mappingEsId) {
|
||||
String mapping = client.getMapping(indexName);
|
||||
JSONObject mappingProps = getMappingProps(indexName, mapping, mappingType);
|
||||
List<String> arrayFields = getArrayFields(mapping);
|
||||
Set<String> keys = (Set<String>) mappingProps.keySet();
|
||||
ObjectNode mappings = getMapping(mapping);
|
||||
ObjectNode rootSchema = getRootSchema(mappings, mappingType);
|
||||
return genColumnsFromEs(indexName, mappingType, rootSchema, mappingEsId);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static List<Column> genColumnsFromEs(String indexName, String mappingType, ObjectNode rootSchema,
|
||||
boolean mappingEsId) {
|
||||
List<Column> columns = new ArrayList<>();
|
||||
if (mappingEsId) {
|
||||
Column column = new Column();
|
||||
@ -233,68 +186,143 @@ public class EsUtil {
|
||||
column.setUniqueId(-1);
|
||||
columns.add(column);
|
||||
}
|
||||
for (String key : keys) {
|
||||
JSONObject field = (JSONObject) mappingProps.get(key);
|
||||
Type type;
|
||||
// Complex types are treating as String types for now.
|
||||
if (field.containsKey("type")) {
|
||||
type = toDorisType(field.get("type").toString());
|
||||
} else {
|
||||
type = Type.STRING;
|
||||
}
|
||||
Column column = new Column();
|
||||
column.setName(key);
|
||||
column.setIsKey(true);
|
||||
column.setIsAllowNull(true);
|
||||
column.setUniqueId(-1);
|
||||
if (arrayFields.contains(key)) {
|
||||
column.setType(ArrayType.create(type, true));
|
||||
} else {
|
||||
column.setType(type);
|
||||
ObjectNode mappingProps = (ObjectNode) rootSchema.get("properties");
|
||||
if (mappingProps == null) {
|
||||
throw new DorisEsException(
|
||||
"index[" + indexName + "] type[" + mappingType + "] mapping not found for the ES Cluster");
|
||||
}
|
||||
List<String> arrayFields = new ArrayList<>();
|
||||
JsonNode meta = mappingProps.get("_meta");
|
||||
if (meta != null) {
|
||||
JsonNode dorisMeta = meta.get("doris");
|
||||
if (dorisMeta != null) {
|
||||
arrayFields = dorisMeta.findValuesAsText("array_fields");
|
||||
}
|
||||
}
|
||||
Iterator<String> iterator = mappingProps.fieldNames();
|
||||
while (iterator.hasNext()) {
|
||||
String fieldName = iterator.next();
|
||||
ObjectNode fieldValue = (ObjectNode) mappingProps.get(fieldName);
|
||||
Column column = parseEsField(fieldName, fieldValue, arrayFields);
|
||||
columns.add(column);
|
||||
}
|
||||
return columns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transfer es type to doris type.
|
||||
**/
|
||||
public static Type toDorisType(String esType) {
|
||||
// reference https://www.elastic.co/guide/en/elasticsearch/reference/8.3/sql-data-types.html
|
||||
switch (esType) {
|
||||
case "null":
|
||||
return Type.NULL;
|
||||
case "boolean":
|
||||
return Type.BOOLEAN;
|
||||
case "byte":
|
||||
return Type.TINYINT;
|
||||
case "short":
|
||||
return Type.SMALLINT;
|
||||
case "integer":
|
||||
return Type.INT;
|
||||
case "long":
|
||||
return Type.BIGINT;
|
||||
case "unsigned_long":
|
||||
return Type.LARGEINT;
|
||||
case "float":
|
||||
case "half_float":
|
||||
return Type.FLOAT;
|
||||
case "double":
|
||||
case "scaled_float":
|
||||
return Type.DOUBLE;
|
||||
case "date":
|
||||
return ScalarType.createDateV2Type();
|
||||
case "keyword":
|
||||
case "text":
|
||||
case "ip":
|
||||
case "nested":
|
||||
case "object":
|
||||
return ScalarType.createStringType();
|
||||
default:
|
||||
return Type.UNSUPPORTED;
|
||||
private static Column parseEsField(String fieldName, ObjectNode fieldValue, List<String> arrayFields) {
|
||||
Column column = new Column();
|
||||
column.setName(fieldName);
|
||||
column.setIsKey(true);
|
||||
column.setIsAllowNull(true);
|
||||
column.setUniqueId(-1);
|
||||
Type type;
|
||||
// Complex types are treating as String types for now.
|
||||
if (fieldValue.has("type")) {
|
||||
String typeStr = fieldValue.get("type").asText();
|
||||
column.setComment("Elasticsearch type is " + typeStr);
|
||||
// reference https://www.elastic.co/guide/en/elasticsearch/reference/8.3/sql-data-types.html
|
||||
switch (typeStr) {
|
||||
case "null":
|
||||
type = Type.NULL;
|
||||
break;
|
||||
case "boolean":
|
||||
type = Type.BOOLEAN;
|
||||
break;
|
||||
case "byte":
|
||||
type = Type.TINYINT;
|
||||
break;
|
||||
case "short":
|
||||
type = Type.SMALLINT;
|
||||
break;
|
||||
case "integer":
|
||||
type = Type.INT;
|
||||
break;
|
||||
case "long":
|
||||
type = Type.BIGINT;
|
||||
break;
|
||||
case "unsigned_long":
|
||||
type = Type.LARGEINT;
|
||||
break;
|
||||
case "float":
|
||||
case "half_float":
|
||||
type = Type.FLOAT;
|
||||
break;
|
||||
case "double":
|
||||
case "scaled_float":
|
||||
type = Type.DOUBLE;
|
||||
break;
|
||||
case "date":
|
||||
type = parseEsDateType(column, fieldValue);
|
||||
break;
|
||||
case "keyword":
|
||||
case "text":
|
||||
case "ip":
|
||||
case "nested":
|
||||
case "object":
|
||||
type = ScalarType.createStringType();
|
||||
break;
|
||||
default:
|
||||
type = Type.UNSUPPORTED;
|
||||
}
|
||||
} else {
|
||||
type = Type.STRING;
|
||||
column.setComment("Elasticsearch no type");
|
||||
}
|
||||
if (arrayFields.contains(fieldName)) {
|
||||
column.setType(ArrayType.create(type, true));
|
||||
} else {
|
||||
column.setType(type);
|
||||
}
|
||||
return column;
|
||||
}
|
||||
|
||||
}
|
||||
private static final List<String> ALLOW_DATE_FORMATS = Lists.newArrayList("yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd",
|
||||
"epoch_millis");
|
||||
|
||||
/**
|
||||
* Parse es date to doris type by format
|
||||
**/
|
||||
private static Type parseEsDateType(Column column, ObjectNode field) {
|
||||
if (!field.has("format")) {
|
||||
// default format
|
||||
column.setComment("Elasticsearch type is date, no format");
|
||||
return ScalarType.createDatetimeV2Type(0);
|
||||
} else {
|
||||
String originFormat = field.get("format").asText();
|
||||
String[] formats = originFormat.split("\\|\\|");
|
||||
boolean dateTimeFlag = false;
|
||||
boolean dateFlag = false;
|
||||
boolean bigIntFlag = false;
|
||||
for (String format : formats) {
|
||||
// pre-check format
|
||||
if (!ALLOW_DATE_FORMATS.contains(format)) {
|
||||
column.setComment(
|
||||
"Elasticsearch type is date, format is " + format + " not support, use String type");
|
||||
return ScalarType.createStringType();
|
||||
}
|
||||
switch (format) {
|
||||
case "yyyy-MM-dd HH:mm:ss":
|
||||
dateTimeFlag = true;
|
||||
break;
|
||||
case "yyyy-MM-dd":
|
||||
dateFlag = true;
|
||||
break;
|
||||
case "epoch_millis":
|
||||
default:
|
||||
bigIntFlag = true;
|
||||
}
|
||||
}
|
||||
column.setComment("Elasticsearch type is date, format is " + originFormat);
|
||||
if (dateTimeFlag) {
|
||||
return ScalarType.createDatetimeV2Type(0);
|
||||
}
|
||||
if (dateFlag) {
|
||||
return ScalarType.createDateV2Type();
|
||||
}
|
||||
if (bigIntFlag) {
|
||||
return Type.BIGINT;
|
||||
}
|
||||
return ScalarType.createStringType();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,8 +20,11 @@ package org.apache.doris.external.elasticsearch;
|
||||
import org.apache.doris.catalog.Column;
|
||||
import org.apache.doris.catalog.EsTable;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.json.simple.JSONObject;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* Get index mapping from remote ES Cluster, and resolved `keyword` and `doc_values` field
|
||||
@ -55,69 +58,88 @@ public class MappingPhase implements SearchPhase {
|
||||
* @param indexMapping the return value of _mapping
|
||||
*/
|
||||
public static void resolveFields(SearchContext searchContext, String indexMapping) throws DorisEsException {
|
||||
JSONObject properties = EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping, searchContext.type());
|
||||
ObjectNode properties = EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping, searchContext.type());
|
||||
for (Column col : searchContext.columns()) {
|
||||
String colName = col.getName();
|
||||
// _id not exist mapping, but be can query it.
|
||||
if (!"_id".equals(colName)) {
|
||||
if (!properties.containsKey(colName)) {
|
||||
if (!properties.has(colName)) {
|
||||
throw new DorisEsException(
|
||||
"index[" + searchContext.sourceIndex() + "] mapping[" + indexMapping + "] not found "
|
||||
+ "column " + colName + " for the ES Cluster");
|
||||
}
|
||||
JSONObject fieldObject = (JSONObject) properties.get(colName);
|
||||
resolveKeywordFields(searchContext, fieldObject, colName);
|
||||
resolveDocValuesFields(searchContext, fieldObject, colName);
|
||||
ObjectNode fieldObject = (ObjectNode) properties.get(colName);
|
||||
if (!fieldObject.has("type")) {
|
||||
continue;
|
||||
}
|
||||
String fieldType = fieldObject.get("type").asText();
|
||||
resolveDateFields(searchContext, fieldObject, colName, fieldType);
|
||||
resolveKeywordFields(searchContext, fieldObject, colName, fieldType);
|
||||
resolveDocValuesFields(searchContext, fieldObject, colName, fieldType);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void resolveDateFields(SearchContext searchContext, ObjectNode fieldObject, String colName,
|
||||
String fieldType) {
|
||||
// Compat use default/strict_date_optional_time format date type, need transform datetime to
|
||||
if ("date".equals(fieldType)) {
|
||||
if (!fieldObject.has("format") || "strict_date_optional_time".equals(fieldObject.get("format").asText())) {
|
||||
searchContext.needCompatDateFields().add(colName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// get a field of keyword type in the fields
|
||||
private static void resolveKeywordFields(SearchContext searchContext, JSONObject fieldObject, String colName) {
|
||||
String fieldType = (String) fieldObject.get("type");
|
||||
private static void resolveKeywordFields(SearchContext searchContext, ObjectNode fieldObject, String colName,
|
||||
String fieldType) {
|
||||
// string-type field used keyword type to generate predicate
|
||||
// if text field type seen, we should use the `field` keyword type?
|
||||
if ("text".equals(fieldType)) {
|
||||
JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
|
||||
JsonNode fieldsObject = fieldObject.get("fields");
|
||||
if (fieldsObject != null) {
|
||||
for (Object key : fieldsObject.keySet()) {
|
||||
JSONObject innerTypeObject = (JSONObject) fieldsObject.get(key);
|
||||
Iterator<String> fieldNames = fieldsObject.fieldNames();
|
||||
while (fieldNames.hasNext()) {
|
||||
String fieldName = fieldNames.next();
|
||||
ObjectNode innerTypeObject = (ObjectNode) fieldsObject.get(fieldName);
|
||||
// just for text type
|
||||
if ("keyword".equals(innerTypeObject.get("type"))) {
|
||||
searchContext.fetchFieldsContext().put(colName, colName + "." + key);
|
||||
if ("keyword".equals(innerTypeObject.get("type").asText())) {
|
||||
searchContext.fetchFieldsContext().put(colName, colName + "." + fieldName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void resolveDocValuesFields(SearchContext searchContext, JSONObject fieldObject, String colName) {
|
||||
String fieldType = (String) fieldObject.get("type");
|
||||
private static void resolveDocValuesFields(SearchContext searchContext, ObjectNode fieldObject, String colName,
|
||||
String fieldType) {
|
||||
String docValueField = null;
|
||||
if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) {
|
||||
JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
|
||||
JsonNode fieldsObject = fieldObject.get("fields");
|
||||
if (fieldsObject != null) {
|
||||
for (Object key : fieldsObject.keySet()) {
|
||||
JSONObject innerTypeObject = (JSONObject) fieldsObject.get((String) key);
|
||||
if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains((String) innerTypeObject.get("type"))) {
|
||||
Iterator<String> fieldNames = fieldsObject.fieldNames();
|
||||
while (fieldNames.hasNext()) {
|
||||
String fieldName = fieldNames.next();
|
||||
ObjectNode innerTypeObject = (ObjectNode) fieldsObject.get(fieldName);
|
||||
if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(innerTypeObject.get("type").asText())) {
|
||||
continue;
|
||||
}
|
||||
if (innerTypeObject.containsKey("doc_values")) {
|
||||
boolean docValue = (Boolean) innerTypeObject.get("doc_values");
|
||||
if (innerTypeObject.has("doc_values")) {
|
||||
boolean docValue = innerTypeObject.get("doc_values").asBoolean();
|
||||
if (docValue) {
|
||||
docValueField = colName;
|
||||
}
|
||||
} else {
|
||||
// a : {c : {}} -> a -> a.c
|
||||
docValueField = colName + "." + key;
|
||||
docValueField = colName + "." + fieldName;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// set doc_value = false manually
|
||||
if (fieldObject.containsKey("doc_values")) {
|
||||
Boolean docValue = (Boolean) fieldObject.get("doc_values");
|
||||
if (fieldObject.has("doc_values")) {
|
||||
boolean docValue = fieldObject.get("doc_values").asBoolean();
|
||||
if (!docValue) {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -42,6 +42,9 @@ import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
import org.joda.time.format.ISODateTimeFormat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
@ -123,8 +126,12 @@ public final class QueryBuilders {
|
||||
.build());
|
||||
}
|
||||
|
||||
private static QueryBuilder parseBinaryPredicate(Expr expr, TExprOpcode opCode, String column) {
|
||||
private static QueryBuilder parseBinaryPredicate(Expr expr, TExprOpcode opCode, String column,
|
||||
boolean needDateCompat) {
|
||||
Object value = toDorisLiteral(expr.getChild(1));
|
||||
if (needDateCompat) {
|
||||
value = compatDefaultDate(value);
|
||||
}
|
||||
switch (opCode) {
|
||||
case EQ:
|
||||
case EQ_FOR_NULL:
|
||||
@ -180,10 +187,14 @@ public final class QueryBuilders {
|
||||
}
|
||||
}
|
||||
|
||||
private static QueryBuilder parseInPredicate(Expr expr, String column) {
|
||||
private static QueryBuilder parseInPredicate(Expr expr, String column, boolean needDateCompat) {
|
||||
InPredicate inPredicate = (InPredicate) expr;
|
||||
List<Object> values = inPredicate.getListChildren().stream().map(QueryBuilders::toDorisLiteral)
|
||||
.collect(Collectors.toList());
|
||||
List<Object> values = inPredicate.getListChildren().stream().map(v -> {
|
||||
if (needDateCompat) {
|
||||
return compatDefaultDate(v);
|
||||
}
|
||||
return toDorisLiteral(v);
|
||||
}).collect(Collectors.toList());
|
||||
if (inPredicate.isNotIn()) {
|
||||
return QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(column, values));
|
||||
}
|
||||
@ -237,10 +248,13 @@ public final class QueryBuilders {
|
||||
notPushDownList.add(expr);
|
||||
return null;
|
||||
}
|
||||
// Check whether the date type need compat, it must before keyword replace.
|
||||
List<String> needCompatDateFields = builderOptions.getNeedCompatDateFields();
|
||||
boolean needDateCompat = needCompatDateFields != null && needCompatDateFields.contains(column);
|
||||
// Replace col with col.keyword if mapping exist.
|
||||
column = fieldsContext.getOrDefault(column, column);
|
||||
if (expr instanceof BinaryPredicate) {
|
||||
return parseBinaryPredicate(expr, opCode, column);
|
||||
return parseBinaryPredicate(expr, opCode, column, needDateCompat);
|
||||
}
|
||||
if (expr instanceof IsNullPredicate) {
|
||||
return parseIsNullPredicate(expr, column);
|
||||
@ -254,7 +268,7 @@ public final class QueryBuilders {
|
||||
}
|
||||
}
|
||||
if (expr instanceof InPredicate) {
|
||||
return parseInPredicate(expr, column);
|
||||
return parseInPredicate(expr, column, needDateCompat);
|
||||
}
|
||||
if (expr instanceof FunctionCallExpr) {
|
||||
return parseFunctionCallExpr(expr);
|
||||
@ -262,6 +276,16 @@ public final class QueryBuilders {
|
||||
return null;
|
||||
}
|
||||
|
||||
private static final DateTimeFormatter dorisFmt = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
|
||||
private static final DateTimeFormatter esFmt = ISODateTimeFormat.dateTime();
|
||||
|
||||
private static Object compatDefaultDate(Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
return dorisFmt.parseDateTime(value.toString()).toString(esFmt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Expr trans to doris literal.
|
||||
**/
|
||||
@ -424,6 +448,8 @@ public final class QueryBuilders {
|
||||
public static class BuilderOptions {
|
||||
|
||||
private boolean likePushDown;
|
||||
|
||||
private List<String> needCompatDateFields;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -20,6 +20,7 @@ package org.apache.doris.external.elasticsearch;
|
||||
import org.apache.doris.catalog.Column;
|
||||
import org.apache.doris.catalog.EsTable;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@ -61,6 +62,8 @@ public class SearchContext {
|
||||
// fetch field value from doc_values, this is activated when `enable_docvalue_scan= true`
|
||||
private Map<String, String> docValueFieldsContext = Maps.newHashMap();
|
||||
|
||||
private List<String> needCompatDateFields = Lists.newArrayList();
|
||||
|
||||
// sourceIndex is the name of index when creating ES external table
|
||||
private final String sourceIndex;
|
||||
|
||||
@ -125,6 +128,10 @@ public class SearchContext {
|
||||
return docValueFieldsContext;
|
||||
}
|
||||
|
||||
public List<String> needCompatDateFields() {
|
||||
return needCompatDateFields;
|
||||
}
|
||||
|
||||
public void version(EsMajorVersion version) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
@ -363,7 +363,8 @@ public class EsScanNode extends ScanNode {
|
||||
List<Expr> notPushDownList = new ArrayList<>();
|
||||
for (Expr expr : conjuncts) {
|
||||
QueryBuilder queryBuilder = QueryBuilders.toEsDsl(expr, notPushDownList, fieldsContext,
|
||||
BuilderOptions.builder().likePushDown(table.isLikePushDown()).build());
|
||||
BuilderOptions.builder().likePushDown(table.isLikePushDown())
|
||||
.needCompatDateFields(table.needCompatDateFields()).build());
|
||||
if (queryBuilder != null) {
|
||||
hasFilter = true;
|
||||
boolQueryBuilder.must(queryBuilder);
|
||||
|
||||
@ -17,31 +17,14 @@
|
||||
|
||||
package org.apache.doris.external.elasticsearch;
|
||||
|
||||
import org.apache.doris.analysis.BinaryPredicate;
|
||||
import org.apache.doris.analysis.BinaryPredicate.Operator;
|
||||
import org.apache.doris.analysis.CastExpr;
|
||||
import org.apache.doris.analysis.CompoundPredicate;
|
||||
import org.apache.doris.analysis.Expr;
|
||||
import org.apache.doris.analysis.FloatLiteral;
|
||||
import org.apache.doris.analysis.FunctionCallExpr;
|
||||
import org.apache.doris.analysis.InPredicate;
|
||||
import org.apache.doris.analysis.IntLiteral;
|
||||
import org.apache.doris.analysis.IsNullPredicate;
|
||||
import org.apache.doris.analysis.LikePredicate;
|
||||
import org.apache.doris.analysis.SlotRef;
|
||||
import org.apache.doris.analysis.StringLiteral;
|
||||
import org.apache.doris.catalog.Column;
|
||||
import org.apache.doris.catalog.EsResource;
|
||||
import org.apache.doris.catalog.EsTable;
|
||||
import org.apache.doris.catalog.PrimitiveType;
|
||||
import org.apache.doris.catalog.Type;
|
||||
import org.apache.doris.common.ExceptionChecker;
|
||||
import org.apache.doris.external.elasticsearch.QueryBuilders.BuilderOptions;
|
||||
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import mockit.Expectations;
|
||||
import mockit.Injectable;
|
||||
import org.json.simple.JSONObject;
|
||||
import org.json.simple.JSONValue;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
@ -51,9 +34,7 @@ import org.junit.rules.ExpectedException;
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Test for es util.
|
||||
@ -62,16 +43,6 @@ public class EsUtilTest extends EsTestCase {
|
||||
|
||||
private List<Column> columns = new ArrayList<>();
|
||||
|
||||
private String jsonStr = "{\"settings\": {\n" + " \"index\": {\n" + " \"bpack\": {\n"
|
||||
+ " \"partition\": {\n" + " \"upperbound\": \"12\"\n"
|
||||
+ " }\n" + " },\n" + " \"number_of_shards\": \"5\",\n"
|
||||
+ " \"provided_name\": \"indexa\",\n"
|
||||
+ " \"creation_date\": \"1539328532060\",\n"
|
||||
+ " \"number_of_replicas\": \"1\",\n"
|
||||
+ " \"uuid\": \"plNNtKiiQ9-n6NpNskFzhQ\",\n" + " \"version\": {\n"
|
||||
+ " \"created\": \"5050099\"\n" + " }\n" + " }\n"
|
||||
+ " }}";
|
||||
|
||||
@Rule
|
||||
public ExpectedException expectedEx = ExpectedException.none();
|
||||
|
||||
@ -149,189 +120,31 @@ public class EsUtilTest extends EsTestCase {
|
||||
Assertions.assertFalse(searchContext.docValueFieldsContext().containsKey("k3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetJsonObject() {
|
||||
JSONObject json = (JSONObject) JSONValue.parse(jsonStr);
|
||||
JSONObject upperBoundSetting = EsUtil.getJsonObject(json, "settings.index.bpack.partition", 0);
|
||||
Assertions.assertTrue(upperBoundSetting.containsKey("upperbound"));
|
||||
Assertions.assertEquals("12", (String) upperBoundSetting.get("upperbound"));
|
||||
|
||||
JSONObject unExistKey = EsUtil.getJsonObject(json, "set", 0);
|
||||
Assertions.assertNull(unExistKey);
|
||||
|
||||
JSONObject singleKey = EsUtil.getJsonObject(json, "settings", 0);
|
||||
Assertions.assertTrue(singleKey.containsKey("index"));
|
||||
}
|
||||
|
||||
@Test(expected = ClassCastException.class)
|
||||
public void testGetJsonObjectWithException() {
|
||||
JSONObject json = (JSONObject) JSONValue.parse(jsonStr);
|
||||
// only support json object could not get string value directly from this api, exception will be threw
|
||||
EsUtil.getJsonObject(json, "settings.index.bpack.partition.upperbound", 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBinaryPredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
IntLiteral intLiteral = new IntLiteral(3);
|
||||
Expr eqExpr = new BinaryPredicate(Operator.EQ, k1, intLiteral);
|
||||
Expr neExpr = new BinaryPredicate(Operator.NE, k1, intLiteral);
|
||||
Expr leExpr = new BinaryPredicate(Operator.LE, k1, intLiteral);
|
||||
Expr geExpr = new BinaryPredicate(Operator.GE, k1, intLiteral);
|
||||
Expr ltExpr = new BinaryPredicate(Operator.LT, k1, intLiteral);
|
||||
Expr gtExpr = new BinaryPredicate(Operator.GT, k1, intLiteral);
|
||||
Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "k1"), new IntLiteral(3));
|
||||
Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr).toJson());
|
||||
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
|
||||
QueryBuilders.toEsDsl(neExpr).toJson());
|
||||
Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr).toJson());
|
||||
Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr).toJson());
|
||||
Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", QueryBuilders.toEsDsl(ltExpr).toJson());
|
||||
Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", QueryBuilders.toEsDsl(gtExpr).toJson());
|
||||
Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(efnExpr).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompoundPredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
IntLiteral intLiteral1 = new IntLiteral(3);
|
||||
SlotRef k2 = new SlotRef(null, "k2");
|
||||
IntLiteral intLiteral2 = new IntLiteral(5);
|
||||
BinaryPredicate binaryPredicate1 = new BinaryPredicate(Operator.EQ, k1, intLiteral1);
|
||||
BinaryPredicate binaryPredicate2 = new BinaryPredicate(Operator.GT, k2, intLiteral2);
|
||||
CompoundPredicate andPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate1,
|
||||
binaryPredicate2);
|
||||
CompoundPredicate orPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, binaryPredicate1,
|
||||
binaryPredicate2);
|
||||
CompoundPredicate notPredicate = new CompoundPredicate(CompoundPredicate.Operator.NOT, binaryPredicate1, null);
|
||||
Assertions.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
|
||||
QueryBuilders.toEsDsl(andPredicate).toJson());
|
||||
Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
|
||||
QueryBuilders.toEsDsl(orPredicate).toJson());
|
||||
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
|
||||
QueryBuilders.toEsDsl(notPredicate).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsNullPredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
IsNullPredicate isNullPredicate = new IsNullPredicate(k1, false);
|
||||
IsNullPredicate isNotNullPredicate = new IsNullPredicate(k1, true);
|
||||
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}",
|
||||
QueryBuilders.toEsDsl(isNullPredicate).toJson());
|
||||
Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", QueryBuilders.toEsDsl(isNotNullPredicate).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLikePredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
StringLiteral stringLiteral1 = new StringLiteral("%1%");
|
||||
StringLiteral stringLiteral2 = new StringLiteral("*1*");
|
||||
StringLiteral stringLiteral3 = new StringLiteral("1_2");
|
||||
LikePredicate likePredicate1 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral1);
|
||||
LikePredicate regexPredicate = new LikePredicate(LikePredicate.Operator.REGEXP, k1, stringLiteral2);
|
||||
LikePredicate likePredicate2 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral3);
|
||||
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(likePredicate1).toJson());
|
||||
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(regexPredicate).toJson());
|
||||
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", QueryBuilders.toEsDsl(likePredicate2).toJson());
|
||||
List<Expr> notPushDownList = new ArrayList<>();
|
||||
Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate2, notPushDownList, new HashMap<>(), BuilderOptions.builder().likePushDown(false).build()));
|
||||
Assertions.assertFalse(notPushDownList.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInPredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
IntLiteral intLiteral1 = new IntLiteral(3);
|
||||
IntLiteral intLiteral2 = new IntLiteral(5);
|
||||
List<Expr> intLiterals = new ArrayList<>();
|
||||
intLiterals.add(intLiteral1);
|
||||
intLiterals.add(intLiteral2);
|
||||
InPredicate isInPredicate = new InPredicate(k1, intLiterals, false);
|
||||
InPredicate isNotInPredicate = new InPredicate(k1, intLiterals, true);
|
||||
Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", QueryBuilders.toEsDsl(isInPredicate).toJson());
|
||||
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}",
|
||||
QueryBuilders.toEsDsl(isNotInPredicate).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFunctionCallConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
String str = "{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}";
|
||||
StringLiteral stringLiteral = new StringLiteral(str);
|
||||
List<Expr> exprs = new ArrayList<>();
|
||||
exprs.add(k1);
|
||||
exprs.add(stringLiteral);
|
||||
FunctionCallExpr functionCallExpr = new FunctionCallExpr("esquery", exprs);
|
||||
Assertions.assertEquals(str, QueryBuilders.toEsDsl(functionCallExpr).toJson());
|
||||
|
||||
SlotRef k2 = new SlotRef(null, "k2");
|
||||
IntLiteral intLiteral = new IntLiteral(5);
|
||||
BinaryPredicate binaryPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral);
|
||||
CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate,
|
||||
functionCallExpr);
|
||||
Assertions.assertEquals(
|
||||
"{\"bool\":{\"must\":[{\"term\":{\"k2\":5}},{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}]}}",
|
||||
QueryBuilders.toEsDsl(compoundPredicate).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCastConvertEsDsl() {
|
||||
FloatLiteral floatLiteral = new FloatLiteral(3.14);
|
||||
CastExpr castExpr = new CastExpr(Type.INT, floatLiteral);
|
||||
BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, castExpr, new IntLiteral(3));
|
||||
List<Expr> notPushDownList = new ArrayList<>();
|
||||
Map<String, String> fieldsContext = new HashMap<>();
|
||||
BuilderOptions builderOptions = BuilderOptions.builder()
|
||||
.likePushDown(Boolean.parseBoolean(EsResource.LIKE_PUSH_DOWN_DEFAULT_VALUE)).build();
|
||||
Assertions.assertNull(QueryBuilders.toEsDsl(castPredicate, notPushDownList, fieldsContext, builderOptions));
|
||||
Assertions.assertEquals(1, notPushDownList.size());
|
||||
|
||||
SlotRef k2 = new SlotRef(null, "k2");
|
||||
IntLiteral intLiteral = new IntLiteral(5);
|
||||
BinaryPredicate eqPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral);
|
||||
CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, castPredicate,
|
||||
eqPredicate);
|
||||
|
||||
QueryBuilders.toEsDsl(compoundPredicate, notPushDownList, fieldsContext, builderOptions);
|
||||
Assertions.assertEquals(3, notPushDownList.size());
|
||||
|
||||
SlotRef k3 = new SlotRef(null, "k3");
|
||||
k3.setType(Type.FLOAT);
|
||||
CastExpr castDoubleExpr = new CastExpr(Type.DOUBLE, k3);
|
||||
BinaryPredicate castDoublePredicate = new BinaryPredicate(Operator.GE, castDoubleExpr,
|
||||
new FloatLiteral(3.0, Type.DOUBLE));
|
||||
QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions);
|
||||
Assertions.assertEquals(3, notPushDownList.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEs6Mapping() throws IOException, URISyntaxException {
|
||||
JSONObject testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_aliases_mapping.json"),
|
||||
ObjectNode testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_aliases_mapping.json"),
|
||||
"doc");
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testAliases.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testAliases.toString());
|
||||
|
||||
JSONObject testAliasesNoType = EsUtil.getMappingProps("test",
|
||||
ObjectNode testAliasesNoType = EsUtil.getMappingProps("test",
|
||||
loadJsonFromFile("data/es/es6_aliases_mapping.json"), null);
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
|
||||
testAliasesNoType.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testAliasesNoType.toString());
|
||||
|
||||
JSONObject testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_index_mapping.json"),
|
||||
ObjectNode testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_index_mapping.json"),
|
||||
"doc");
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testIndex.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testIndex.toString());
|
||||
|
||||
JSONObject testDynamicTemplates = EsUtil.getMappingProps("test",
|
||||
ObjectNode testDynamicTemplates = EsUtil.getMappingProps("test",
|
||||
loadJsonFromFile("data/es/es6_dynamic_templates_mapping.json"), "doc");
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
|
||||
testDynamicTemplates.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testDynamicTemplates.toString());
|
||||
|
||||
expectedEx.expect(DorisEsException.class);
|
||||
expectedEx.expectMessage("Do not support index without explicit mapping.");
|
||||
@ -344,31 +157,29 @@ public class EsUtilTest extends EsTestCase {
|
||||
|
||||
@Test
|
||||
public void testEs7Mapping() throws IOException, URISyntaxException {
|
||||
JSONObject testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_aliases_mapping.json"),
|
||||
ObjectNode testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_aliases_mapping.json"),
|
||||
null);
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testAliases.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testAliases.toString());
|
||||
|
||||
JSONObject testAliasesErrorType = EsUtil.getMappingProps("test",
|
||||
ObjectNode testAliasesErrorType = EsUtil.getMappingProps("test",
|
||||
loadJsonFromFile("data/es/es7_aliases_mapping.json"), "doc");
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
|
||||
testAliasesErrorType.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testAliasesErrorType.toString());
|
||||
|
||||
JSONObject testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_index_mapping.json"),
|
||||
ObjectNode testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es7_index_mapping.json"),
|
||||
"doc");
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testIndex.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testIndex.toString());
|
||||
|
||||
JSONObject testDynamicTemplates = EsUtil.getMappingProps("test",
|
||||
ObjectNode testDynamicTemplates = EsUtil.getMappingProps("test",
|
||||
loadJsonFromFile("data/es/es7_dynamic_templates_mapping.json"), null);
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
|
||||
testDynamicTemplates.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testDynamicTemplates.toString());
|
||||
|
||||
expectedEx.expect(DorisEsException.class);
|
||||
expectedEx.expectMessage("Do not support index without explicit mapping.");
|
||||
@ -377,35 +188,66 @@ public class EsUtilTest extends EsTestCase {
|
||||
|
||||
@Test
|
||||
public void testEs8Mapping() throws IOException, URISyntaxException {
|
||||
JSONObject testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_aliases_mapping.json"),
|
||||
ObjectNode testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_aliases_mapping.json"),
|
||||
null);
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testAliases.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testAliases.toString());
|
||||
|
||||
JSONObject testAliasesErrorType = EsUtil.getMappingProps("test",
|
||||
ObjectNode testAliasesErrorType = EsUtil.getMappingProps("test",
|
||||
loadJsonFromFile("data/es/es8_aliases_mapping.json"), "doc");
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
|
||||
testAliasesErrorType.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testAliasesErrorType.toString());
|
||||
|
||||
JSONObject testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_index_mapping.json"),
|
||||
ObjectNode testIndex = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_index_mapping.json"),
|
||||
"doc");
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testIndex.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testIndex.toString());
|
||||
|
||||
JSONObject testDynamicTemplates = EsUtil.getMappingProps("test",
|
||||
ObjectNode testDynamicTemplates = EsUtil.getMappingProps("test",
|
||||
loadJsonFromFile("data/es/es8_dynamic_templates_mapping.json"), "doc");
|
||||
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
|
||||
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
|
||||
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
|
||||
testDynamicTemplates.toJSONString());
|
||||
Assertions.assertEquals(
|
||||
"{\"test1\":{\"type\":\"keyword\"},\"test2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"test3\":{\"type\":\"double\"},\"test4\":{\"type\":\"date\"}}",
|
||||
testDynamicTemplates.toString());
|
||||
|
||||
expectedEx.expect(DorisEsException.class);
|
||||
expectedEx.expectMessage("Do not support index without explicit mapping.");
|
||||
EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es8_only_dynamic_templates_mapping.json"), "doc");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDateType() throws IOException, URISyntaxException {
|
||||
ObjectNode testDateFormat = EsUtil.getRootSchema(
|
||||
EsUtil.getMapping(loadJsonFromFile("data/es/test_date_format.json")), null);
|
||||
List<Column> parseColumns = EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false);
|
||||
Assertions.assertEquals(8, parseColumns.size());
|
||||
for (Column column : parseColumns) {
|
||||
String name = column.getName();
|
||||
String type = column.getType().toSql();
|
||||
if ("test2".equals(name)) {
|
||||
Assertions.assertEquals("datetimev2(0)", type);
|
||||
}
|
||||
if ("test3".equals(name)) {
|
||||
Assertions.assertEquals("datetimev2(0)", type);
|
||||
}
|
||||
if ("test4".equals(name)) {
|
||||
Assertions.assertEquals("datev2", type);
|
||||
}
|
||||
if ("test5".equals(name)) {
|
||||
Assertions.assertEquals("datetimev2(0)", type);
|
||||
}
|
||||
if ("test6".equals(name)) {
|
||||
Assertions.assertEquals("datev2", type);
|
||||
}
|
||||
if ("test7".equals(name)) {
|
||||
Assertions.assertEquals("datetimev2(0)", type);
|
||||
}
|
||||
if ("test8".equals(name)) {
|
||||
Assertions.assertEquals("bigint(20)", type);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -17,17 +17,39 @@
|
||||
|
||||
package org.apache.doris.external.elasticsearch;
|
||||
|
||||
import org.apache.doris.analysis.BinaryPredicate;
|
||||
import org.apache.doris.analysis.BinaryPredicate.Operator;
|
||||
import org.apache.doris.analysis.CastExpr;
|
||||
import org.apache.doris.analysis.CompoundPredicate;
|
||||
import org.apache.doris.analysis.Expr;
|
||||
import org.apache.doris.analysis.FloatLiteral;
|
||||
import org.apache.doris.analysis.FunctionCallExpr;
|
||||
import org.apache.doris.analysis.InPredicate;
|
||||
import org.apache.doris.analysis.IntLiteral;
|
||||
import org.apache.doris.analysis.IsNullPredicate;
|
||||
import org.apache.doris.analysis.LikePredicate;
|
||||
import org.apache.doris.analysis.SlotRef;
|
||||
import org.apache.doris.analysis.StringLiteral;
|
||||
import org.apache.doris.catalog.Type;
|
||||
import org.apache.doris.external.elasticsearch.QueryBuilders.BuilderOptions;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Lists;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
/**
|
||||
@ -37,41 +59,170 @@ public class QueryBuildersTest {
|
||||
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
public void testBinaryPredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
IntLiteral intLiteral = new IntLiteral(3);
|
||||
Expr eqExpr = new BinaryPredicate(Operator.EQ, k1, intLiteral);
|
||||
Expr neExpr = new BinaryPredicate(Operator.NE, k1, intLiteral);
|
||||
Expr leExpr = new BinaryPredicate(Operator.LE, k1, intLiteral);
|
||||
Expr geExpr = new BinaryPredicate(Operator.GE, k1, intLiteral);
|
||||
Expr ltExpr = new BinaryPredicate(Operator.LT, k1, intLiteral);
|
||||
Expr gtExpr = new BinaryPredicate(Operator.GT, k1, intLiteral);
|
||||
Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "k1"), new IntLiteral(3));
|
||||
Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(eqExpr).toJson());
|
||||
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
|
||||
QueryBuilders.toEsDsl(neExpr).toJson());
|
||||
Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", QueryBuilders.toEsDsl(leExpr).toJson());
|
||||
Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", QueryBuilders.toEsDsl(geExpr).toJson());
|
||||
Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", QueryBuilders.toEsDsl(ltExpr).toJson());
|
||||
Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", QueryBuilders.toEsDsl(gtExpr).toJson());
|
||||
Assertions.assertEquals("{\"term\":{\"k1\":3}}", QueryBuilders.toEsDsl(efnExpr).toJson());
|
||||
|
||||
SlotRef k2 = new SlotRef(null, "k2");
|
||||
Expr dateTimeLiteral = new StringLiteral("2023-02-19 22:00:00");
|
||||
Expr dateTimeEqExpr = new BinaryPredicate(Operator.EQ, k2, dateTimeLiteral);
|
||||
Assertions.assertEquals("{\"term\":{\"k2\":\"2023-02-19T22:00:00.000+08:00\"}}",
|
||||
QueryBuilders.toEsDsl(dateTimeEqExpr, new ArrayList<>(), new HashMap<>(),
|
||||
BuilderOptions.builder().needCompatDateFields(Lists.newArrayList("k2")).build()).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompoundPredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
IntLiteral intLiteral1 = new IntLiteral(3);
|
||||
SlotRef k2 = new SlotRef(null, "k2");
|
||||
IntLiteral intLiteral2 = new IntLiteral(5);
|
||||
BinaryPredicate binaryPredicate1 = new BinaryPredicate(Operator.EQ, k1, intLiteral1);
|
||||
BinaryPredicate binaryPredicate2 = new BinaryPredicate(Operator.GT, k2, intLiteral2);
|
||||
CompoundPredicate andPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate1,
|
||||
binaryPredicate2);
|
||||
CompoundPredicate orPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, binaryPredicate1,
|
||||
binaryPredicate2);
|
||||
CompoundPredicate notPredicate = new CompoundPredicate(CompoundPredicate.Operator.NOT, binaryPredicate1, null);
|
||||
Assertions.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
|
||||
QueryBuilders.toEsDsl(andPredicate).toJson());
|
||||
Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
|
||||
QueryBuilders.toEsDsl(orPredicate).toJson());
|
||||
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
|
||||
QueryBuilders.toEsDsl(notPredicate).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsNullPredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
IsNullPredicate isNullPredicate = new IsNullPredicate(k1, false);
|
||||
IsNullPredicate isNotNullPredicate = new IsNullPredicate(k1, true);
|
||||
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}",
|
||||
QueryBuilders.toEsDsl(isNullPredicate).toJson());
|
||||
Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", QueryBuilders.toEsDsl(isNotNullPredicate).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLikePredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
StringLiteral stringLiteral1 = new StringLiteral("%1%");
|
||||
StringLiteral stringLiteral2 = new StringLiteral("*1*");
|
||||
StringLiteral stringLiteral3 = new StringLiteral("1_2");
|
||||
LikePredicate likePredicate1 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral1);
|
||||
LikePredicate regexPredicate = new LikePredicate(LikePredicate.Operator.REGEXP, k1, stringLiteral2);
|
||||
LikePredicate likePredicate2 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral3);
|
||||
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(likePredicate1).toJson());
|
||||
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", QueryBuilders.toEsDsl(regexPredicate).toJson());
|
||||
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", QueryBuilders.toEsDsl(likePredicate2).toJson());
|
||||
List<Expr> notPushDownList = new ArrayList<>();
|
||||
Assertions.assertNull(QueryBuilders.toEsDsl(likePredicate2, notPushDownList, new HashMap<>(),
|
||||
BuilderOptions.builder().likePushDown(false).build()));
|
||||
Assertions.assertFalse(notPushDownList.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInPredicateConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
IntLiteral intLiteral1 = new IntLiteral(3);
|
||||
IntLiteral intLiteral2 = new IntLiteral(5);
|
||||
List<Expr> intLiterals = new ArrayList<>();
|
||||
intLiterals.add(intLiteral1);
|
||||
intLiterals.add(intLiteral2);
|
||||
InPredicate isInPredicate = new InPredicate(k1, intLiterals, false);
|
||||
InPredicate isNotInPredicate = new InPredicate(k1, intLiterals, true);
|
||||
Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", QueryBuilders.toEsDsl(isInPredicate).toJson());
|
||||
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}",
|
||||
QueryBuilders.toEsDsl(isNotInPredicate).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFunctionCallConvertEsDsl() {
|
||||
SlotRef k1 = new SlotRef(null, "k1");
|
||||
String str = "{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}";
|
||||
StringLiteral stringLiteral = new StringLiteral(str);
|
||||
List<Expr> exprs = new ArrayList<>();
|
||||
exprs.add(k1);
|
||||
exprs.add(stringLiteral);
|
||||
FunctionCallExpr functionCallExpr = new FunctionCallExpr("esquery", exprs);
|
||||
Assertions.assertEquals(str, QueryBuilders.toEsDsl(functionCallExpr).toJson());
|
||||
|
||||
SlotRef k2 = new SlotRef(null, "k2");
|
||||
IntLiteral intLiteral = new IntLiteral(5);
|
||||
BinaryPredicate binaryPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral);
|
||||
CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate,
|
||||
functionCallExpr);
|
||||
Assertions.assertEquals(
|
||||
"{\"bool\":{\"must\":[{\"term\":{\"k2\":5}},{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}]}}",
|
||||
QueryBuilders.toEsDsl(compoundPredicate).toJson());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCastConvertEsDsl() {
|
||||
FloatLiteral floatLiteral = new FloatLiteral(3.14);
|
||||
CastExpr castExpr = new CastExpr(Type.INT, floatLiteral);
|
||||
BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, castExpr, new IntLiteral(3));
|
||||
List<Expr> notPushDownList = new ArrayList<>();
|
||||
Map<String, String> fieldsContext = new HashMap<>();
|
||||
BuilderOptions builderOptions = BuilderOptions.builder().likePushDown(true).build();
|
||||
Assertions.assertNull(QueryBuilders.toEsDsl(castPredicate, notPushDownList, fieldsContext, builderOptions));
|
||||
Assertions.assertEquals(1, notPushDownList.size());
|
||||
|
||||
SlotRef k2 = new SlotRef(null, "k2");
|
||||
IntLiteral intLiteral = new IntLiteral(5);
|
||||
BinaryPredicate eqPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral);
|
||||
CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, castPredicate,
|
||||
eqPredicate);
|
||||
|
||||
QueryBuilders.toEsDsl(compoundPredicate, notPushDownList, fieldsContext, builderOptions);
|
||||
Assertions.assertEquals(3, notPushDownList.size());
|
||||
|
||||
SlotRef k3 = new SlotRef(null, "k3");
|
||||
k3.setType(Type.FLOAT);
|
||||
CastExpr castDoubleExpr = new CastExpr(Type.DOUBLE, k3);
|
||||
BinaryPredicate castDoublePredicate = new BinaryPredicate(Operator.GE, castDoubleExpr,
|
||||
new FloatLiteral(3.0, Type.DOUBLE));
|
||||
QueryBuilders.toEsDsl(castDoublePredicate, notPushDownList, fieldsContext, builderOptions);
|
||||
Assertions.assertEquals(3, notPushDownList.size());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTermQuery() throws Exception {
|
||||
Assert.assertEquals("{\"term\":{\"k\":\"aaaa\"}}",
|
||||
toJson(QueryBuilders.termQuery("k", "aaaa")));
|
||||
Assert.assertEquals("{\"term\":{\"aaaa\":\"k\"}}",
|
||||
toJson(QueryBuilders.termQuery("aaaa", "k")));
|
||||
Assert.assertEquals("{\"term\":{\"k\":0}}",
|
||||
toJson(QueryBuilders.termQuery("k", (byte) 0)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":123}}",
|
||||
toJson(QueryBuilders.termQuery("k", (long) 123)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":41}}",
|
||||
toJson(QueryBuilders.termQuery("k", (short) 41)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":128}}",
|
||||
toJson(QueryBuilders.termQuery("k", 128)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":42.42}}",
|
||||
toJson(QueryBuilders.termQuery("k", 42.42D)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":1.1}}",
|
||||
toJson(QueryBuilders.termQuery("k", 1.1F)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":1}}",
|
||||
toJson(QueryBuilders.termQuery("k", new BigDecimal(1))));
|
||||
Assert.assertEquals("{\"term\":{\"k\":121}}",
|
||||
toJson(QueryBuilders.termQuery("k", new BigInteger("121"))));
|
||||
Assert.assertEquals("{\"term\":{\"k\":true}}",
|
||||
toJson(QueryBuilders.termQuery("k", new AtomicBoolean(true))));
|
||||
Assert.assertEquals("{\"term\":{\"k\":\"aaaa\"}}", toJson(QueryBuilders.termQuery("k", "aaaa")));
|
||||
Assert.assertEquals("{\"term\":{\"aaaa\":\"k\"}}", toJson(QueryBuilders.termQuery("aaaa", "k")));
|
||||
Assert.assertEquals("{\"term\":{\"k\":0}}", toJson(QueryBuilders.termQuery("k", (byte) 0)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":123}}", toJson(QueryBuilders.termQuery("k", (long) 123)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":41}}", toJson(QueryBuilders.termQuery("k", (short) 41)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":128}}", toJson(QueryBuilders.termQuery("k", 128)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":42.42}}", toJson(QueryBuilders.termQuery("k", 42.42D)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":1.1}}", toJson(QueryBuilders.termQuery("k", 1.1F)));
|
||||
Assert.assertEquals("{\"term\":{\"k\":1}}", toJson(QueryBuilders.termQuery("k", new BigDecimal(1))));
|
||||
Assert.assertEquals("{\"term\":{\"k\":121}}", toJson(QueryBuilders.termQuery("k", new BigInteger("121"))));
|
||||
Assert.assertEquals("{\"term\":{\"k\":true}}", toJson(QueryBuilders.termQuery("k", new AtomicBoolean(true))));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTermsQuery() throws Exception {
|
||||
|
||||
Assert.assertEquals("{\"terms\":{\"k\":[]}}",
|
||||
toJson(QueryBuilders.termsQuery("k", Collections.emptySet())));
|
||||
Assert.assertEquals("{\"terms\":{\"k\":[]}}", toJson(QueryBuilders.termsQuery("k", Collections.emptySet())));
|
||||
|
||||
Assert.assertEquals("{\"terms\":{\"k\":[0]}}",
|
||||
toJson(QueryBuilders.termsQuery("k", Collections.singleton(0))));
|
||||
Assert.assertEquals("{\"terms\":{\"k\":[0]}}", toJson(QueryBuilders.termsQuery("k", Collections.singleton(0))));
|
||||
|
||||
Assert.assertEquals("{\"terms\":{\"k\":[\"aaa\"]}}",
|
||||
toJson(QueryBuilders.termsQuery("k", Collections.singleton("aaa"))));
|
||||
@ -91,45 +242,44 @@ public class QueryBuildersTest {
|
||||
|
||||
@Test
|
||||
public void testBoolQuery() throws Exception {
|
||||
QueryBuilders.QueryBuilder q1 = QueryBuilders.boolQuery()
|
||||
.must(QueryBuilders.termQuery("k", "aaa"));
|
||||
QueryBuilders.QueryBuilder q1 = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("k", "aaa"));
|
||||
|
||||
Assert.assertEquals("{\"bool\":{\"must\":{\"term\":{\"k\":\"aaa\"}}}}",
|
||||
toJson(q1));
|
||||
Assert.assertEquals("{\"bool\":{\"must\":{\"term\":{\"k\":\"aaa\"}}}}", toJson(q1));
|
||||
|
||||
QueryBuilders.QueryBuilder q2 = QueryBuilders.boolQuery()
|
||||
.must(QueryBuilders.termQuery("k1", "aaa")).must(QueryBuilders.termQuery("k2", "bbb"));
|
||||
QueryBuilders.QueryBuilder q2 = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("k1", "aaa"))
|
||||
.must(QueryBuilders.termQuery("k2", "bbb"));
|
||||
|
||||
Assert.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":\"aaa\"}},{\"term\":{\"k2\":\"bbb\"}}]}}",
|
||||
toJson(q2));
|
||||
|
||||
QueryBuilders.QueryBuilder q3 = QueryBuilders.boolQuery()
|
||||
.mustNot(QueryBuilders.termQuery("k", "fff"));
|
||||
QueryBuilders.QueryBuilder q3 = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("k", "fff"));
|
||||
|
||||
Assert.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k\":\"fff\"}}}}",
|
||||
toJson(q3));
|
||||
Assert.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k\":\"fff\"}}}}", toJson(q3));
|
||||
|
||||
QueryBuilders.QueryBuilder q4 = QueryBuilders.rangeQuery("k1").lt(200).gt(-200);
|
||||
QueryBuilders.QueryBuilder q5 = QueryBuilders.termsQuery("k2", Arrays.asList("aaa", "bbb", "ccc"));
|
||||
QueryBuilders.QueryBuilder q6 = QueryBuilders.boolQuery().must(q4).should(q5);
|
||||
Assert.assertEquals("{\"bool\":{\"must\":{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},\"should\":{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}}}", toJson(q6));
|
||||
Assert.assertEquals("{\"bool\":{\"filter\":[{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}]}}", toJson(QueryBuilders.boolQuery().filter(q4).filter(q5)));
|
||||
Assert.assertEquals("{\"bool\":{\"filter\":{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},\"must_not\":{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}}}", toJson(QueryBuilders.boolQuery().filter(q4).mustNot(q5)));
|
||||
Assert.assertEquals(
|
||||
"{\"bool\":{\"must\":{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},\"should\":{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}}}",
|
||||
toJson(q6));
|
||||
Assert.assertEquals(
|
||||
"{\"bool\":{\"filter\":[{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}]}}",
|
||||
toJson(QueryBuilders.boolQuery().filter(q4).filter(q5)));
|
||||
Assert.assertEquals(
|
||||
"{\"bool\":{\"filter\":{\"range\":{\"k1\":{\"gt\":-200,\"lt\":200}}},\"must_not\":{\"terms\":{\"k2\":[\"aaa\",\"bbb\",\"ccc\"]}}}}",
|
||||
toJson(QueryBuilders.boolQuery().filter(q4).mustNot(q5)));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExistsQuery() throws Exception {
|
||||
Assert.assertEquals("{\"exists\":{\"field\":\"k\"}}",
|
||||
toJson(QueryBuilders.existsQuery("k")));
|
||||
Assert.assertEquals("{\"exists\":{\"field\":\"k\"}}", toJson(QueryBuilders.existsQuery("k")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRangeQuery() throws Exception {
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"lt\":123}}}",
|
||||
toJson(QueryBuilders.rangeQuery("k").lt(123)));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"gt\":123}}}",
|
||||
toJson(QueryBuilders.rangeQuery("k").gt(123)));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"lt\":123}}}", toJson(QueryBuilders.rangeQuery("k").lt(123)));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"gt\":123}}}", toJson(QueryBuilders.rangeQuery("k").gt(123)));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"gte\":12345678}}}",
|
||||
toJson(QueryBuilders.rangeQuery("k").gte(12345678)));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"lte\":12345678}}}",
|
||||
@ -142,22 +292,19 @@ public class QueryBuildersTest {
|
||||
toJson(QueryBuilders.rangeQuery("k").gt(6789.33f).lte(9999.99f)));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"gte\":1,\"lte\":\"zzz\"}}}",
|
||||
toJson(QueryBuilders.rangeQuery("k").gte(1).lte("zzz")));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"gte\":\"zzz\"}}}",
|
||||
toJson(QueryBuilders.rangeQuery("k").gte("zzz")));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"gte\":\"zzz\"}}}", toJson(QueryBuilders.rangeQuery("k").gte("zzz")));
|
||||
Assert.assertEquals("{\"range\":{\"k\":{\"gt\":\"aaa\",\"lt\":\"zzz\"}}}",
|
||||
toJson(QueryBuilders.rangeQuery("k").gt("aaa").lt("zzz")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMatchAllQuery() throws IOException {
|
||||
Assert.assertEquals("{\"match_all\":{}}",
|
||||
toJson(QueryBuilders.matchAllQuery()));
|
||||
Assert.assertEquals("{\"match_all\":{}}", toJson(QueryBuilders.matchAllQuery()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWildCardQuery() throws IOException {
|
||||
Assert.assertEquals("{\"wildcard\":{\"k1\":\"?aa*\"}}",
|
||||
toJson(QueryBuilders.wildcardQuery("k1", "?aa*")));
|
||||
Assert.assertEquals("{\"wildcard\":{\"k1\":\"?aa*\"}}", toJson(QueryBuilders.wildcardQuery("k1", "?aa*")));
|
||||
}
|
||||
|
||||
private String toJson(QueryBuilders.QueryBuilder builder) throws IOException {
|
||||
|
||||
38
fe/fe-core/src/test/resources/data/es/test_date_format.json
Normal file
38
fe/fe-core/src/test/resources/data/es/test_date_format.json
Normal file
@ -0,0 +1,38 @@
|
||||
{
|
||||
"test_date_format": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"test1": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"test2": {
|
||||
"type": "date"
|
||||
},
|
||||
"test3": {
|
||||
"type": "date",
|
||||
"format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
|
||||
},
|
||||
"test4": {
|
||||
"type": "date",
|
||||
"format": "yyyy-MM-dd||epoch_millis"
|
||||
},
|
||||
"test5": {
|
||||
"type": "date",
|
||||
"format": "yyyy-MM-dd HH:mm:ss||epoch_millis"
|
||||
},
|
||||
"test6": {
|
||||
"type": "date",
|
||||
"format": "yyyy-MM-dd"
|
||||
},
|
||||
"test7": {
|
||||
"type": "date",
|
||||
"format": "yyyy-MM-dd HH:mm:ss"
|
||||
},
|
||||
"test8": {
|
||||
"type": "date",
|
||||
"format": "epoch_millis"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user