database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](doe) Fix some bug (#11594)

Browse Source
This commit is contained in:
Stalary
2022-08-10 21:00:05 +08:00
committed by GitHub
parent 290f0400d3
commit d8427037be
8 changed files with 127 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java
View File

@ -63,7 +63,7 @@ public class EsShardProcDir implements ProcDirInterface {
for (EsShardRouting esShardRouting : shardRoutings) {
List<Comparable> shardInfo = new ArrayList<Comparable>();
shardInfo.add(shardId);
shardInfo.add(esShardRouting.getAddress().toString());
shardInfo.add(esShardRouting.getHttpAddress().toString());
shardInfo.add(esShardRouting.isPrimary());
shardInfos.add(shardInfo);
}

31
fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
View File

@ -98,6 +98,21 @@ public class EsExternalDataSource extends ExternalDataSource {
throw new DdlException("Hosts of ES table is null.");
}
nodes = properties.get(PROP_HOSTS).trim().split(",");
// check protocol
for (String seed : nodes) {
if (!seed.startsWith("http")) {
throw new DdlException("the protocol must be used");
}
if (properties.containsKey(PROP_SSL)) {
enableSsl = EsUtil.getBoolean(properties, PROP_SSL);
if (enableSsl && seed.startsWith("http://")) {
throw new DdlException("if ssl_enabled is true, the https protocol must be used");
}
if (!enableSsl && seed.startsWith("https://")) {
throw new DdlException("if ssl_enabled is false, the http protocol must be used");
}
}
}
if (StringUtils.isNotBlank(properties.get(PROP_USERNAME))) {
username = properties.get(PROP_USERNAME).trim();
@ -119,18 +134,6 @@ public class EsExternalDataSource extends ExternalDataSource {
enableNodesDiscovery = EsUtil.getBoolean(properties, PROP_NODES_DISCOVERY);
}
if (properties.containsKey(PROP_SSL)) {
enableSsl = EsUtil.getBoolean(properties, PROP_SSL);
// check protocol
for (String seed : nodes) {
if (enableSsl && seed.startsWith("http://")) {
throw new DdlException("if ssl_enabled is true, the https protocol must be used");
}
if (!enableSsl && seed.startsWith("https://")) {
throw new DdlException("if ssl_enabled is false, the http protocol must be used");
}
}
}
}
/**
@ -155,7 +158,7 @@ public class EsExternalDataSource extends ExternalDataSource {
this.esRestClient = new EsRestClient(this.nodes, this.username, this.password, this.enableSsl);
long defaultDbId = Env.getCurrentEnv().getNextId();
dbNameToId.put(DEFAULT_DB, defaultDbId);
idToDb.put(defaultDbId, new EsExternalDatabase(this, defaultDbId, "default"));
idToDb.put(defaultDbId, new EsExternalDatabase(this, defaultDbId, DEFAULT_DB));
}
@Override
@ -177,7 +180,7 @@ public class EsExternalDataSource extends ExternalDataSource {
if (!dbNameToId.containsKey(realDbName)) {
return null;
}
return new EsExternalDatabase(this, dbNameToId.get(realDbName), realDbName);
return idToDb.get(dbNameToId.get(realDbName));
}
@Override

5
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java vendored
View File

@ -189,12 +189,11 @@ public class EsRestClient {
**/
public List<String> listTable() {
List<String> indices = getIndices().stream().distinct().collect(Collectors.toList());
getAliases().entrySet().stream().filter(e -> indices.contains(e.getKey()))
.flatMap(e -> e.getValue().stream()).distinct().forEach(indices::add);
getAliases().entrySet().stream().filter(e -> indices.contains(e.getKey())).flatMap(e -> e.getValue().stream())
.distinct().forEach(indices::add);
return indices;
}
/**
* Get Shard location.
**/

22
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java vendored
View File

@ -54,12 +54,11 @@ public class EsShardPartitions {
/**
* Parse shardRoutings from the json
*
* @param indexName indexName(alias or really name)
* @param indexName indexName(alias or really name)
* @param searchShards the return value of _search_shards
* @return shardRoutings is used for searching
*/
public static EsShardPartitions findShardPartitions(String indexName, String searchShards) throws DorisEsException {
EsShardPartitions partitions = new EsShardPartitions(indexName);
JSONObject jsonObject = (JSONObject) JSONValue.parse(searchShards);
JSONArray shards = (JSONArray) jsonObject.get("shards");
@ -67,23 +66,18 @@ public class EsShardPartitions {
for (int i = 0; i < size; i++) {
List<EsShardRouting> singleShardRouting = Lists.newArrayList();
JSONArray shardsArray = (JSONArray) shards.get(i);
int arraySize = shardsArray.size();
for (int j = 0; j < arraySize; j++) {
JSONObject indexShard = (JSONObject) shardsArray.get(j);
for (Object o : shardsArray) {
JSONObject indexShard = (JSONObject) o;
String shardState = (String) indexShard.get("state");
if ("STARTED".equalsIgnoreCase(shardState) || "RELOCATING".equalsIgnoreCase(shardState)) {
try {
singleShardRouting.add(
EsShardRouting.newSearchShard(
(String) indexShard.get("index"),
((Long) indexShard.get("shard")).intValue(),
(Boolean) indexShard.get("primary"),
(String) indexShard.get("node"),
(JSONObject) jsonObject.get("nodes")));
singleShardRouting.add(new EsShardRouting((String) indexShard.get("index"),
((Long) indexShard.get("shard")).intValue(), (Boolean) indexShard.get("primary"),
(String) indexShard.get("node")));
} catch (Exception e) {
LOG.error("fetch index [{}] shard partitions failure", indexName, e);
throw new DorisEsException("fetch [" + indexName
+ "] shard partitions failure [" + e.getMessage() + "]");
throw new DorisEsException(
"fetch [" + indexName + "] shard partitions failure [" + e.getMessage() + "]");
}
}
}

36
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java vendored
View File

@ -19,41 +19,21 @@ package org.apache.doris.external.elasticsearch;
import org.apache.doris.thrift.TNetworkAddress;
import com.google.common.base.Strings;
import org.json.simple.JSONObject;
public class EsShardRouting {
private final String indexName;
private final int shardId;
private final boolean isPrimary;
private final TNetworkAddress address;
private TNetworkAddress httpAddress;
private final String nodeId;
public EsShardRouting(String indexName, int shardId, boolean isPrimary, TNetworkAddress address, String nodeId) {
public EsShardRouting(String indexName, int shardId, boolean isPrimary, String nodeId) {
this.indexName = indexName;
this.shardId = shardId;
this.isPrimary = isPrimary;
this.address = address;
this.nodeId = nodeId;
}
public static EsShardRouting newSearchShard(String indexName, int shardId, boolean isPrimary,
String nodeId, JSONObject nodesMap) {
JSONObject nodeInfo = (JSONObject) nodesMap.get(nodeId);
String[] transportAddr = ((String) nodeInfo.get("transport_address")).split(":");
// get thrift port from node info
String thriftPort = (String) ((JSONObject) nodeInfo.get("attributes")).get("thrift_port");
// In http transport mode, should ignore thrift_port, set address to null
TNetworkAddress addr = null;
if (!Strings.isNullOrEmpty(thriftPort)) {
addr = new TNetworkAddress(transportAddr[0], Integer.parseInt(thriftPort));
}
return new EsShardRouting(indexName, shardId, isPrimary, addr, nodeId);
}
public int getShardId() {
return shardId;
}
@ -62,10 +42,6 @@ public class EsShardRouting {
return isPrimary;
}
public TNetworkAddress getAddress() {
return address;
}
public String getIndexName() {
return indexName;
}
@ -84,13 +60,7 @@ public class EsShardRouting {
@Override
public String toString() {
return "EsShardRouting{"
+ "indexName='" + indexName + '\''
+ ", shardId=" + shardId
+ ", isPrimary=" + isPrimary
+ ", address=" + address
+ ", httpAddress=" + httpAddress
+ ", nodeId='" + nodeId + '\''
+ '}';
return "EsShardRouting{" + "indexName='" + indexName + '\'' + ", shardId=" + shardId + ", isPrimary="
+ isPrimary + ", httpAddress=" + httpAddress + ", nodeId='" + nodeId + '\'' + '}';
}
}

48
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java vendored
View File

@ -19,8 +19,8 @@ package org.apache.doris.external.elasticsearch;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.BoolLiteral;
import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.analysis.DecimalLiteral;
import org.apache.doris.analysis.DistributionDesc;
import org.apache.doris.analysis.Expr;
@ -35,7 +35,6 @@ import org.apache.doris.analysis.LikePredicate.Operator;
import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.RangePartitionDesc;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Type;
@ -192,27 +191,39 @@ public class EsUtil {
return properties;
}
private static QueryBuilder toCompoundEsDsl(Expr expr) {
private static QueryBuilder toCompoundEsDsl(Expr expr, List<Expr> notPushDownList) {
CompoundPredicate compoundPredicate = (CompoundPredicate) expr;
switch (compoundPredicate.getOp()) {
case AND: {
QueryBuilder left = toEsDsl(compoundPredicate.getChild(0));
QueryBuilder right = toEsDsl(compoundPredicate.getChild(1));
QueryBuilder left = toEsDsl(compoundPredicate.getChild(0), notPushDownList);
QueryBuilder right = toEsDsl(compoundPredicate.getChild(1), notPushDownList);
if (left != null && right != null) {
return QueryBuilders.boolQuery().must(left).must(right);
}
return null;
}
case OR: {
QueryBuilder left = toEsDsl(compoundPredicate.getChild(0));
QueryBuilder right = toEsDsl(compoundPredicate.getChild(1));
int beforeSize = notPushDownList.size();
QueryBuilder left = toEsDsl(compoundPredicate.getChild(0), notPushDownList);
QueryBuilder right = toEsDsl(compoundPredicate.getChild(1), notPushDownList);
int afterSize = notPushDownList.size();
if (left != null && right != null) {
return QueryBuilders.boolQuery().should(left).should(right);
}
// One 'or' association cannot be pushed down and the other cannot be pushed down
if (afterSize > beforeSize) {
if (left != null) {
// add right if right don't pushdown
notPushDownList.add(compoundPredicate.getChild(0));
} else if (right != null) {
// add left if left don't pushdown
notPushDownList.add(compoundPredicate.getChild(1));
}
}
return null;
}
case NOT: {
QueryBuilder child = toEsDsl(compoundPredicate.getChild(0));
QueryBuilder child = toEsDsl(compoundPredicate.getChild(0), notPushDownList);
if (child != null) {
return QueryBuilders.boolQuery().mustNot(child);
}
@ -223,18 +234,27 @@ public class EsUtil {
}
}
public static QueryBuilder toEsDsl(Expr expr) {
return toEsDsl(expr, new ArrayList<>());
}
/**
* Doris expr to es dsl.
**/
public static QueryBuilder toEsDsl(Expr expr) {
public static QueryBuilder toEsDsl(Expr expr, List<Expr> notPushDownList) {
if (expr == null) {
return null;
}
// CompoundPredicate, `between` also converted to CompoundPredicate.
if (expr instanceof CompoundPredicate) {
return toCompoundEsDsl(expr);
return toCompoundEsDsl(expr, notPushDownList);
}
TExprOpcode opCode = expr.getOpcode();
// Cast can not pushdown
if (expr.getChild(0) instanceof CastExpr || expr.getChild(1) instanceof CastExpr) {
notPushDownList.add(expr);
return null;
}
String column = ((SlotRef) expr.getChild(0)).getColumnName();
if (expr instanceof BinaryPredicate) {
Object value = toDorisLiteral(expr.getChild(1));
@ -386,9 +406,6 @@ public class EsUtil {
if (expr instanceof BoolLiteral) {
BoolLiteral boolLiteral = (BoolLiteral) expr;
return boolLiteral.getValue();
} else if (expr instanceof DateLiteral) {
DateLiteral dateLiteral = (DateLiteral) expr;
return dateLiteral.getStringValue();
} else if (expr instanceof DecimalLiteral) {
DecimalLiteral decimalLiteral = (DecimalLiteral) expr;
return decimalLiteral.getValue();
@ -401,11 +418,8 @@ public class EsUtil {
} else if (expr instanceof LargeIntLiteral) {
LargeIntLiteral largeIntLiteral = (LargeIntLiteral) expr;
return largeIntLiteral.getLongValue();
} else if (expr instanceof StringLiteral) {
StringLiteral stringLiteral = (StringLiteral) expr;
return stringLiteral.getStringValue();
}
return null;
return expr.getStringValue();
}
}

13
fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
View File

@ -338,15 +338,11 @@ public class EsScanNode extends ScanNode {
}
if (!conjuncts.isEmpty()) {
output.append(prefix).append("PREDICATES: ").append(getExplainString(conjuncts)).append("\n");
// reserved for later using: LOCAL_PREDICATES is processed by Doris EsScanNode
output.append(prefix).append("LOCAL_PREDICATES: ").append(" ").append("\n");
// reserved for later using: REMOTE_PREDICATES is processed by remote ES Cluster
output.append(prefix).append("REMOTE_PREDICATES: ").append(" ").append("\n");
output.append(prefix).append("LOCAL_PREDICATES: ").append(getExplainString(conjuncts)).append("\n");
buildQuery();
output.append(prefix).append("ES_QUERY_DSL: ").append(queryBuilder.toJson()).append("\n");
output.append(prefix).append("REMOTE_PREDICATES: ").append(queryBuilder.toJson()).append("\n");
} else {
output.append(prefix).append("ES_QUERY_DSL: ").append("{\"match_all\": {}}").append("\n");
output.append(prefix).append("REMOTE_PREDICATES: ").append("{\"match_all\": {}}").append("\n");
}
String indexName = table.getIndexName();
String typeName = table.getMappingType();
@ -360,8 +356,9 @@ public class EsScanNode extends ScanNode {
} else {
boolean hasFilter = false;
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
List<Expr> notPushDownList = new ArrayList<>();
for (Expr expr : conjuncts) {
QueryBuilder queryBuilder = EsUtil.toEsDsl(expr);
QueryBuilder queryBuilder = EsUtil.toEsDsl(expr, notPushDownList);
if (queryBuilder != null) {
hasFilter = true;
boolQueryBuilder.must(queryBuilder);

97
fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java vendored
View File

@ -19,8 +19,10 @@ package org.apache.doris.external.elasticsearch;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.BinaryPredicate.Operator;
import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FloatLiteral;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.IntLiteral;
@ -28,6 +30,7 @@ import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.analysis.TypeDef;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.EsTable;
import org.apache.doris.catalog.PrimitiveType;
@ -37,7 +40,6 @@ import mockit.Expectations;
import mockit.Injectable;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
@ -84,19 +86,19 @@ public class EsUtilTest extends EsTestCase {
EsTable esTableBefore7X = fakeEsTable("fake", "test", "doc", columns);
SearchContext searchContext = new SearchContext(esTableBefore7X);
MappingPhase.resolveFields(searchContext, loadJsonFromFile("data/es/test_index_mapping.json"));
Assert.assertEquals("k3.keyword", searchContext.fetchFieldsContext().get("k3"));
Assert.assertEquals("k3.keyword", searchContext.docValueFieldsContext().get("k3"));
Assert.assertEquals("k1", searchContext.docValueFieldsContext().get("k1"));
Assert.assertEquals("k2", searchContext.docValueFieldsContext().get("k2"));
Assertions.assertEquals("k3.keyword", searchContext.fetchFieldsContext().get("k3"));
Assertions.assertEquals("k3.keyword", searchContext.docValueFieldsContext().get("k3"));
Assertions.assertEquals("k1", searchContext.docValueFieldsContext().get("k1"));
Assertions.assertEquals("k2", searchContext.docValueFieldsContext().get("k2"));
// ES version >= 7.0
EsTable esTableAfter7X = fakeEsTable("fake", "test", "_doc", columns);
SearchContext searchContext1 = new SearchContext(esTableAfter7X);
MappingPhase.resolveFields(searchContext1, loadJsonFromFile("data/es/test_index_mapping_after_7x.json"));
Assert.assertEquals("k3.keyword", searchContext1.fetchFieldsContext().get("k3"));
Assert.assertEquals("k3.keyword", searchContext1.docValueFieldsContext().get("k3"));
Assert.assertEquals("k1", searchContext1.docValueFieldsContext().get("k1"));
Assert.assertEquals("k2", searchContext1.docValueFieldsContext().get("k2"));
Assertions.assertEquals("k3.keyword", searchContext1.fetchFieldsContext().get("k3"));
Assertions.assertEquals("k3.keyword", searchContext1.docValueFieldsContext().get("k3"));
Assertions.assertEquals("k1", searchContext1.docValueFieldsContext().get("k1"));
Assertions.assertEquals("k2", searchContext1.docValueFieldsContext().get("k2"));
}
@Test
@ -114,10 +116,10 @@ public class EsUtilTest extends EsTestCase {
MappingPhase mappingPhase = new MappingPhase(client);
ExceptionChecker.expectThrowsNoException(() -> mappingPhase.execute(searchContext1));
ExceptionChecker.expectThrowsNoException(() -> mappingPhase.postProcess(searchContext1));
Assert.assertEquals("k3.keyword", searchContext1.fetchFieldsContext().get("k3"));
Assert.assertEquals("k3.keyword", searchContext1.docValueFieldsContext().get("k3"));
Assert.assertEquals("k1", searchContext1.docValueFieldsContext().get("k1"));
Assert.assertEquals("k2", searchContext1.docValueFieldsContext().get("k2"));
Assertions.assertEquals("k3.keyword", searchContext1.fetchFieldsContext().get("k3"));
Assertions.assertEquals("k3.keyword", searchContext1.docValueFieldsContext().get("k3"));
Assertions.assertEquals("k1", searchContext1.docValueFieldsContext().get("k1"));
Assertions.assertEquals("k2", searchContext1.docValueFieldsContext().get("k2"));
}
@ -127,21 +129,21 @@ public class EsUtilTest extends EsTestCase {
SearchContext searchContext = new SearchContext(esTableAfter7X);
MappingPhase.resolveFields(searchContext,
loadJsonFromFile("data/es/test_index_mapping_field_mult_analyzer.json"));
Assert.assertFalse(searchContext.docValueFieldsContext().containsKey("k3"));
Assertions.assertFalse(searchContext.docValueFieldsContext().containsKey("k3"));
}
@Test
public void testGetJsonObject() {
JSONObject json = (JSONObject) JSONValue.parse(jsonStr);
JSONObject upperBoundSetting = EsUtil.getJsonObject(json, "settings.index.bpack.partition", 0);
Assert.assertTrue(upperBoundSetting.containsKey("upperbound"));
Assert.assertEquals("12", (String) upperBoundSetting.get("upperbound"));
Assertions.assertTrue(upperBoundSetting.containsKey("upperbound"));
Assertions.assertEquals("12", (String) upperBoundSetting.get("upperbound"));
JSONObject unExistKey = EsUtil.getJsonObject(json, "set", 0);
Assert.assertNull(unExistKey);
Assertions.assertNull(unExistKey);
JSONObject singleKey = EsUtil.getJsonObject(json, "settings", 0);
Assert.assertTrue(singleKey.containsKey("index"));
Assertions.assertTrue(singleKey.containsKey("index"));
}
@Test(expected = ClassCastException.class)
@ -162,13 +164,13 @@ public class EsUtilTest extends EsTestCase {
Expr ltExpr = new BinaryPredicate(Operator.LT, k1, intLiteral);
Expr gtExpr = new BinaryPredicate(Operator.GT, k1, intLiteral);
Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new SlotRef(null, "k1"), new IntLiteral(3));
Assert.assertEquals("{\"term\":{\"k1\":3}}", EsUtil.toEsDsl(eqExpr).toJson());
Assert.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", EsUtil.toEsDsl(neExpr).toJson());
Assert.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", EsUtil.toEsDsl(leExpr).toJson());
Assert.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", EsUtil.toEsDsl(geExpr).toJson());
Assert.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", EsUtil.toEsDsl(ltExpr).toJson());
Assert.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", EsUtil.toEsDsl(gtExpr).toJson());
Assert.assertEquals("{\"term\":{\"k1\":3}}", EsUtil.toEsDsl(efnExpr).toJson());
Assertions.assertEquals("{\"term\":{\"k1\":3}}", EsUtil.toEsDsl(eqExpr).toJson());
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", EsUtil.toEsDsl(neExpr).toJson());
Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}", EsUtil.toEsDsl(leExpr).toJson());
Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}", EsUtil.toEsDsl(geExpr).toJson());
Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}", EsUtil.toEsDsl(ltExpr).toJson());
Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}", EsUtil.toEsDsl(gtExpr).toJson());
Assertions.assertEquals("{\"term\":{\"k1\":3}}", EsUtil.toEsDsl(efnExpr).toJson());
}
@Test
@ -184,11 +186,12 @@ public class EsUtilTest extends EsTestCase {
CompoundPredicate orPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, binaryPredicate1,
binaryPredicate2);
CompoundPredicate notPredicate = new CompoundPredicate(CompoundPredicate.Operator.NOT, binaryPredicate1, null);
Assert.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
Assertions.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
EsUtil.toEsDsl(andPredicate).toJson());
Assert.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
EsUtil.toEsDsl(orPredicate).toJson());
Assert.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}", EsUtil.toEsDsl(notPredicate).toJson());
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
EsUtil.toEsDsl(notPredicate).toJson());
}
@Test
@ -196,9 +199,9 @@ public class EsUtilTest extends EsTestCase {
SlotRef k1 = new SlotRef(null, "k1");
IsNullPredicate isNullPredicate = new IsNullPredicate(k1, false);
IsNullPredicate isNotNullPredicate = new IsNullPredicate(k1, true);
Assert.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}",
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}",
EsUtil.toEsDsl(isNullPredicate).toJson());
Assert.assertEquals("{\"exists\":{\"field\":\"k1\"}}", EsUtil.toEsDsl(isNotNullPredicate).toJson());
Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}", EsUtil.toEsDsl(isNotNullPredicate).toJson());
}
@Test
@ -210,9 +213,9 @@ public class EsUtilTest extends EsTestCase {
LikePredicate likePredicate1 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral1);
LikePredicate regexPredicate = new LikePredicate(LikePredicate.Operator.REGEXP, k1, stringLiteral2);
LikePredicate likePredicate2 = new LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral3);
Assert.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", EsUtil.toEsDsl(likePredicate1).toJson());
Assert.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", EsUtil.toEsDsl(regexPredicate).toJson());
Assert.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", EsUtil.toEsDsl(likePredicate2).toJson());
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", EsUtil.toEsDsl(likePredicate1).toJson());
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}", EsUtil.toEsDsl(regexPredicate).toJson());
Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}", EsUtil.toEsDsl(likePredicate2).toJson());
}
@Test
@ -225,8 +228,8 @@ public class EsUtilTest extends EsTestCase {
intLiterals.add(intLiteral2);
InPredicate isInPredicate = new InPredicate(k1, intLiterals, false);
InPredicate isNotInPredicate = new InPredicate(k1, intLiterals, true);
Assert.assertEquals("{\"terms\":{\"k1\":[3,5]}}", EsUtil.toEsDsl(isInPredicate).toJson());
Assert.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}",
Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}", EsUtil.toEsDsl(isInPredicate).toJson());
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}",
EsUtil.toEsDsl(isNotInPredicate).toJson());
}
@ -239,18 +242,37 @@ public class EsUtilTest extends EsTestCase {
exprs.add(k1);
exprs.add(stringLiteral);
FunctionCallExpr functionCallExpr = new FunctionCallExpr("esquery", exprs);
Assert.assertEquals(str, EsUtil.toEsDsl(functionCallExpr).toJson());
Assertions.assertEquals(str, EsUtil.toEsDsl(functionCallExpr).toJson());
SlotRef k2 = new SlotRef(null, "k2");
IntLiteral intLiteral = new IntLiteral(5);
BinaryPredicate binaryPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral);
CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate,
functionCallExpr);
Assert.assertEquals(
Assertions.assertEquals(
"{\"bool\":{\"must\":[{\"term\":{\"k2\":5}},{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}]}}",
EsUtil.toEsDsl(compoundPredicate).toJson());
}
@Test
public void testCastConvertEsDsl() {
SlotRef k1 = new SlotRef(null, "k1");
FloatLiteral floatLiteral = new FloatLiteral(3.14);
CastExpr castExpr = new CastExpr(TypeDef.create(PrimitiveType.INT), floatLiteral);
BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, k1, castExpr);
List<Expr> notPushDownList = new ArrayList<>();
Assertions.assertNull(EsUtil.toEsDsl(castPredicate, notPushDownList));
Assertions.assertEquals(1, notPushDownList.size());
SlotRef k2 = new SlotRef(null, "k2");
IntLiteral intLiteral = new IntLiteral(5);
BinaryPredicate eqPredicate = new BinaryPredicate(Operator.EQ, k2, intLiteral);
CompoundPredicate compoundPredicate = new CompoundPredicate(CompoundPredicate.Operator.OR, castPredicate,
eqPredicate);
EsUtil.toEsDsl(compoundPredicate, notPushDownList);
Assertions.assertEquals(3, notPushDownList.size());
}
@Test
public void testEs6Mapping() throws IOException, URISyntaxException {
JSONObject testAliases = EsUtil.getMappingProps("test", loadJsonFromFile("data/es/es6_aliases_mapping.json"),
@ -310,4 +332,5 @@ public class EsUtilTest extends EsTestCase {
+ "\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ "\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}", testIndex.toJSONString());
}
}