database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhancement](Doe) Be query es use fe generate dsl. (#11840)

Browse Source
This commit is contained in:
Stalary
2022-08-18 10:31:17 +08:00
committed by GitHub
parent cfb90b39c7
commit e1a1a04c2f
11 changed files with 78 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
View File

@ -66,7 +66,7 @@ public class EsTable extends Table {
public static final String MAX_DOCVALUE_FIELDS = "max_docvalue_fields";
public static final String NODES_DISCOVERY = "nodes_discovery";
public static final String HTTP_SSL_ENABLED = "http_ssl_enabled";
public static final String ES_DSL = "es_dsl";
public static final String QUERY_DSL = "query_dsl";
private static final Logger LOG = LogManager.getLogger(EsTable.class);
// Solr doc_values vs stored_fields performance-smackdown indicate:

6
fe/fe-core/src/main/java/org/apache/doris/common/Config.java
View File

@ -1738,4 +1738,10 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = true, masterOnly = true)
public static boolean enable_array_type = false;
/**
* Use new fe generate es dsl.
*/
@ConfField(mutable = true)
public static boolean enable_new_es_dsl = true;
}

27
fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java vendored
View File

@ -234,6 +234,13 @@ public class EsUtil {
}
}
private static Expr exprWithoutCast(Expr expr) {
if (expr instanceof CastExpr) {
return exprWithoutCast(expr.getChild(0));
}
return expr;
}
public static QueryBuilder toEsDsl(Expr expr) {
return toEsDsl(expr, new ArrayList<>());
}
@ -250,12 +257,22 @@ public class EsUtil {
return toCompoundEsDsl(expr, notPushDownList);
}
TExprOpcode opCode = expr.getOpcode();
// Cast can not pushdown
if (expr.getChild(0) instanceof CastExpr || expr.getChild(1) instanceof CastExpr) {
notPushDownList.add(expr);
return null;
String column;
Expr leftExpr = expr.getChild(0);
// Type transformed cast can not pushdown
if (leftExpr instanceof CastExpr) {
Expr withoutCastExpr = exprWithoutCast(leftExpr);
// pushdown col(float) >= 3
if (withoutCastExpr.getType().equals(leftExpr.getType()) || (withoutCastExpr.getType().isFloatingPointType()
&& leftExpr.getType().isFloatingPointType())) {
column = ((SlotRef) withoutCastExpr).getColumnName();
} else {
notPushDownList.add(expr);
return null;
}
} else {
column = ((SlotRef) leftExpr).getColumnName();
}
String column = ((SlotRef) expr.getChild(0)).getColumnName();
if (expr instanceof BinaryPredicate) {
Object value = toDorisLiteral(expr.getChild(1));
switch (opCode) {

17
fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.RangePartitionInfo;
import org.apache.doris.catalog.external.EsExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.external.elasticsearch.EsShardPartitions;
import org.apache.doris.external.elasticsearch.EsShardRouting;
@ -105,6 +106,7 @@ public class EsScanNode extends ScanNode {
computeColumnFilter();
assignBackends();
computeStats(analyzer);
buildQuery();
}
@Override
@ -162,7 +164,6 @@ public class EsScanNode extends ScanNode {
@SneakyThrows
@Override
protected void toThrift(TPlanNode msg) {
buildQuery();
msg.node_type = TPlanNodeType.ES_HTTP_SCAN_NODE;
Map<String, String> properties = Maps.newHashMap();
if (table.getUserName() != null) {
@ -173,15 +174,17 @@ public class EsScanNode extends ScanNode {
}
properties.put(EsTable.HTTP_SSL_ENABLED, String.valueOf(table.isHttpSslEnabled()));
TEsScanNode esScanNode = new TEsScanNode(desc.getId().asInt());
esScanNode.setProperties(properties);
if (table.isEnableDocValueScan()) {
esScanNode.setDocvalueContext(table.docValueContext());
properties.put(EsTable.DOC_VALUES_MODE, String.valueOf(useDocValueScan(desc, table.docValueContext())));
}
properties.put(EsTable.ES_DSL, queryBuilder.toJson());
if (Config.enable_new_es_dsl) {
properties.put(EsTable.QUERY_DSL, queryBuilder.toJson());
}
if (table.isEnableKeywordSniff() && table.fieldsContext().size() > 0) {
esScanNode.setFieldsContext(table.fieldsContext());
}
esScanNode.setProperties(properties);
msg.es_scan_node = esScanNode;
}
@ -339,11 +342,8 @@ public class EsScanNode extends ScanNode {
if (!conjuncts.isEmpty()) {
output.append(prefix).append("LOCAL_PREDICATES: ").append(getExplainString(conjuncts)).append("\n");
buildQuery();
output.append(prefix).append("REMOTE_PREDICATES: ").append(queryBuilder.toJson()).append("\n");
} else {
output.append(prefix).append("REMOTE_PREDICATES: ").append("{\"match_all\": {}}").append("\n");
}
output.append(prefix).append("REMOTE_PREDICATES: ").append(queryBuilder.toJson()).append("\n");
String indexName = table.getIndexName();
String typeName = table.getMappingType();
output.append(prefix).append(String.format("ES index/type: %s/%s", indexName, typeName)).append("\n");
@ -369,6 +369,9 @@ public class EsScanNode extends ScanNode {
} else {
queryBuilder = boolQueryBuilder;
}
if (Config.enable_new_es_dsl) {
conjuncts.removeIf(expr -> !notPushDownList.contains(expr));
}
}
}
}

15
fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java vendored
View File

@ -30,10 +30,10 @@ import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.analysis.TypeDef;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.EsTable;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.ExceptionChecker;
import mockit.Expectations;
@ -256,10 +256,9 @@ public class EsUtilTest extends EsTestCase {
@Test
public void testCastConvertEsDsl() {
SlotRef k1 = new SlotRef(null, "k1");
FloatLiteral floatLiteral = new FloatLiteral(3.14);
CastExpr castExpr = new CastExpr(TypeDef.create(PrimitiveType.INT), floatLiteral);
BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, k1, castExpr);
CastExpr castExpr = new CastExpr(Type.INT, floatLiteral);
BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, castExpr, new IntLiteral(3));
List<Expr> notPushDownList = new ArrayList<>();
Assertions.assertNull(EsUtil.toEsDsl(castPredicate, notPushDownList));
Assertions.assertEquals(1, notPushDownList.size());
@ -271,6 +270,14 @@ public class EsUtilTest extends EsTestCase {
eqPredicate);
EsUtil.toEsDsl(compoundPredicate, notPushDownList);
Assertions.assertEquals(3, notPushDownList.size());
SlotRef k3 = new SlotRef(null, "k3");
k3.setType(Type.FLOAT);
CastExpr castDoubleExpr = new CastExpr(Type.DOUBLE, k3);
BinaryPredicate castDoublePredicate = new BinaryPredicate(Operator.GE, castDoubleExpr,
new FloatLiteral(3.0, Type.DOUBLE));
EsUtil.toEsDsl(castDoublePredicate, notPushDownList);
Assertions.assertEquals(3, notPushDownList.size());
}
@Test