From 647b6e843a67cff095ebd661a134470574b279af Mon Sep 17 00:00:00 2001 From: Fy Date: Mon, 8 Aug 2022 19:59:54 +0800 Subject: [PATCH] [feature](nereids)add InPredicate in expressions (#11264) 1. Add InPredicate expression parser and translator 2. Add regression-test for In predicate (in nereids_syntax) 3. Support NOT EqualTo and NOT InPredicate in ExpressionTranslator#visitNot() --- .../apache/doris/analysis/InPredicate.java | 10 ++ .../glue/translator/ExpressionTranslator.java | 37 +++++- .../nereids/parser/LogicalPlanBuilder.java | 14 ++- .../trees/expressions/InPredicate.java | 106 ++++++++++++++++++ .../visitor/ExpressionVisitor.java | 5 + .../expressions/ExpressionParserTest.java | 13 ++- .../data/nereids_syntax_p0/inpredicate.out | 25 +++++ .../nereids_syntax_p0/inpredicate.groovy | 59 ++++++++++ 8 files changed, 260 insertions(+), 9 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java create mode 100644 regression-test/data/nereids_syntax_p0/inpredicate.out create mode 100644 regression-test/suites/nereids_syntax_p0/inpredicate.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java index 5d01e08a9c..8c1a31942f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java @@ -338,6 +338,16 @@ public class InPredicate extends Predicate { // fn = getBuiltinFunction(analyzer, isNotIn ? NOT_IN_SET_LOOKUP : IN_SET_LOOKUP, // argTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); opcode = isNotIn ? TExprOpcode.FILTER_NOT_IN : TExprOpcode.FILTER_IN; + // Todo: need to implement completely type cast compatibility, like castAllToCompatibleType(); + Type compatibleType = getChild(0).getType(); + for (int i = 1; i < children.size(); ++i) { + compatibleType = Type.getCmpType(compatibleType, getChild(i).getType()); + } + for (int i = 0; i < children.size(); ++i) { + if (!getChild(i).getType().equals(compatibleType)) { + getChild(i).setType(compatibleType); + } + } } else { fn = getBuiltinFunction(isNotIn ? NOT_IN_ITERATE : IN_ITERATE, argTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 94c6e3511f..8e69d88159 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.BoolLiteral; import org.apache.doris.analysis.CaseExpr; import org.apache.doris.analysis.CaseWhenClause; import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.FloatLiteral; import org.apache.doris.analysis.FunctionCallExpr; @@ -48,6 +49,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.LessThanEqual; @@ -66,6 +68,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisit import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * Used to translate expression of new optimizer to stale expr. @@ -143,10 +146,26 @@ public class ExpressionTranslator extends DefaultExpressionVisitor inList = inPredicate.getOptions().stream() + .map(e -> translate(e, context)) + .collect(Collectors.toList()); + return new org.apache.doris.analysis.InPredicate( + inPredicate.getCompareExpr().accept(this, context), + inList, + true); + } else if (not.child() instanceof EqualTo) { + EqualTo equalTo = (EqualTo) not.child(); + BinaryPredicate binaryPredicate = new BinaryPredicate(Operator.NE, + equalTo.child(0).accept(this, context), + equalTo.child(1).accept(this, context)); + return binaryPredicate; + } else { + return new CompoundPredicate(CompoundPredicate.Operator.NOT, + not.child(0).accept(this, context), + null); + } } @Override @@ -255,6 +274,16 @@ public class ExpressionTranslator extends DefaultExpressionVisitor inList = inPredicate.getOptions().stream() + .map(e -> translate(e, context)) + .collect(Collectors.toList()); + return new org.apache.doris.analysis.InPredicate(inPredicate.getCompareExpr().accept(this, context), + inList, + false); + } + // TODO: Supports for `distinct` @Override public Expr visitBoundFunction(BoundFunction function, PlanTranslatorContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 9eeebe4515..2453759199 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -85,6 +85,7 @@ import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.InSubquery; import org.apache.doris.nereids.trees.expressions.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.IntervalLiteral; @@ -824,9 +825,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { break; case DorisParser.IN: if (ctx.query() == null) { - //TODO: InPredicate - outExpression = null; - throw new IllegalStateException("Unsupported predicate type: " + ctx.kind.getText()); + outExpression = new InPredicate( + valueExpression, + withInList(ctx) + ); } else { outExpression = new InSubquery( valueExpression, @@ -864,4 +866,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { public Expression visitExist(ExistContext context) { return ParserUtils.withOrigin(context, () -> new Exists(typedVisit(context.query()))); } + + public List withInList(PredicateContext ctx) { + List expressions = ctx.expression().stream() + .map(this::getExpression).collect(ImmutableList.toImmutableList()); + return expressions; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java new file mode 100644 index 0000000000..2a52c98ed0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.DataType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * In predicate expression. + */ +public class InPredicate extends Expression { + + private final Expression compareExpr; + private final List options; + + public InPredicate(Expression compareExpr, List optionsList) { + super(new Builder().add(compareExpr).addAll(optionsList).build().toArray(new Expression[0])); + this.compareExpr = Objects.requireNonNull(compareExpr, "Compare Expr cannot be null"); + this.options = ImmutableList.copyOf(Objects.requireNonNull(optionsList, "In list cannot be null")); + } + + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitInPredicate(this, context); + } + + @Override + public DataType getDataType() throws UnboundException { + return BooleanType.INSTANCE; + } + + @Override + public boolean nullable() throws UnboundException { + return children().stream().anyMatch(Expression::nullable); + } + + @Override + public Expression withChildren(List children) { + Preconditions.checkArgument(children.size() > 1); + return new InPredicate(children.get(0), ImmutableList.copyOf(children).subList(1, children.size())); + } + + @Override + public String toString() { + return compareExpr + " IN " + options.stream() + .map(Expression::toString) + .collect(Collectors.joining(", ", "(", ")")); + } + + @Override + public String toSql() { + return compareExpr.toSql() + " IN " + options.stream() + .map(Expression::toSql) + .collect(Collectors.joining(", ", "(", ")")); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + InPredicate that = (InPredicate) o; + return Objects.equals(compareExpr, that.getCompareExpr()) + && Objects.equals(options, that.getOptions()); + } + + @Override + public int hashCode() { + return Objects.hash(compareExpr, options); + } + + public Expression getCompareExpr() { + return compareExpr; + } + + public List getOptions() { + return options; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index e39d215a8c..e63d6bdd10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -40,6 +40,7 @@ import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.InSubquery; import org.apache.doris.nereids.trees.expressions.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.LessThan; @@ -226,6 +227,10 @@ public abstract class ExpressionVisitor { return visit(caseWhen, context); } + public R visitInPredicate(InPredicate inPredicate, C context) { + return visit(inPredicate, context); + } + public R visitInSubquery(InSubquery in, C context) { return visitSubqueryExpr(in, context); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java index 7a6775d307..ece7f547e1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java @@ -46,9 +46,18 @@ public class ExpressionParserTest { } @Test - public void testSqlAnd() { + public void testInPredicate() throws Exception { + String in = "select * from test1 where d1 in (1, 2, 3)"; + assertSql(in); + + String inExpr = "c IN (a, b)"; + assertExpr(inExpr); + } + + @Test + public void testSqlAnd() throws Exception { String sql = "select * from test1 where a > 1 and b > 1"; - PARSER.parseSingle(sql); + assertSql(sql); } @Test diff --git a/regression-test/data/nereids_syntax_p0/inpredicate.out b/regression-test/data/nereids_syntax_p0/inpredicate.out new file mode 100644 index 0000000000..daf3a79e43 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/inpredicate.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !in_predicate_1 -- + +-- !in_predicate_2 -- +15 Supplier#000000015 DF35PepL5saAK INDIA 0 INDIA ASIA 18-687-542-7601 +29 Supplier#000000029 VVSymB3fbwaN ARGENTINA4 ARGENTINA AMERICA 11-773-203-7342 +9 Supplier#000000009 ,gJ6K2MKveYxQT IRAN 6 IRAN MIDDLE EAST 20-338-906-3675 + +-- !in_predicate_3 -- + +-- !in_predicate_4 -- + +-- !in_predicate_5 -- + +-- !in_predicate_6 -- +15 Supplier#000000015 DF35PepL5saAK INDIA 0 INDIA ASIA 18-687-542-7601 +29 Supplier#000000029 VVSymB3fbwaN ARGENTINA4 ARGENTINA AMERICA 11-773-203-7342 +9 Supplier#000000009 ,gJ6K2MKveYxQT IRAN 6 IRAN MIDDLE EAST 20-338-906-3675 + +-- !in_predicate_7 -- + +-- !in_predicate_8 -- +15 Supplier#000000015 DF35PepL5saAK INDIA 0 INDIA ASIA 18-687-542-7601 +29 Supplier#000000029 VVSymB3fbwaN ARGENTINA4 ARGENTINA AMERICA 11-773-203-7342 +9 Supplier#000000009 ,gJ6K2MKveYxQT IRAN 6 IRAN MIDDLE EAST 20-338-906-3675 diff --git a/regression-test/suites/nereids_syntax_p0/inpredicate.groovy b/regression-test/suites/nereids_syntax_p0/inpredicate.groovy new file mode 100644 index 0000000000..d17a2dc198 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/inpredicate.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("inpredicate") { + sql """ + SET enable_vectorized_engine=true + """ + + sql """ + SET enable_nereids_planner=true + """ + + order_qt_in_predicate_1 """ + SELECT * FROM supplier WHERE s_suppkey in (1, 2, 3); + """ + + order_qt_in_predicate_2 """ + SELECT * FROM supplier WHERE s_suppkey not in (1, 2, 3); + """ + + order_qt_in_predicate_3 """ + SELECT * FROM supplier WHERE s_suppkey in (1, 2, 128, 129); + """ + + order_qt_in_predicate_4 """ + SELECT * FROM supplier WHERE s_suppkey in (1, 2, 128, 32768, 32769); + """ + + order_qt_in_predicate_5 """ + SELECT * FROM supplier WHERE s_suppkey in (1, 2, 128, 32768, 2147483648); + """ + + order_qt_in_predicate_6 """ + SELECT * FROM supplier WHERE s_suppkey not in (1, 2, 128, 32768, 2147483648); + """ + + order_qt_in_predicate_7 """ + SELECT * FROM supplier WHERE s_nation in ('PERU', 'ETHIOPIA'); + """ + + order_qt_in_predicate_8 """ + SELECT * FROM supplier WHERE s_nation not in ('PERU', 'ETHIOPIA'); + """ +} +