database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids)add InPredicate in expressions (#11264)

Browse Source 
1. Add InPredicate expression parser and translator
2. Add regression-test for In predicate (in nereids_syntax)
3. Support NOT EqualTo and NOT InPredicate in ExpressionTranslator#visitNot()
This commit is contained in:
Fy
2022-08-08 19:59:54 +08:00
committed by GitHub
parent 25a6be850d
commit 647b6e843a
8 changed files with 260 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
View File

@ -338,6 +338,16 @@ public class InPredicate extends Predicate {
// fn = getBuiltinFunction(analyzer, isNotIn ? NOT_IN_SET_LOOKUP : IN_SET_LOOKUP,
// argTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
opcode = isNotIn ? TExprOpcode.FILTER_NOT_IN : TExprOpcode.FILTER_IN;
// Todo: need to implement completely type cast compatibility, like castAllToCompatibleType();
Type compatibleType = getChild(0).getType();
for (int i = 1; i < children.size(); ++i) {
compatibleType = Type.getCmpType(compatibleType, getChild(i).getType());
}
for (int i = 0; i < children.size(); ++i) {
if (!getChild(i).getType().equals(compatibleType)) {
getChild(i).setType(compatibleType);
}
}
} else {
fn = getBuiltinFunction(isNotIn ? NOT_IN_ITERATE : IN_ITERATE,
argTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);

37
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.analysis.BoolLiteral;
import org.apache.doris.analysis.CaseExpr;
import org.apache.doris.analysis.CaseWhenClause;
import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FloatLiteral;
import org.apache.doris.analysis.FunctionCallExpr;
@ -48,6 +49,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
@ -66,6 +68,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisit
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* Used to translate expression of new optimizer to stale expr.
@ -143,10 +146,26 @@ public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTra
@Override
public Expr visitNot(Not not, PlanTranslatorContext context) {
return new org.apache.doris.analysis.CompoundPredicate(
org.apache.doris.analysis.CompoundPredicate.Operator.NOT,
not.child(0).accept(this, context),
null);
if (not.child() instanceof InPredicate) {
InPredicate inPredicate = (InPredicate) not.child();
List<Expr> inList = inPredicate.getOptions().stream()
.map(e -> translate(e, context))
.collect(Collectors.toList());
return new org.apache.doris.analysis.InPredicate(
inPredicate.getCompareExpr().accept(this, context),
inList,
true);
} else if (not.child() instanceof EqualTo) {
EqualTo equalTo = (EqualTo) not.child();
BinaryPredicate binaryPredicate = new BinaryPredicate(Operator.NE,
equalTo.child(0).accept(this, context),
equalTo.child(1).accept(this, context));
return binaryPredicate;
} else {
return new CompoundPredicate(CompoundPredicate.Operator.NOT,
not.child(0).accept(this, context),
null);
}
}
@Override
@ -255,6 +274,16 @@ public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTra
cast.left().accept(this, context));
}
@Override
public Expr visitInPredicate(InPredicate inPredicate, PlanTranslatorContext context) {
List<Expr> inList = inPredicate.getOptions().stream()
.map(e -> translate(e, context))
.collect(Collectors.toList());
return new org.apache.doris.analysis.InPredicate(inPredicate.getCompareExpr().accept(this, context),
inList,
false);
}
// TODO: Supports for `distinct`
@Override
public Expr visitBoundFunction(BoundFunction function, PlanTranslatorContext context) {

14
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -85,6 +85,7 @@ import org.apache.doris.nereids.trees.expressions.Exists;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.InSubquery;
import org.apache.doris.nereids.trees.expressions.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.IntervalLiteral;
@ -824,9 +825,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
break;
case DorisParser.IN:
if (ctx.query() == null) {
//TODO: InPredicate
outExpression = null;
throw new IllegalStateException("Unsupported predicate type: " + ctx.kind.getText());
outExpression = new InPredicate(
valueExpression,
withInList(ctx)
);
} else {
outExpression = new InSubquery(
valueExpression,
@ -864,4 +866,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
public Expression visitExist(ExistContext context) {
return ParserUtils.withOrigin(context, () -> new Exists(typedVisit(context.query())));
}
public List<Expression> withInList(PredicateContext ctx) {
List<Expression> expressions = ctx.expression().stream()
.map(this::getExpression).collect(ImmutableList.toImmutableList());
return expressions;
}
}

106
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java Normal file
View File

@ -0,0 +1,106 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DataType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* In predicate expression.
*/
public class InPredicate extends Expression {
private final Expression compareExpr;
private final List<Expression> options;
public InPredicate(Expression compareExpr, List<Expression> optionsList) {
super(new Builder<Expression>().add(compareExpr).addAll(optionsList).build().toArray(new Expression[0]));
this.compareExpr = Objects.requireNonNull(compareExpr, "Compare Expr cannot be null");
this.options = ImmutableList.copyOf(Objects.requireNonNull(optionsList, "In list cannot be null"));
}
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitInPredicate(this, context);
}
@Override
public DataType getDataType() throws UnboundException {
return BooleanType.INSTANCE;
}
@Override
public boolean nullable() throws UnboundException {
return children().stream().anyMatch(Expression::nullable);
}
@Override
public Expression withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() > 1);
return new InPredicate(children.get(0), ImmutableList.copyOf(children).subList(1, children.size()));
}
@Override
public String toString() {
return compareExpr + " IN " + options.stream()
.map(Expression::toString)
.collect(Collectors.joining(", ", "(", ")"));
}
@Override
public String toSql() {
return compareExpr.toSql() + " IN " + options.stream()
.map(Expression::toSql)
.collect(Collectors.joining(", ", "(", ")"));
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
InPredicate that = (InPredicate) o;
return Objects.equals(compareExpr, that.getCompareExpr())
&& Objects.equals(options, that.getOptions());
}
@Override
public int hashCode() {
return Objects.hash(compareExpr, options);
}
public Expression getCompareExpr() {
return compareExpr;
}
public List<Expression> getOptions() {
return options;
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
View File

@ -40,6 +40,7 @@ import org.apache.doris.nereids.trees.expressions.Exists;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.InSubquery;
import org.apache.doris.nereids.trees.expressions.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.LessThan;
@ -226,6 +227,10 @@ public abstract class ExpressionVisitor<R, C> {
return visit(caseWhen, context);
}
public R visitInPredicate(InPredicate inPredicate, C context) {
return visit(inPredicate, context);
}
public R visitInSubquery(InSubquery in, C context) {
return visitSubqueryExpr(in, context);
}

13
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java
View File

@ -46,9 +46,18 @@ public class ExpressionParserTest {
}
@Test
public void testSqlAnd() {
public void testInPredicate() throws Exception {
String in = "select * from test1 where d1 in (1, 2, 3)";
assertSql(in);
String inExpr = "c IN (a, b)";
assertExpr(inExpr);
}
@Test
public void testSqlAnd() throws Exception {
String sql = "select * from test1 where a > 1 and b > 1";
PARSER.parseSingle(sql);
assertSql(sql);
}
@Test

25
regression-test/data/nereids_syntax_p0/inpredicate.out Normal file
View File

@ -0,0 +1,25 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !in_predicate_1 --
-- !in_predicate_2 --
15 Supplier#000000015 DF35PepL5saAK INDIA 0 INDIA ASIA 18-687-542-7601
29 Supplier#000000029 VVSymB3fbwaN ARGENTINA4 ARGENTINA AMERICA 11-773-203-7342
9 Supplier#000000009 ,gJ6K2MKveYxQT IRAN 6 IRAN MIDDLE EAST 20-338-906-3675
-- !in_predicate_3 --
-- !in_predicate_4 --
-- !in_predicate_5 --
-- !in_predicate_6 --
15 Supplier#000000015 DF35PepL5saAK INDIA 0 INDIA ASIA 18-687-542-7601
29 Supplier#000000029 VVSymB3fbwaN ARGENTINA4 ARGENTINA AMERICA 11-773-203-7342
9 Supplier#000000009 ,gJ6K2MKveYxQT IRAN 6 IRAN MIDDLE EAST 20-338-906-3675
-- !in_predicate_7 --
-- !in_predicate_8 --
15 Supplier#000000015 DF35PepL5saAK INDIA 0 INDIA ASIA 18-687-542-7601
29 Supplier#000000029 VVSymB3fbwaN ARGENTINA4 ARGENTINA AMERICA 11-773-203-7342
9 Supplier#000000009 ,gJ6K2MKveYxQT IRAN 6 IRAN MIDDLE EAST 20-338-906-3675

59
regression-test/suites/nereids_syntax_p0/inpredicate.groovy Normal file
View File

@ -0,0 +1,59 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("inpredicate") {
sql """
SET enable_vectorized_engine=true
"""
sql """
SET enable_nereids_planner=true
"""
order_qt_in_predicate_1 """
SELECT * FROM supplier WHERE s_suppkey in (1, 2, 3);
"""
order_qt_in_predicate_2 """
SELECT * FROM supplier WHERE s_suppkey not in (1, 2, 3);
"""
order_qt_in_predicate_3 """
SELECT * FROM supplier WHERE s_suppkey in (1, 2, 128, 129);
"""
order_qt_in_predicate_4 """
SELECT * FROM supplier WHERE s_suppkey in (1, 2, 128, 32768, 32769);
"""
order_qt_in_predicate_5 """
SELECT * FROM supplier WHERE s_suppkey in (1, 2, 128, 32768, 2147483648);
"""
order_qt_in_predicate_6 """
SELECT * FROM supplier WHERE s_suppkey not in (1, 2, 128, 32768, 2147483648);
"""
order_qt_in_predicate_7 """
SELECT * FROM supplier WHERE s_nation in ('PERU', 'ETHIOPIA');
"""
order_qt_in_predicate_8 """
SELECT * FROM supplier WHERE s_nation not in ('PERU', 'ETHIOPIA');
"""
}