database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids) support cast and extract date for TPC-H (#10999)

Browse Source 
support cast and extract date for TPC-H, for example:

select cast(a as datetime) as d from test;
select extract(year from datetime_column) as y from test
This commit is contained in:
yinzhijian
2022-07-27 12:38:56 +08:00
committed by GitHub
parent 6933f5e328
commit 0cdd70e9c9
12 changed files with 243 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
View File

@ -145,6 +145,7 @@ DAY: 'DAY';
DATA: 'DATA';
DATABASE: 'DATABASE';
DATABASES: 'DATABASES';
DATE: 'DATE';
DATEADD: 'DATEADD';
DATE_ADD: 'DATE_ADD';
DATEDIFF: 'DATEDIFF';

5
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View File

@ -196,6 +196,7 @@ valueExpression
primaryExpression
: CASE whenClause+ (ELSE elseExpression=expression)? END #searchedCase
| CASE value=expression whenClause+ (ELSE elseExpression=expression)? END #simpleCase
| name=CAST LEFT_PAREN expression AS identifier RIGHT_PAREN #cast
| constant #constantDefault
| ASTERISK #star
| qualifiedName DOT ASTERISK #star
@ -205,6 +206,8 @@ primaryExpression
| identifier #columnReference
| base=primaryExpression DOT fieldName=identifier #dereference
| LEFT_PAREN expression RIGHT_PAREN #parenthesizedExpression
| EXTRACT LEFT_PAREN field=identifier FROM (DATE | TIMESTAMP)?
source=valueExpression RIGHT_PAREN #extract
;
qualifiedName
@ -324,6 +327,7 @@ ansiNonReserved
| DATA
| DATABASE
| DATABASES
| DATE
| DATEADD
| DATE_ADD
| DATEDIFF
@ -577,6 +581,7 @@ nonReserved
| DATA
| DATABASE
| DATABASES
| DATE
| DATEADD
| DATE_ADD
| DATEDIFF

20
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
View File

@ -529,4 +529,24 @@ public class CastExpr extends Expr {
|| (children.get(0).getType().isStringType() && !getType().isStringType())
|| (!children.get(0).getType().isDateType() && getType().isDateType());
}
@Override
public void finalizeImplForNereids() throws AnalysisException {
FunctionName fnName = new FunctionName(getFnName(type));
Function searchDesc = new Function(fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false);
if (type.isScalarType()) {
if (isImplicit) {
fn = Env.getCurrentEnv().getFunction(
searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
} else {
fn = Env.getCurrentEnv().getFunction(
searchDesc, Function.CompareMode.IS_IDENTICAL);
}
} else if (type.isArrayType()) {
fn = ScalarFunction.createBuiltin(getFnName(Type.ARRAY),
type, Function.NullableMode.ALWAYS_NULLABLE,
Lists.newArrayList(Type.VARCHAR), false,
"doris::CastFunctions::cast_to_array_val", null, null, true);
}
}
}

8
fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View File

@ -1318,10 +1318,16 @@ public class FunctionCallExpr extends Expr {
fn = getBuiltinFunction(fnName.getFunction(), new Type[]{childType},
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
type = fn.getReturnType();
} else if (fnName.getFunction().equalsIgnoreCase("substring")) {
} else if (fnName.getFunction().equalsIgnoreCase("substring")
|| fnName.getFunction().equalsIgnoreCase("cast")) {
Type[] childTypes = getChildren().stream().map(t -> t.type).toArray(Type[]::new);
fn = getBuiltinFunction(fnName.getFunction(), childTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
type = fn.getReturnType();
} else if (fnName.getFunction().equalsIgnoreCase("year")) {
Type childType = getChild(0).type;
fn = getBuiltinFunction(fnName.getFunction(), new Type[]{childType},
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
type = fn.getReturnType();
}
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.analysis.BinaryPredicate.Operator;
import org.apache.doris.analysis.BoolLiteral;
import org.apache.doris.analysis.CaseExpr;
import org.apache.doris.analysis.CaseWhenClause;
import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FloatLiteral;
import org.apache.doris.analysis.FunctionCallExpr;
@ -38,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Arithmetic;
import org.apache.doris.nereids.trees.expressions.Between;
import org.apache.doris.nereids.trees.expressions.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.CaseWhen;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.DateLiteral;
import org.apache.doris.nereids.trees.expressions.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.DoubleLiteral;
@ -244,6 +246,13 @@ public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTra
return new CaseExpr(null, caseWhenClauses, elseExpr);
}
@Override
public Expr visitCast(Cast cast, PlanTranslatorContext context) {
// left child of cast is expression, right child of cast is target type
return new CastExpr(cast.getDataType().toCatalogDataType(),
cast.left().accept(this, context));
}
// TODO: Supports for `distinct`
@Override
public Expr visitBoundFunction(BoundFunction function, PlanTranslatorContext context) {

16
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -73,6 +73,7 @@ import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.Between;
import org.apache.doris.nereids.trees.expressions.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.CaseWhen;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.DateLiteral;
import org.apache.doris.nereids.trees.expressions.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.Divide;
@ -117,6 +118,7 @@ import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.RuleNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@ -424,6 +426,20 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new CaseWhen(whenClauses, getExpression(context.elseExpression));
}
@Override
public Expression visitCast(DorisParser.CastContext ctx) {
return ParserUtils.withOrigin(ctx, () ->
new Cast(getExpression(ctx.expression()), ctx.identifier().getText()));
}
@Override
public UnboundFunction visitExtract(DorisParser.ExtractContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {
String functionName = ctx.field.getText();
return new UnboundFunction(functionName, false, Arrays.asList(getExpression(ctx.source)));
});
}
@Override
public UnboundFunction visitFunctionCall(DorisParser.FunctionCallContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
import org.apache.doris.nereids.trees.expressions.functions.Substring;
import org.apache.doris.nereids.trees.expressions.functions.Sum;
import org.apache.doris.nereids.trees.expressions.functions.Year;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@ -90,7 +91,6 @@ public class BindFunction implements AnalysisRuleFactory {
}
return new Sum(unboundFunction.getArguments().get(0));
} else if (name.equalsIgnoreCase("substr") || name.equalsIgnoreCase("substring")) {
List<Expression> arguments = unboundFunction.getArguments();
if (arguments.size() == 2) {
return new Substring(unboundFunction.getArguments().get(0),
@ -100,6 +100,12 @@ public class BindFunction implements AnalysisRuleFactory {
unboundFunction.getArguments().get(2));
}
return unboundFunction;
} else if (name.equalsIgnoreCase("year")) {
List<Expression> arguments = unboundFunction.getArguments();
if (arguments.size() != 1) {
return unboundFunction;
}
return new Year(unboundFunction.getArguments().get(0));
}
return unboundFunction;
}

69
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java Normal file
View File

@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import com.google.common.base.Preconditions;
import java.util.List;
/**
* cast function.
*/
public class Cast extends Expression implements BinaryExpression {
public Cast(Expression child, String type) {
super(child, new StringLiteral(type));
}
@Override
public DataType getDataType() {
StringLiteral type = (StringLiteral) right();
return DataType.convertFromString(type.getValue());
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitCast(this, context);
}
@Override
public boolean nullable() {
return left().nullable();
}
@Override
public Expression withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() == 2);
Preconditions.checkArgument(children.get(1) instanceof StringLiteral);
return new Cast(children.get(0), ((StringLiteral) children.get(1)).getValue());
}
@Override
public String toSql() throws UnboundException {
return "CAST(" + left().toSql() + " AS " + ((StringLiteral) right()).getValue() + ")";
}
@Override
public String toString() {
return toSql();
}
}

53
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Year.java Normal file
View File

@ -0,0 +1,53 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.functions;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.UnaryExpression;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.IntegerType;
import com.google.common.base.Preconditions;
import java.util.List;
/**
* year function.
*/
public class Year extends BoundFunction implements UnaryExpression {
public Year(Expression child) {
super("year", child);
}
@Override
public DataType getDataType() {
return IntegerType.INSTANCE;
}
@Override
public boolean nullable() {
return child().nullable();
}
@Override
public Expression withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() == 1);
return new Year(children.get(0));
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.Arithmetic;
import org.apache.doris.nereids.trees.expressions.Between;
import org.apache.doris.nereids.trees.expressions.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.CaseWhen;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.DateLiteral;
@ -179,6 +180,10 @@ public abstract class ExpressionVisitor<R, C> {
return visitStringRegexPredicate(regexp, context);
}
public R visitCast(Cast cast, C context) {
return visit(cast, context);
}
public R visitBoundFunction(BoundFunction boundFunction, C context) {
return visit(boundFunction, context);
}

30
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
View File

@ -76,6 +76,36 @@ public abstract class DataType {
}
}
/**
* Convert to data type in Nereids.
* throw exception when cannot convert to Nereids type
*
* @param type data type in string representation
* @return data type in Nereids
*/
public static DataType convertFromString(String type) {
// TODO: use a better way to resolve types
switch (type.toLowerCase()) {
case "bool":
case "boolean":
return BooleanType.INSTANCE;
case "int":
return IntegerType.INSTANCE;
case "bigint":
return BigIntType.INSTANCE;
case "double":
return DoubleType.INSTANCE;
case "string":
return StringType.INSTANCE;
case "null":
return NullType.INSTANCE;
case "datetime":
return DateTimeType.INSTANCE;
default:
throw new AnalysisException("Nereids do not support type: " + type);
}
}
public abstract Type toCatalogDataType();
@Override

21
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java
View File

@ -206,4 +206,25 @@ public class ExpressionParserTest {
interval = "tt > now() - interval 1+1 day";
assertExpr(interval);
}
@Test
public void testExtract() throws Exception {
String extract = "SELECT EXTRACT(YEAR FROM TIMESTAMP '2022-02-21 00:00:00') AS year FROM TEST;";
assertSql(extract);
String extract2 = "SELECT EXTRACT(YEAR FROM DATE '2022-02-21 00:00:00') AS year FROM TEST;";
assertSql(extract2);
String extract3 = "SELECT EXTRACT(YEAR FROM '2022-02-21 00:00:00') AS year FROM TEST;";
assertSql(extract3);
}
@Test
public void testCast() throws Exception {
String cast = "SELECT CAST(A AS STRING) FROM TEST;";
assertSql(cast);
String cast2 = "SELECT CAST(A AS INT) AS I FROM TEST;";
assertSql(cast2);
}
}