diff --git a/fe/check/checkstyle/checkstyle.xml b/fe/check/checkstyle/checkstyle.xml index 39a1e5c569..5c308bcfd8 100644 --- a/fe/check/checkstyle/checkstyle.xml +++ b/fe/check/checkstyle/checkstyle.xml @@ -438,5 +438,8 @@ under the License. default="checkstyle-xpath-suppressions.xml" /> + + + diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 34f7858112..fa640dc3ef 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -643,7 +643,8 @@ under the License. mariadb-java-client - + org.antlr antlr4-runtime @@ -702,6 +703,12 @@ under the License. kryo-shaded + + + io.trino + trino-parser + + org.apache.arrow diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java new file mode 100644 index 0000000000..50af01dd2e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.parser.trino.TrinoFnCallTransformer.PlaceholderCollector; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** + * Expression placeHolder, the expression in PlaceHolderExpression will be collected by + * + * @see PlaceholderCollector + */ +public class PlaceholderExpression extends Expression implements AlwaysNotNullable { + + private final Class delegateClazz; + /** + * 1 based + */ + private final int position; + + public PlaceholderExpression(List children, Class delegateClazz, int position) { + super(children); + this.delegateClazz = Objects.requireNonNull(delegateClazz, "delegateClazz should not be null"); + this.position = position; + } + + public static PlaceholderExpression of(Class delegateClazz, int position) { + return new PlaceholderExpression(ImmutableList.of(), delegateClazz, position); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visit(this, context); + } + + public Class getDelegateClazz() { + return delegateClazz; + } + + public int getPosition() { + return position; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PlaceholderExpression that = (PlaceholderExpression) o; + return position == that.position && Objects.equals(delegateClazz, that.delegateClazz); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), delegateClazz, position); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java new file mode 100644 index 0000000000..38a028c71d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.exceptions; + +/** + * DialectTransformException when have not supported transforming for the + * {@link io.trino.sql.tree.Node}. + */ +public class DialectTransformException extends UnsupportedOperationException { + + public DialectTransformException(String msg) { + super(String.format("Unsupported dialect transformation is %s", msg)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java new file mode 100644 index 0000000000..9e977fedbc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.exceptions; + +import org.apache.doris.nereids.parser.ParseDialect; + +/** + * UnsupportedDialectException when not match any in + * {@link org.apache.doris.nereids.parser.ParseDialect}. + */ +public class UnsupportedDialectException extends UnsupportedOperationException { + + public UnsupportedDialectException(ParseDialect dialect) { + super(String.format("Unsupported dialect name is %s, version is %s", + dialect.getDialect().getDialectName(), dialect.getVersion().getVersionName())); + } + + public UnsupportedDialectException(String type, String msg) { + super(String.format("Unsupported dialect type is %s, msg is %s", type, msg)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 6867c9c465..aa711a8e27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -314,7 +314,6 @@ import org.apache.doris.nereids.trees.plans.commands.info.RollupDefinition; import org.apache.doris.nereids.trees.plans.commands.info.StepPartition; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; -import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -432,7 +431,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { @Override public LogicalPlan visitUpdate(UpdateContext ctx) { - LogicalPlan query = withCheckPolicy(new UnboundRelation( + LogicalPlan query = LogicalPlanBuilderAssistant.withCheckPolicy(new UnboundRelation( StatementScopeIdGenerator.newRelationId(), visitMultipartIdentifier(ctx.tableName))); query = withTableAlias(query, ctx.tableAlias()); if (ctx.fromClause() != null) { @@ -455,7 +454,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { public LogicalPlan visitDelete(DeleteContext ctx) { List tableName = visitMultipartIdentifier(ctx.tableName); List partitions = ctx.partition == null ? ImmutableList.of() : visitIdentifierList(ctx.partition); - LogicalPlan query = withTableAlias(withCheckPolicy( + LogicalPlan query = withTableAlias(LogicalPlanBuilderAssistant.withCheckPolicy( new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableName)), ctx.tableAlias()); if (ctx.USING() != null) { query = withRelations(query, ctx.relation()); @@ -480,7 +479,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { // handle path string String tmpPath = ctx.filePath.getText(); - String path = escapeBackSlash(tmpPath.substring(1, tmpPath.length() - 1)); + String path = LogicalPlanBuilderAssistant.escapeBackSlash(tmpPath.substring(1, tmpPath.length() - 1)); Optional expr = Optional.empty(); if (ctx.whereClause() != null) { @@ -630,7 +629,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { String labelName = ctx.lableName.getText(); Map properties = visitPropertyItemList(ctx.properties); String commentSpec = ctx.commentSpec() == null ? "" : ctx.commentSpec().STRING_LITERAL().getText(); - String comment = escapeBackSlash(commentSpec.substring(1, commentSpec.length() - 1)); + String comment = + LogicalPlanBuilderAssistant.escapeBackSlash(commentSpec.substring(1, commentSpec.length() - 1)); return new LoadCommand(labelName, dataDescriptions.build(), bulkDesc, properties, comment); } @@ -855,10 +855,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { }); } - private LogicalPlan withCheckPolicy(LogicalPlan plan) { - return new LogicalCheckPolicy<>(plan); - } - @Override public LogicalPlan visitTableName(TableNameContext ctx) { List tableId = visitMultipartIdentifier(ctx.multipartIdentifier()); @@ -888,7 +884,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { } TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); - LogicalPlan checkedRelation = withCheckPolicy( + LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy( new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, Optional.ofNullable(tableSample))); @@ -1713,53 +1709,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { public Literal visitStringLiteral(StringLiteralContext ctx) { // TODO: add unescapeSQLString. String txt = ctx.STRING_LITERAL().getText(); - String s = escapeBackSlash(txt.substring(1, txt.length() - 1)); + String s = LogicalPlanBuilderAssistant.escapeBackSlash(txt.substring(1, txt.length() - 1)); return new VarcharLiteral(s); } - private String escapeBackSlash(String str) { - StringBuilder sb = new StringBuilder(); - int strLen = str.length(); - for (int i = 0; i < strLen; ++i) { - char c = str.charAt(i); - if (c == '\\' && (i + 1) < strLen) { - switch (str.charAt(i + 1)) { - case 'n': - sb.append('\n'); - break; - case 't': - sb.append('\t'); - break; - case 'r': - sb.append('\r'); - break; - case 'b': - sb.append('\b'); - break; - case '0': - sb.append('\0'); // Ascii null - break; - case 'Z': // ^Z must be escaped on Win32 - sb.append('\032'); - break; - case '_': - case '%': - sb.append('\\'); // remember prefix for wildcard - sb.append(str.charAt(i + 1)); - break; - default: - sb.append(str.charAt(i + 1)); - break; - } - i++; - } else { - sb.append(c); - } - } - - return sb.toString(); - } - /** * cast all items to same types. * TODO remove this function after we refactor type coercion. @@ -2695,7 +2648,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { String comment; if (ctx.commentSpec() != null) { comment = ctx.commentSpec().STRING_LITERAL().getText(); - comment = escapeBackSlash(comment.substring(1, comment.length() - 1)); + comment = LogicalPlanBuilderAssistant.escapeBackSlash(comment.substring(1, comment.length() - 1)); } else { comment = ""; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java new file mode 100644 index 0000000000..606fd7a159 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import java.math.BigInteger; + +/** + * Logical plan builder assistant for buildIn dialect and other dialect. + * The same logical in {@link org.apache.doris.nereids.parser.LogicalPlanBuilder} + * and {@link org.apache.doris.nereids.parser.trino.LogicalPlanTrinoBuilder} can be + * extracted to here. + */ +public class LogicalPlanBuilderAssistant { + + private LogicalPlanBuilderAssistant() { + } + + /** + * EscapeBackSlash such \n, \t + */ + public static String escapeBackSlash(String str) { + StringBuilder sb = new StringBuilder(); + int strLen = str.length(); + for (int i = 0; i < strLen; ++i) { + char c = str.charAt(i); + if (c == '\\' && (i + 1) < strLen) { + switch (str.charAt(i + 1)) { + case 'n': + sb.append('\n'); + break; + case 't': + sb.append('\t'); + break; + case 'r': + sb.append('\r'); + break; + case 'b': + sb.append('\b'); + break; + case '0': + sb.append('\0'); // Ascii null + break; + case 'Z': // ^Z must be escaped on Win32 + sb.append('\032'); + break; + case '_': + case '%': + sb.append('\\'); // remember prefix for wildcard + sb.append(str.charAt(i + 1)); + break; + default: + sb.append(str.charAt(i + 1)); + break; + } + i++; + } else { + sb.append(c); + } + } + return sb.toString(); + } + + /** + * Handle Integer literal by BigInteger. + */ + public static Literal handleIntegerLiteral(String value) { + BigInteger bigInt = new BigInteger(value); + if (BigInteger.valueOf(bigInt.byteValue()).equals(bigInt)) { + return new TinyIntLiteral(bigInt.byteValue()); + } else if (BigInteger.valueOf(bigInt.shortValue()).equals(bigInt)) { + return new SmallIntLiteral(bigInt.shortValue()); + } else if (BigInteger.valueOf(bigInt.intValue()).equals(bigInt)) { + return new IntegerLiteral(bigInt.intValue()); + } else if (BigInteger.valueOf(bigInt.longValue()).equals(bigInt)) { + return new BigIntLiteral(bigInt.longValueExact()); + } else { + return new LargeIntLiteral(bigInt); + } + } + + /** + * Wrap plan withCheckPolicy. + */ + public static LogicalPlan withCheckPolicy(LogicalPlan plan) { + return new LogicalCheckPolicy<>(plan); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index db1b9bc286..9e5506cc99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -22,10 +22,14 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.DorisLexer; import org.apache.doris.nereids.DorisParser; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.exceptions.UnsupportedDialectException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.parser.trino.LogicalPlanTrinoBuilder; +import org.apache.doris.nereids.parser.trino.TrinoParser; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.qe.SessionVariable; import com.google.common.collect.Lists; import org.antlr.v4.runtime.CharStreams; @@ -33,15 +37,20 @@ import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Function; /** * Sql parser, convert sql DSL to logical plan. */ public class NereidsParser { + public static final Logger LOG = LogManager.getLogger(NereidsParser.class); private static final ParseErrorListener PARSE_ERROR_LISTENER = new ParseErrorListener(); private static final PostProcessor POST_PROCESSOR = new PostProcessor(); @@ -58,6 +67,39 @@ public class NereidsParser { return statementBases; } + /** + * ParseSQL with dialect. + */ + public List parseSQL(String sql, SessionVariable sessionVariable) { + if (ParseDialect.TRINO_395.getDialect().getDialectName() + .equalsIgnoreCase(sessionVariable.getSqlDialect())) { + return parseSQLWithDialect(sql, sessionVariable); + } else { + return parseSQL(sql); + } + } + + private List parseSQLWithDialect(String sql, SessionVariable sessionVariable) { + final List logicalPlans = new ArrayList<>(); + try { + io.trino.sql.parser.StatementSplitter splitter = new io.trino.sql.parser.StatementSplitter(sql); + ParserContext parserContext = new ParserContext(ParseDialect.TRINO_395); + StatementContext statementContext = new StatementContext(); + for (io.trino.sql.parser.StatementSplitter.Statement statement : splitter.getCompleteStatements()) { + Object parsedPlan = parseSingleWithDialect(statement.statement(), parserContext); + logicalPlans.add(parsedPlan == null + ? null : new LogicalPlanAdapter((LogicalPlan) parsedPlan, statementContext)); + } + } catch (io.trino.sql.parser.ParsingException | UnsupportedDialectException e) { + LOG.debug("Failed to parse logical plan from trino, sql is :{}", sql, e); + return parseSQL(sql); + } + if (logicalPlans.isEmpty() || logicalPlans.stream().anyMatch(Objects::isNull)) { + return parseSQL(sql); + } + return logicalPlans; + } + /** * parse sql DSL string. * @@ -90,6 +132,25 @@ public class NereidsParser { return (T) logicalPlanBuilder.visit(tree); } + /** + * Parse dialect sql. + * + * @param sql sql string + * @param parserContext parse context + * @return logical plan + */ + public T parseSingleWithDialect(String sql, ParserContext parserContext) { + if (ParseDialect.TRINO_395.equals(parserContext.getParserDialect())) { + io.trino.sql.tree.Statement statement = TrinoParser.parse(sql); + return (T) new LogicalPlanTrinoBuilder().visit(statement, parserContext); + } else { + LOG.debug("Failed to parse logical plan, the dialect name is {}, version is {}", + parserContext.getParserDialect().getDialect().getDialectName(), + parserContext.getParserDialect().getVersion()); + throw new UnsupportedDialectException(parserContext.getParserDialect()); + } + } + private ParserRuleContext toAst(String sql, Function parseFunction) { DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); CommonTokenStream tokenStream = new CommonTokenStream(lexer); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseDialect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseDialect.java new file mode 100644 index 0000000000..e4255ff355 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseDialect.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +/** + * ParseDialect enum, maybe support other dialect. + */ +public enum ParseDialect { + + /** + * Trino parser and it's version is 395. + */ + TRINO_395(Dialect.TRINO, Version.TRINO_395), + /** + * Doris parser and it's version is 2.0.0. + */ + DORIS_2_ALL(Dialect.DORIS, Version.DORIS_2_ALL); + + private final Dialect dialect; + private final Version version; + + ParseDialect(Dialect dialect, Version version) { + this.dialect = dialect; + this.version = version; + } + + public Version getVersion() { + return version; + } + + public Dialect getDialect() { + return dialect; + } + + /** + * The version of parse dialect. + */ + public enum Version { + /** + * Trino parser and it's version is 395. + */ + TRINO_395("395"), + /** + * Doris parser and it's version is 2.0.0. + */ + DORIS_2_ALL("2.*"); + private final String version; + + Version(String version) { + this.version = version; + } + + public String getVersionName() { + return version; + } + } + + /** + * The dialect name of parse dialect. + */ + public enum Dialect { + /** + * Trino parser dialect + */ + TRINO("trino"), + /** + * Doris parser dialect + */ + DORIS("doris"); + + private String dialectName; + + Dialect(String dialectName) { + this.dialectName = dialectName; + } + + public String getDialectName() { + return dialectName; + } + + /** + * Get dialect by name + */ + public static Dialect getByName(String dialectName) { + if (dialectName == null) { + return null; + } + for (Dialect dialect : Dialect.values()) { + if (dialect.getDialectName().equals(dialectName.toLowerCase())) { + return dialect; + } + } + return null; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserContext.java new file mode 100644 index 0000000000..c36767f1be --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserContext.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +/** + * SQL parser context, support additional variable to control parse process. + */ +public class ParserContext { + + private final ParseDialect parseDialect; + + public ParserContext(ParseDialect parseDialect) { + this.parseDialect = parseDialect; + } + + public ParseDialect getParserDialect() { + return parseDialect; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/AbstractFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/AbstractFnCallTransformer.java new file mode 100644 index 0000000000..4bdb5bcd08 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/AbstractFnCallTransformer.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +import org.apache.doris.nereids.parser.ParserContext; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.Function; + +import java.util.List; + +/** + * Abstract function transformer, dialect function transformer should extend this. + */ +public abstract class AbstractFnCallTransformer { + + /** + * Check source function signature is the same between function from SQL and + * definition in function call transformer. + * Check the targetArgs param matches the definition in function call transformer. + */ + protected abstract boolean check(String sourceFnName, + List sourceFnTransformedArguments, + ParserContext context); + + /** + * After check, do transform for function mapping. + */ + protected abstract Function transform(String sourceFnName, + List sourceFnTransformedArguments, + ParserContext context); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java new file mode 100644 index 0000000000..e13229423c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +/** + * Trino complex function transformer + */ +public abstract class ComplexTrinoFnCallTransformer extends AbstractFnCallTransformer { + + protected abstract String getSourceFnName(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java new file mode 100644 index 0000000000..b59a9327bd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.parser.ParserContext; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * DateDiff complex function transformer + */ +public class DateDiffFnCallTransformer extends ComplexTrinoFnCallTransformer { + + private static final String SECOND = "second"; + private static final String HOUR = "hour"; + private static final String DAY = "day"; + private static final String MILLI_SECOND = "millisecond"; + + @Override + public String getSourceFnName() { + return "date_diff"; + } + + @Override + protected boolean check(String sourceFnName, List sourceFnTransformedArguments, + ParserContext context) { + return getSourceFnName().equalsIgnoreCase(sourceFnName); + } + + @Override + protected Function transform(String sourceFnName, List sourceFnTransformedArguments, + ParserContext context) { + if (sourceFnTransformedArguments.size() != 3) { + return null; + } + VarcharLiteral diffGranularity = (VarcharLiteral) sourceFnTransformedArguments.get(0); + if (SECOND.equals(diffGranularity.getValue())) { + return new UnboundFunction( + "seconds_diff", + ImmutableList.of(sourceFnTransformedArguments.get(1), sourceFnTransformedArguments.get(2))); + } + // TODO: support other date diff granularity + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/LogicalPlanTrinoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/LogicalPlanTrinoBuilder.java new file mode 100644 index 0000000000..c810e49b17 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/LogicalPlanTrinoBuilder.java @@ -0,0 +1,325 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +import org.apache.doris.nereids.analyzer.UnboundAlias; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundRelation; +import org.apache.doris.nereids.analyzer.UnboundResultSink; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.exceptions.DialectTransformException; +import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant; +import org.apache.doris.nereids.parser.ParserContext; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.coercion.CharacterType; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * The actually planBuilder for Trino SQL to Doris logical plan. + * It depends on {@link io.trino.sql.tree.AstVisitor} + */ +public class LogicalPlanTrinoBuilder extends io.trino.sql.tree.AstVisitor { + + public Object visit(io.trino.sql.tree.Node node, ParserContext context) { + return this.process(node, context); + } + + public T visit(io.trino.sql.tree.Node node, ParserContext context, Class clazz) { + return clazz.cast(this.process(node, context)); + } + + public List visit(List nodes, ParserContext context, Class clazz) { + return nodes.stream() + .map(node -> clazz.cast(this.process(node, context))) + .collect(Collectors.toList()); + } + + public Object processOptional(Optional node, ParserContext context) { + return node.map(value -> this.process(value, context)).orElse(null); + } + + public T processOptional(Optional node, + ParserContext context, Class clazz) { + return node.map(value -> clazz.cast(this.process(value, context))).orElse(null); + } + + @Override + protected LogicalPlan visitQuery(io.trino.sql.tree.Query node, ParserContext context) { + io.trino.sql.tree.QueryBody queryBody = node.getQueryBody(); + LogicalPlan logicalPlan = (LogicalPlan) visit(queryBody, context); + if (!(queryBody instanceof io.trino.sql.tree.QuerySpecification)) { + // TODO: need to handle orderBy and limit + throw new DialectTransformException("transform querySpecification"); + } + return logicalPlan; + } + + @Override + protected LogicalPlan visitQuerySpecification(io.trino.sql.tree.QuerySpecification node, + ParserContext context) { + // from -> where -> group by -> having -> select + Optional from = node.getFrom(); + LogicalPlan fromPlan = processOptional(from, context, LogicalPlan.class); + List selectItems = node.getSelect().getSelectItems(); + if (from == null || !from.isPresent()) { + // TODO: support query values + List expressions = selectItems.stream() + .map(item -> visit(item, context, NamedExpression.class)) + .collect(ImmutableList.toImmutableList()); + return new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), expressions); + } + // TODO: support predicate, aggregate, having, order by, limit + // TODO: support distinct + boolean isDistinct = node.getSelect().isDistinct(); + return new UnboundResultSink<>(withProjection(selectItems, fromPlan, isDistinct, context)); + } + + private LogicalProject withProjection(List selectItems, + LogicalPlan input, + boolean isDistinct, + ParserContext context) { + List expressions = selectItems.stream() + .map(item -> visit(item, context, NamedExpression.class)) + .collect(Collectors.toList()); + return new LogicalProject(expressions, ImmutableList.of(), isDistinct, input); + } + + @Override + protected Expression visitSingleColumn(io.trino.sql.tree.SingleColumn node, ParserContext context) { + String alias = node.getAlias().map(io.trino.sql.tree.Identifier::getValue).orElse(null); + Expression expr = visit(node.getExpression(), context, Expression.class); + if (expr instanceof NamedExpression) { + return (NamedExpression) expr; + } else { + return alias == null ? new UnboundAlias(expr) : new UnboundAlias(expr, alias); + } + } + + @Override + protected Object visitIdentifier(io.trino.sql.tree.Identifier node, ParserContext context) { + return new UnboundSlot(ImmutableList.of(node.getValue())); + } + + /* ******************************************************************************************** + * visitFunction + * ******************************************************************************************** */ + + @Override + protected Function visitFunctionCall(io.trino.sql.tree.FunctionCall node, ParserContext context) { + List exprs = visit(node.getArguments(), context, Expression.class); + Function transformedFn = + TrinoFnCallTransformers.transform(node.getName().toString(), exprs, context); + if (transformedFn == null) { + transformedFn = new UnboundFunction(node.getName().toString(), exprs); + + } + return transformedFn; + } + + /* ******************************************************************************************** + * visitTable + * ******************************************************************************************** */ + + @Override + protected LogicalPlan visitTable(io.trino.sql.tree.Table node, ParserContext context) { + io.trino.sql.tree.QualifiedName name = node.getName(); + List tableId = name.getParts(); + // build table + return LogicalPlanBuilderAssistant.withCheckPolicy( + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, + ImmutableList.of(), false)); + } + + /* ******************************************************************************************** + * visit buildIn function + * ******************************************************************************************** */ + + @Override + protected Expression visitCast(io.trino.sql.tree.Cast node, ParserContext context) { + Expression expr = visit(node.getExpression(), context, Expression.class); + DataType dataType = mappingType(node.getType()); + Expression cast = new Cast(expr, dataType); + if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) { + List args = ImmutableList.of( + cast, + new TinyIntLiteral((byte) 1), + Literal.of(((CharacterType) dataType).getLen()) + ); + return new UnboundFunction("substr", args); + } else { + return cast; + } + } + + /* ******************************************************************************************** + * visitLiteral + * ******************************************************************************************** */ + + @Override + protected Object visitLiteral(io.trino.sql.tree.Literal node, ParserContext context) { + // TODO: support literal transform + throw new DialectTransformException("transform literal"); + } + + @Override + protected Literal visitLongLiteral(io.trino.sql.tree.LongLiteral node, ParserContext context) { + return LogicalPlanBuilderAssistant.handleIntegerLiteral(String.valueOf(node.getValue())); + } + + @Override + protected Object visitDoubleLiteral(io.trino.sql.tree.DoubleLiteral node, ParserContext context) { + // TODO: support double literal transform + throw new DialectTransformException("transform double literal"); + } + + @Override + protected Object visitDecimalLiteral(io.trino.sql.tree.DecimalLiteral node, ParserContext context) { + // TODO: support decimal literal transform + throw new DialectTransformException("transform decimal literal"); + } + + @Override + protected Object visitTimestampLiteral(io.trino.sql.tree.TimestampLiteral node, ParserContext context) { + try { + String value = node.getValue(); + if (value.length() <= 10) { + value += " 00:00:00"; + } + return new DateTimeLiteral(value); + } catch (AnalysisException e) { + throw new DialectTransformException("transform timestamp literal"); + } + } + + @Override + protected Object visitGenericLiteral(io.trino.sql.tree.GenericLiteral node, ParserContext context) { + // TODO: support generic literal transform + throw new DialectTransformException("transform generic literal"); + } + + @Override + protected Object visitTimeLiteral(io.trino.sql.tree.TimeLiteral node, ParserContext context) { + // TODO: support time literal transform + throw new DialectTransformException("transform time literal"); + } + + @Override + protected Object visitCharLiteral(io.trino.sql.tree.CharLiteral node, ParserContext context) { + // TODO: support char literal transform + throw new DialectTransformException("transform char literal"); + } + + @Override + protected Expression visitStringLiteral(io.trino.sql.tree.StringLiteral node, ParserContext context) { + // TODO: add unescapeSQLString. + String txt = node.getValue(); + if (txt.length() <= 1) { + return new VarcharLiteral(txt); + } + return new VarcharLiteral(LogicalPlanBuilderAssistant.escapeBackSlash(txt.substring(0, txt.length()))); + } + + @Override + protected Object visitIntervalLiteral(io.trino.sql.tree.IntervalLiteral node, ParserContext context) { + // TODO: support interval literal transform + throw new DialectTransformException("transform char literal"); + } + + @Override + protected Object visitBinaryLiteral(io.trino.sql.tree.BinaryLiteral node, ParserContext context) { + // TODO: support binary literal transform + throw new DialectTransformException("transform binary literal"); + } + + @Override + protected Object visitNullLiteral(io.trino.sql.tree.NullLiteral node, ParserContext context) { + return NullLiteral.INSTANCE; + } + + @Override + protected Object visitBooleanLiteral(io.trino.sql.tree.BooleanLiteral node, ParserContext context) { + return BooleanLiteral.of(node.getValue()); + } + + private DataType mappingType(io.trino.sql.tree.DataType dataType) { + + if (dataType instanceof io.trino.sql.tree.GenericDataType) { + io.trino.sql.tree.GenericDataType genericDataType = (io.trino.sql.tree.GenericDataType) dataType; + String typeName = genericDataType.getName().getValue().toLowerCase(); + List types = Lists.newArrayList(typeName); + + String length = null; + String precision = null; + String scale = null; + List arguments = genericDataType.getArguments(); + if (!arguments.isEmpty()) { + if (arguments.get(0) instanceof io.trino.sql.tree.NumericParameter) { + precision = length = ((io.trino.sql.tree.NumericParameter) arguments.get(0)).getValue(); + } + if (arguments.size() > 1 && arguments.get(1) instanceof io.trino.sql.tree.NumericParameter) { + scale = ((io.trino.sql.tree.NumericParameter) arguments.get(1)).getValue(); + } + } + if ("decimal".equals(typeName)) { + if (precision != null) { + types.add(precision); + } + if (scale != null) { + types.add(scale); + } + } + if ("varchar".equals(typeName) || "char".equals(typeName)) { + if (length != null) { + types.add(length); + } + } + + // unsigned decimal in Trino is longDecimal, not handle now, support it later + if (!"decimal".equals(typeName) && typeName.contains("decimal")) { + throw new DialectTransformException("transform not standard decimal data type "); + } + // Trino only support signed, safe unsigned is false here + return DataType.convertPrimitiveFromStrings(types, false); + } else if (dataType instanceof io.trino.sql.tree.DateTimeDataType) { + // TODO: support date data type mapping + throw new DialectTransformException("transform date data type"); + } + throw new AnalysisException("Nereids do not support type: " + dataType); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformer.java new file mode 100644 index 0000000000..7ed99f4c48 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformer.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +import org.apache.doris.nereids.analyzer.PlaceholderExpression; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.parser.ParserContext; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Trino function transformer + */ +public class TrinoFnCallTransformer extends AbstractFnCallTransformer { + private final UnboundFunction targetFunction; + private final List targetArguments; + private final boolean variableArgument; + private final int sourceArgumentsNum; + + /** + * Trino function transformer, mostly this handle common function. + */ + public TrinoFnCallTransformer(UnboundFunction targetFunction, + boolean variableArgument, + int sourceArgumentsNum) { + this.targetFunction = targetFunction; + this.variableArgument = variableArgument; + this.sourceArgumentsNum = sourceArgumentsNum; + PlaceholderCollector placeHolderCollector = new PlaceholderCollector(variableArgument); + placeHolderCollector.visit(targetFunction, null); + this.targetArguments = placeHolderCollector.getPlaceholderExpressions(); + } + + @Override + protected boolean check(String sourceFnName, + List sourceFnTransformedArguments, + ParserContext context) { + List> sourceFnTransformedArgClazz = sourceFnTransformedArguments.stream() + .map(Expression::getClass) + .collect(Collectors.toList()); + if (variableArgument) { + if (targetArguments.isEmpty()) { + return false; + } + Class targetArgumentClazz = targetArguments.get(0).getDelegateClazz(); + for (Expression argument : sourceFnTransformedArguments) { + if (!targetArgumentClazz.isAssignableFrom(argument.getClass())) { + return false; + } + } + } + if (sourceFnTransformedArguments.size() != sourceArgumentsNum) { + return false; + } + for (int i = 0; i < targetArguments.size(); i++) { + if (!targetArguments.get(i).getDelegateClazz().isAssignableFrom(sourceFnTransformedArgClazz.get(i))) { + return false; + } + } + return true; + } + + @Override + protected Function transform(String sourceFnName, + List sourceFnTransformedArguments, + ParserContext context) { + return targetFunction.withChildren(sourceFnTransformedArguments); + } + + /** + * This is the collector for placeholder expression, which placeholder expression + * identify the expression that we want to use later but current now is not confirmed. + */ + public static final class PlaceholderCollector extends DefaultExpressionVisitor { + + private final List placeholderExpressions = new ArrayList<>(); + private final boolean variableArgument; + + public PlaceholderCollector(boolean variableArgument) { + this.variableArgument = variableArgument; + } + + @Override + public Void visitPlaceholderExpression(PlaceholderExpression placeholderExpression, Void context) { + + if (variableArgument) { + placeholderExpressions.add(placeholderExpression); + return null; + } + placeholderExpressions.set(placeholderExpression.getPosition() - 1, placeholderExpression); + return null; + } + + public List getPlaceholderExpressions() { + return placeholderExpressions; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java new file mode 100644 index 0000000000..a5e7f07fe1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +import org.apache.doris.nereids.analyzer.PlaceholderExpression; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.parser.ParserContext; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.Function; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * The builder and factory for {@link org.apache.doris.nereids.parser.trino.TrinoFnCallTransformer}, + * and supply transform facade ability. + */ +public class TrinoFnCallTransformers { + + private static ImmutableListMultimap TRANSFORMER_MAP; + private static ImmutableListMultimap COMPLEX_TRANSFORMER_MAP; + private static final ImmutableListMultimap.Builder transformerBuilder = + ImmutableListMultimap.builder(); + private static final ImmutableListMultimap.Builder complexTransformerBuilder = + ImmutableListMultimap.builder(); + + static { + registerTransformers(); + registerComplexTransformers(); + } + + private TrinoFnCallTransformers() { + } + + /** + * Function transform facade + */ + public static Function transform(String sourceFnName, List sourceFnTransformedArguments, + ParserContext context) { + List transformers = getTransformers(sourceFnName); + return doTransform(transformers, sourceFnName, sourceFnTransformedArguments, context); + } + + private static Function doTransform(List transformers, + String sourceFnName, + List sourceFnTransformedArguments, + ParserContext context) { + for (AbstractFnCallTransformer transformer : transformers) { + if (transformer.check(sourceFnName, sourceFnTransformedArguments, context)) { + Function transformedFunction = + transformer.transform(sourceFnName, sourceFnTransformedArguments, context); + if (transformedFunction == null) { + continue; + } + return transformedFunction; + } + } + return null; + } + + private static List getTransformers(String sourceFnName) { + ImmutableList fnCallTransformers = + TRANSFORMER_MAP.get(sourceFnName); + ImmutableList complexFnCallTransformers = + COMPLEX_TRANSFORMER_MAP.get(sourceFnName); + return ImmutableList.copyOf(Iterables.concat(fnCallTransformers, complexFnCallTransformers)); + } + + private static void registerTransformers() { + registerStringFunctionTransformer(); + // TODO: add other function transformer + // build transformer map in the end + TRANSFORMER_MAP = transformerBuilder.build(); + } + + private static void registerComplexTransformers() { + DateDiffFnCallTransformer dateDiffFnCallTransformer = new DateDiffFnCallTransformer(); + doRegister(dateDiffFnCallTransformer.getSourceFnName(), dateDiffFnCallTransformer); + // TODO: add other complex function transformer + // build complex transformer map in the end + COMPLEX_TRANSFORMER_MAP = complexTransformerBuilder.build(); + } + + private static void registerStringFunctionTransformer() { + doRegister("codepoint", 1, "ascii", + Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)), false); + // TODO: add other string function transformer + } + + private static void doRegister( + String sourceFnNme, + int sourceFnArgumentsNum, + String targetFnName, + List targetFnArguments, + boolean variableArgument) { + + List castedTargetFnArguments = targetFnArguments + .stream() + .map(each -> (Expression) each) + .collect(Collectors.toList()); + transformerBuilder.put(sourceFnNme, new TrinoFnCallTransformer(new UnboundFunction( + targetFnName, castedTargetFnArguments), variableArgument, sourceFnArgumentsNum)); + } + + private static void doRegister( + String sourceFnNme, + AbstractFnCallTransformer transformer) { + complexTransformerBuilder.put(sourceFnNme, transformer); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java new file mode 100644 index 0000000000..b781bfc47f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +/** + * Trino Parser, depends on 395 trino-parser, and 4.9.3 antlr-runtime + */ +public class TrinoParser { + private static final io.trino.sql.parser.ParsingOptions PARSING_OPTIONS = + new io.trino.sql.parser.ParsingOptions( + io.trino.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL); + + public static io.trino.sql.tree.Statement parse(String query) { + io.trino.sql.parser.SqlParser sqlParser = new io.trino.sql.parser.SqlParser(); + return sqlParser.createStatement(query, PARSING_OPTIONS); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index a275cadd87..01853926ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.expressions.visitor; +import org.apache.doris.nereids.analyzer.PlaceholderExpression; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -501,4 +502,12 @@ public abstract class ExpressionVisitor public R visitUnboundVariable(UnboundVariable unboundVariable, C context) { return visit(unboundVariable, context); } + + /* ******************************************************************************************** + * Placeholder expressions + * ********************************************************************************************/ + + public R visitPlaceholderExpression(PlaceholderExpression placeholderExpression, C context) { + return visit(placeholderExpression, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 1ad2b2b0df..ee93e96919 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -28,6 +28,8 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.nereids.metrics.Event; import org.apache.doris.nereids.metrics.EventSwitchParser; +import org.apache.doris.nereids.parser.ParseDialect; +import org.apache.doris.nereids.parser.ParseDialect.Dialect; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.qe.VariableMgr.VarAttr; import org.apache.doris.thrift.TQueryOptions; @@ -39,6 +41,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.simple.JSONObject; @@ -410,6 +413,8 @@ public class SessionVariable implements Serializable, Writable { public static final String FULL_AUTO_ANALYZE_END_TIME = "full_auto_analyze_end_time"; + public static final String SQL_DIALECT = "sql_dialect"; + public static final String EXPAND_RUNTIME_FILTER_BY_INNER_JION = "expand_runtime_filter_by_inner_join"; public static final String TEST_QUERY_CACHE_HIT = "test_query_cache_hit"; @@ -1211,6 +1216,10 @@ public class SessionVariable implements Serializable, Writable { flag = VariableMgr.GLOBAL) public String fullAutoAnalyzeEndTime = "02:00:00"; + @VariableMgr.VarAttr(name = SQL_DIALECT, needForward = true, checker = "checkSqlDialect", + description = {"解析sql使用的方言", "The dialect used to parse sql."}) + public String sqlDialect = "doris"; + @VariableMgr.VarAttr(name = ENABLE_UNIQUE_KEY_PARTIAL_UPDATE, needForward = true) public boolean enableUniqueKeyPartialUpdate = false; @@ -1933,6 +1942,17 @@ public class SessionVariable implements Serializable, Writable { this.enableOrcLazyMat = enableOrcLazyMat; } + public String getSqlDialect() { + return sqlDialect; + } + + public ParseDialect.Dialect getSqlParseDialect() { + return ParseDialect.Dialect.getByName(sqlDialect); + } + + public void setSqlDialect(String sqlDialect) { + this.sqlDialect = sqlDialect == null ? null : sqlDialect.toLowerCase(); + } /** * getInsertVisibleTimeoutMs. @@ -2699,4 +2719,16 @@ public class SessionVariable implements Serializable, Writable { public boolean fasterFloatConvert() { return this.fasterFloatConvert; } + + public void checkSqlDialect(String sqlDialect) { + if (StringUtils.isEmpty(sqlDialect)) { + LOG.warn("sqlDialect value is empty"); + throw new UnsupportedOperationException("sqlDialect value is empty"); + } + if (Arrays.stream(Dialect.values()) + .noneMatch(dialect -> dialect.getDialectName().equalsIgnoreCase(sqlDialect))) { + LOG.warn("sqlDialect value is invalid, the invalid value is {}", sqlDialect); + throw new UnsupportedOperationException("sqlDialect value is invalid, the invalid value is " + sqlDialect); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index b185c2f5b7..f45d0f5269 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.expressions.Cast; @@ -38,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; +import org.apache.doris.qe.SessionVariable; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -174,6 +176,23 @@ public class NereidsParserTest extends ParserTestBase { Assertions.assertTrue(logicalPlan1 instanceof ExplainCommand); } + @Test + public void testParseSQLWithDialect() { + String sql = "select `AD``D` from t1 where a = 1;explain graph select `AD``D` from t1 where a = 1;"; + NereidsParser nereidsParser = new NereidsParser(); + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setSqlDialect("trino"); + // test fall back to doris parser + List statementBases = nereidsParser.parseSQL(sql, sessionVariable); + Assertions.assertEquals(2, statementBases.size()); + Assertions.assertTrue(statementBases.get(0) instanceof LogicalPlanAdapter); + Assertions.assertTrue(statementBases.get(1) instanceof LogicalPlanAdapter); + LogicalPlan logicalPlan0 = ((LogicalPlanAdapter) statementBases.get(0)).getLogicalPlan(); + LogicalPlan logicalPlan1 = ((LogicalPlanAdapter) statementBases.get(1)).getLogicalPlan(); + Assertions.assertTrue(logicalPlan0 instanceof UnboundResultSink); + Assertions.assertTrue(logicalPlan1 instanceof ExplainCommand); + } + @Test public void testParseJoin() { NereidsParser nereidsParser = new NereidsParser(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java index cb179ef531..a5ba2b4e05 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.parser; import org.apache.doris.nereids.util.ExpressionParseChecker; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanParseChecker; +import org.apache.doris.nereids.util.TrinoDialectPlanParseChecker; /** * Base class to check SQL parsing result. @@ -32,4 +33,8 @@ public abstract class ParserTestBase implements MemoPatternMatchSupported { public ExpressionParseChecker parseExpression(String sql) { return new ExpressionParseChecker(sql); } + + public TrinoDialectPlanParseChecker trinoDialectParsePlan(String sql) { + return new TrinoDialectPlanParseChecker(sql); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/FnTransformTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/FnTransformTest.java new file mode 100644 index 0000000000..eefcf8599d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/FnTransformTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.parser.ParserTestBase; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import org.junit.jupiter.api.Test; + +/** + * Trino to Doris function mapping test. + */ +public class FnTransformTest extends ParserTestBase { + + @Test + public void testStringFnTransform() { + String sql = "SELECT ascii('a') as b FROM t"; + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); + + String dialectSql = "SELECT codepoint('a') as b FROM t"; + trinoDialectParsePlan(dialectSql).assertEquals(logicalPlan); + } + + @Test + public void testDateDiffFnTransform() { + String dialectSql = "SELECT date_diff('second', TIMESTAMP '2020-12-25 22:00:00', TIMESTAMP '2020-12-25 21:00:00')"; + trinoDialectParsePlan(dialectSql).assertContains("seconds_diff(2020-12-25 22:00:00, 2020-12-25 21:00:00)"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/QueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/QueryTest.java new file mode 100644 index 0000000000..f438f4c252 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/QueryTest.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser.trino; + +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.parser.ParserTestBase; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import org.junit.jupiter.api.Test; + +/** + * Trino query tests. + */ +public class QueryTest extends ParserTestBase { + + @Test + public void testParseCast1() { + String sql = "SELECT CAST(1 AS DECIMAL(20, 6)) FROM t"; + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); + trinoDialectParsePlan(sql).assertEquals(logicalPlan); + + sql = "SELECT CAST(a AS DECIMAL(20, 6)) FROM t"; + logicalPlan = nereidsParser.parseSingle(sql); + trinoDialectParsePlan(sql).assertEquals(logicalPlan); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TrinoDialectPlanParseChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TrinoDialectPlanParseChecker.java new file mode 100644 index 0000000000..0217eb7b8b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TrinoDialectPlanParseChecker.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import org.apache.doris.nereids.parser.ParseDialect; +import org.apache.doris.nereids.parser.ParserContext; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Assertions; + +/** + * Plan parse checker for trino. + * It supports equals or contain pattern match assert and so on. + */ +public class TrinoDialectPlanParseChecker extends ParseChecker { + + private final Supplier parsedPlanSupplier; + + public TrinoDialectPlanParseChecker(String sql) { + super(sql); + this.parsedPlanSupplier = + Suppliers.memoize(() -> PARSER.parseSingleWithDialect(sql, new ParserContext(ParseDialect.TRINO_395))); + } + + public TrinoDialectPlanParseChecker assertEquals(LogicalPlan plan) { + LogicalPlan target = parsedPlanSupplier.get(); + Assertions.assertEquals(plan, target); + return this; + } + + public TrinoDialectPlanParseChecker assertContains(String... expects) { + LogicalPlan logicalPlan = parsedPlanSupplier.get(); + Assertions.assertNotNull(logicalPlan); + String targetPlanString = logicalPlan.toString(); + for (String expected : expects) { + Assertions.assertTrue(StringUtils.containsIgnoreCase(targetPlanString.toLowerCase(), expected), + "expected contain is: " + expected + " but plan is \n" + targetPlanString); + } + return this; + } +} diff --git a/fe/pom.xml b/fe/pom.xml index a0dc1ac221..cfb63957bc 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -320,6 +320,7 @@ under the License. 0.4.0-incubating 3.4.4 + 395 shade-format-flatbuffers 1.12.0 @@ -1452,6 +1453,13 @@ under the License. client ${vesoft.client.version} + + + io.trino + trino-parser + ${trino.parser.version} + io.grpc grpc-netty