[feature-wip] (Nereids) Support transforming trino dialect SQL to logical plan (#21855)
Support transforming trino dialect SQL to logical plan (#21854) ## Proposed changes Issue Number: #21854 Use io.trino.sql.tree.AstVisitor as vistor, visit coorresponding trino node and transform it to doris logical plan. ## Further comments Here are some examples for function transforming as following: **ascii('a')** function is in doris and **codepoint('a')** funtion in trino, they have the same feature and have the same method signature, so we can use [TrinoFnCallTransformer](3b37b76886/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformer.java) to handle them. another example for ComplexTransformer as following: **date_diff('second', TIMESTAMP '2020-12-25 22:00:00', TIMESTAMP '2020-12-25 21:00:00')"** fuction in trino and **seconds_diff(2020-12-25 22:00:00, 2020-12-25 21:00:00)")** fuction in doris. They have different method signature, we cant not handle it by TrinoFnCallTransformer simply and we should handle it by individual complex transformer [DateDiffFnCallTransformer](3b37b76886/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java).
This commit is contained in:
@ -0,0 +1,85 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.analyzer;
|
||||
|
||||
import org.apache.doris.nereids.parser.trino.TrinoFnCallTransformer.PlaceholderCollector;
|
||||
import org.apache.doris.nereids.trees.expressions.Expression;
|
||||
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
|
||||
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Expression placeHolder, the expression in PlaceHolderExpression will be collected by
|
||||
*
|
||||
* @see PlaceholderCollector
|
||||
*/
|
||||
public class PlaceholderExpression extends Expression implements AlwaysNotNullable {
|
||||
|
||||
private final Class<? extends Expression> delegateClazz;
|
||||
/**
|
||||
* 1 based
|
||||
*/
|
||||
private final int position;
|
||||
|
||||
public PlaceholderExpression(List<Expression> children, Class<? extends Expression> delegateClazz, int position) {
|
||||
super(children);
|
||||
this.delegateClazz = Objects.requireNonNull(delegateClazz, "delegateClazz should not be null");
|
||||
this.position = position;
|
||||
}
|
||||
|
||||
public static PlaceholderExpression of(Class<? extends Expression> delegateClazz, int position) {
|
||||
return new PlaceholderExpression(ImmutableList.of(), delegateClazz, position);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
|
||||
return visitor.visit(this, context);
|
||||
}
|
||||
|
||||
public Class<? extends Expression> getDelegateClazz() {
|
||||
return delegateClazz;
|
||||
}
|
||||
|
||||
public int getPosition() {
|
||||
return position;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
if (!super.equals(o)) {
|
||||
return false;
|
||||
}
|
||||
PlaceholderExpression that = (PlaceholderExpression) o;
|
||||
return position == that.position && Objects.equals(delegateClazz, that.delegateClazz);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(super.hashCode(), delegateClazz, position);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.exceptions;
|
||||
|
||||
/**
|
||||
* DialectTransformException when have not supported transforming for the
|
||||
* {@link io.trino.sql.tree.Node}.
|
||||
*/
|
||||
public class DialectTransformException extends UnsupportedOperationException {
|
||||
|
||||
public DialectTransformException(String msg) {
|
||||
super(String.format("Unsupported dialect transformation is %s", msg));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.exceptions;
|
||||
|
||||
import org.apache.doris.nereids.parser.ParseDialect;
|
||||
|
||||
/**
|
||||
* UnsupportedDialectException when not match any in
|
||||
* {@link org.apache.doris.nereids.parser.ParseDialect}.
|
||||
*/
|
||||
public class UnsupportedDialectException extends UnsupportedOperationException {
|
||||
|
||||
public UnsupportedDialectException(ParseDialect dialect) {
|
||||
super(String.format("Unsupported dialect name is %s, version is %s",
|
||||
dialect.getDialect().getDialectName(), dialect.getVersion().getVersionName()));
|
||||
}
|
||||
|
||||
public UnsupportedDialectException(String type, String msg) {
|
||||
super(String.format("Unsupported dialect type is %s, msg is %s", type, msg));
|
||||
}
|
||||
}
|
||||
@ -314,7 +314,6 @@ import org.apache.doris.nereids.trees.plans.commands.info.RollupDefinition;
|
||||
import org.apache.doris.nereids.trees.plans.commands.info.StepPartition;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
|
||||
@ -432,7 +431,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
|
||||
|
||||
@Override
|
||||
public LogicalPlan visitUpdate(UpdateContext ctx) {
|
||||
LogicalPlan query = withCheckPolicy(new UnboundRelation(
|
||||
LogicalPlan query = LogicalPlanBuilderAssistant.withCheckPolicy(new UnboundRelation(
|
||||
StatementScopeIdGenerator.newRelationId(), visitMultipartIdentifier(ctx.tableName)));
|
||||
query = withTableAlias(query, ctx.tableAlias());
|
||||
if (ctx.fromClause() != null) {
|
||||
@ -455,7 +454,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
|
||||
public LogicalPlan visitDelete(DeleteContext ctx) {
|
||||
List<String> tableName = visitMultipartIdentifier(ctx.tableName);
|
||||
List<String> partitions = ctx.partition == null ? ImmutableList.of() : visitIdentifierList(ctx.partition);
|
||||
LogicalPlan query = withTableAlias(withCheckPolicy(
|
||||
LogicalPlan query = withTableAlias(LogicalPlanBuilderAssistant.withCheckPolicy(
|
||||
new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableName)), ctx.tableAlias());
|
||||
if (ctx.USING() != null) {
|
||||
query = withRelations(query, ctx.relation());
|
||||
@ -480,7 +479,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
|
||||
|
||||
// handle path string
|
||||
String tmpPath = ctx.filePath.getText();
|
||||
String path = escapeBackSlash(tmpPath.substring(1, tmpPath.length() - 1));
|
||||
String path = LogicalPlanBuilderAssistant.escapeBackSlash(tmpPath.substring(1, tmpPath.length() - 1));
|
||||
|
||||
Optional<Expression> expr = Optional.empty();
|
||||
if (ctx.whereClause() != null) {
|
||||
@ -630,7 +629,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
|
||||
String labelName = ctx.lableName.getText();
|
||||
Map<String, String> properties = visitPropertyItemList(ctx.properties);
|
||||
String commentSpec = ctx.commentSpec() == null ? "" : ctx.commentSpec().STRING_LITERAL().getText();
|
||||
String comment = escapeBackSlash(commentSpec.substring(1, commentSpec.length() - 1));
|
||||
String comment =
|
||||
LogicalPlanBuilderAssistant.escapeBackSlash(commentSpec.substring(1, commentSpec.length() - 1));
|
||||
return new LoadCommand(labelName, dataDescriptions.build(), bulkDesc, properties, comment);
|
||||
}
|
||||
|
||||
@ -855,10 +855,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
|
||||
});
|
||||
}
|
||||
|
||||
private LogicalPlan withCheckPolicy(LogicalPlan plan) {
|
||||
return new LogicalCheckPolicy<>(plan);
|
||||
}
|
||||
|
||||
@Override
|
||||
public LogicalPlan visitTableName(TableNameContext ctx) {
|
||||
List<String> tableId = visitMultipartIdentifier(ctx.multipartIdentifier());
|
||||
@ -888,7 +884,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
|
||||
}
|
||||
|
||||
TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample());
|
||||
LogicalPlan checkedRelation = withCheckPolicy(
|
||||
LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(
|
||||
new UnboundRelation(StatementScopeIdGenerator.newRelationId(),
|
||||
tableId, partitionNames, isTempPart, tabletIdLists, relationHints,
|
||||
Optional.ofNullable(tableSample)));
|
||||
@ -1713,53 +1709,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
|
||||
public Literal visitStringLiteral(StringLiteralContext ctx) {
|
||||
// TODO: add unescapeSQLString.
|
||||
String txt = ctx.STRING_LITERAL().getText();
|
||||
String s = escapeBackSlash(txt.substring(1, txt.length() - 1));
|
||||
String s = LogicalPlanBuilderAssistant.escapeBackSlash(txt.substring(1, txt.length() - 1));
|
||||
return new VarcharLiteral(s);
|
||||
}
|
||||
|
||||
private String escapeBackSlash(String str) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int strLen = str.length();
|
||||
for (int i = 0; i < strLen; ++i) {
|
||||
char c = str.charAt(i);
|
||||
if (c == '\\' && (i + 1) < strLen) {
|
||||
switch (str.charAt(i + 1)) {
|
||||
case 'n':
|
||||
sb.append('\n');
|
||||
break;
|
||||
case 't':
|
||||
sb.append('\t');
|
||||
break;
|
||||
case 'r':
|
||||
sb.append('\r');
|
||||
break;
|
||||
case 'b':
|
||||
sb.append('\b');
|
||||
break;
|
||||
case '0':
|
||||
sb.append('\0'); // Ascii null
|
||||
break;
|
||||
case 'Z': // ^Z must be escaped on Win32
|
||||
sb.append('\032');
|
||||
break;
|
||||
case '_':
|
||||
case '%':
|
||||
sb.append('\\'); // remember prefix for wildcard
|
||||
sb.append(str.charAt(i + 1));
|
||||
break;
|
||||
default:
|
||||
sb.append(str.charAt(i + 1));
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* cast all items to same types.
|
||||
* TODO remove this function after we refactor type coercion.
|
||||
@ -2695,7 +2648,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
|
||||
String comment;
|
||||
if (ctx.commentSpec() != null) {
|
||||
comment = ctx.commentSpec().STRING_LITERAL().getText();
|
||||
comment = escapeBackSlash(comment.substring(1, comment.length() - 1));
|
||||
comment = LogicalPlanBuilderAssistant.escapeBackSlash(comment.substring(1, comment.length() - 1));
|
||||
} else {
|
||||
comment = "";
|
||||
}
|
||||
|
||||
@ -0,0 +1,111 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser;
|
||||
|
||||
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.Literal;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* Logical plan builder assistant for buildIn dialect and other dialect.
|
||||
* The same logical in {@link org.apache.doris.nereids.parser.LogicalPlanBuilder}
|
||||
* and {@link org.apache.doris.nereids.parser.trino.LogicalPlanTrinoBuilder} can be
|
||||
* extracted to here.
|
||||
*/
|
||||
public class LogicalPlanBuilderAssistant {
|
||||
|
||||
private LogicalPlanBuilderAssistant() {
|
||||
}
|
||||
|
||||
/**
|
||||
* EscapeBackSlash such \n, \t
|
||||
*/
|
||||
public static String escapeBackSlash(String str) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int strLen = str.length();
|
||||
for (int i = 0; i < strLen; ++i) {
|
||||
char c = str.charAt(i);
|
||||
if (c == '\\' && (i + 1) < strLen) {
|
||||
switch (str.charAt(i + 1)) {
|
||||
case 'n':
|
||||
sb.append('\n');
|
||||
break;
|
||||
case 't':
|
||||
sb.append('\t');
|
||||
break;
|
||||
case 'r':
|
||||
sb.append('\r');
|
||||
break;
|
||||
case 'b':
|
||||
sb.append('\b');
|
||||
break;
|
||||
case '0':
|
||||
sb.append('\0'); // Ascii null
|
||||
break;
|
||||
case 'Z': // ^Z must be escaped on Win32
|
||||
sb.append('\032');
|
||||
break;
|
||||
case '_':
|
||||
case '%':
|
||||
sb.append('\\'); // remember prefix for wildcard
|
||||
sb.append(str.charAt(i + 1));
|
||||
break;
|
||||
default:
|
||||
sb.append(str.charAt(i + 1));
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
} else {
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Integer literal by BigInteger.
|
||||
*/
|
||||
public static Literal handleIntegerLiteral(String value) {
|
||||
BigInteger bigInt = new BigInteger(value);
|
||||
if (BigInteger.valueOf(bigInt.byteValue()).equals(bigInt)) {
|
||||
return new TinyIntLiteral(bigInt.byteValue());
|
||||
} else if (BigInteger.valueOf(bigInt.shortValue()).equals(bigInt)) {
|
||||
return new SmallIntLiteral(bigInt.shortValue());
|
||||
} else if (BigInteger.valueOf(bigInt.intValue()).equals(bigInt)) {
|
||||
return new IntegerLiteral(bigInt.intValue());
|
||||
} else if (BigInteger.valueOf(bigInt.longValue()).equals(bigInt)) {
|
||||
return new BigIntLiteral(bigInt.longValueExact());
|
||||
} else {
|
||||
return new LargeIntLiteral(bigInt);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap plan withCheckPolicy.
|
||||
*/
|
||||
public static LogicalPlan withCheckPolicy(LogicalPlan plan) {
|
||||
return new LogicalCheckPolicy<>(plan);
|
||||
}
|
||||
}
|
||||
@ -22,10 +22,14 @@ import org.apache.doris.common.Pair;
|
||||
import org.apache.doris.nereids.DorisLexer;
|
||||
import org.apache.doris.nereids.DorisParser;
|
||||
import org.apache.doris.nereids.StatementContext;
|
||||
import org.apache.doris.nereids.exceptions.UnsupportedDialectException;
|
||||
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
|
||||
import org.apache.doris.nereids.parser.trino.LogicalPlanTrinoBuilder;
|
||||
import org.apache.doris.nereids.parser.trino.TrinoParser;
|
||||
import org.apache.doris.nereids.trees.expressions.Expression;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
|
||||
import org.apache.doris.nereids.types.DataType;
|
||||
import org.apache.doris.qe.SessionVariable;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.antlr.v4.runtime.CharStreams;
|
||||
@ -33,15 +37,20 @@ import org.antlr.v4.runtime.CommonTokenStream;
|
||||
import org.antlr.v4.runtime.ParserRuleContext;
|
||||
import org.antlr.v4.runtime.atn.PredictionMode;
|
||||
import org.antlr.v4.runtime.misc.ParseCancellationException;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* Sql parser, convert sql DSL to logical plan.
|
||||
*/
|
||||
public class NereidsParser {
|
||||
public static final Logger LOG = LogManager.getLogger(NereidsParser.class);
|
||||
private static final ParseErrorListener PARSE_ERROR_LISTENER = new ParseErrorListener();
|
||||
private static final PostProcessor POST_PROCESSOR = new PostProcessor();
|
||||
|
||||
@ -58,6 +67,39 @@ public class NereidsParser {
|
||||
return statementBases;
|
||||
}
|
||||
|
||||
/**
|
||||
* ParseSQL with dialect.
|
||||
*/
|
||||
public List<StatementBase> parseSQL(String sql, SessionVariable sessionVariable) {
|
||||
if (ParseDialect.TRINO_395.getDialect().getDialectName()
|
||||
.equalsIgnoreCase(sessionVariable.getSqlDialect())) {
|
||||
return parseSQLWithDialect(sql, sessionVariable);
|
||||
} else {
|
||||
return parseSQL(sql);
|
||||
}
|
||||
}
|
||||
|
||||
private List<StatementBase> parseSQLWithDialect(String sql, SessionVariable sessionVariable) {
|
||||
final List<StatementBase> logicalPlans = new ArrayList<>();
|
||||
try {
|
||||
io.trino.sql.parser.StatementSplitter splitter = new io.trino.sql.parser.StatementSplitter(sql);
|
||||
ParserContext parserContext = new ParserContext(ParseDialect.TRINO_395);
|
||||
StatementContext statementContext = new StatementContext();
|
||||
for (io.trino.sql.parser.StatementSplitter.Statement statement : splitter.getCompleteStatements()) {
|
||||
Object parsedPlan = parseSingleWithDialect(statement.statement(), parserContext);
|
||||
logicalPlans.add(parsedPlan == null
|
||||
? null : new LogicalPlanAdapter((LogicalPlan) parsedPlan, statementContext));
|
||||
}
|
||||
} catch (io.trino.sql.parser.ParsingException | UnsupportedDialectException e) {
|
||||
LOG.debug("Failed to parse logical plan from trino, sql is :{}", sql, e);
|
||||
return parseSQL(sql);
|
||||
}
|
||||
if (logicalPlans.isEmpty() || logicalPlans.stream().anyMatch(Objects::isNull)) {
|
||||
return parseSQL(sql);
|
||||
}
|
||||
return logicalPlans;
|
||||
}
|
||||
|
||||
/**
|
||||
* parse sql DSL string.
|
||||
*
|
||||
@ -90,6 +132,25 @@ public class NereidsParser {
|
||||
return (T) logicalPlanBuilder.visit(tree);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse dialect sql.
|
||||
*
|
||||
* @param sql sql string
|
||||
* @param parserContext parse context
|
||||
* @return logical plan
|
||||
*/
|
||||
public <T> T parseSingleWithDialect(String sql, ParserContext parserContext) {
|
||||
if (ParseDialect.TRINO_395.equals(parserContext.getParserDialect())) {
|
||||
io.trino.sql.tree.Statement statement = TrinoParser.parse(sql);
|
||||
return (T) new LogicalPlanTrinoBuilder().visit(statement, parserContext);
|
||||
} else {
|
||||
LOG.debug("Failed to parse logical plan, the dialect name is {}, version is {}",
|
||||
parserContext.getParserDialect().getDialect().getDialectName(),
|
||||
parserContext.getParserDialect().getVersion());
|
||||
throw new UnsupportedDialectException(parserContext.getParserDialect());
|
||||
}
|
||||
}
|
||||
|
||||
private ParserRuleContext toAst(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
|
||||
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
|
||||
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
|
||||
|
||||
@ -0,0 +1,111 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser;
|
||||
|
||||
/**
|
||||
* ParseDialect enum, maybe support other dialect.
|
||||
*/
|
||||
public enum ParseDialect {
|
||||
|
||||
/**
|
||||
* Trino parser and it's version is 395.
|
||||
*/
|
||||
TRINO_395(Dialect.TRINO, Version.TRINO_395),
|
||||
/**
|
||||
* Doris parser and it's version is 2.0.0.
|
||||
*/
|
||||
DORIS_2_ALL(Dialect.DORIS, Version.DORIS_2_ALL);
|
||||
|
||||
private final Dialect dialect;
|
||||
private final Version version;
|
||||
|
||||
ParseDialect(Dialect dialect, Version version) {
|
||||
this.dialect = dialect;
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
public Version getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
||||
public Dialect getDialect() {
|
||||
return dialect;
|
||||
}
|
||||
|
||||
/**
|
||||
* The version of parse dialect.
|
||||
*/
|
||||
public enum Version {
|
||||
/**
|
||||
* Trino parser and it's version is 395.
|
||||
*/
|
||||
TRINO_395("395"),
|
||||
/**
|
||||
* Doris parser and it's version is 2.0.0.
|
||||
*/
|
||||
DORIS_2_ALL("2.*");
|
||||
private final String version;
|
||||
|
||||
Version(String version) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
public String getVersionName() {
|
||||
return version;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The dialect name of parse dialect.
|
||||
*/
|
||||
public enum Dialect {
|
||||
/**
|
||||
* Trino parser dialect
|
||||
*/
|
||||
TRINO("trino"),
|
||||
/**
|
||||
* Doris parser dialect
|
||||
*/
|
||||
DORIS("doris");
|
||||
|
||||
private String dialectName;
|
||||
|
||||
Dialect(String dialectName) {
|
||||
this.dialectName = dialectName;
|
||||
}
|
||||
|
||||
public String getDialectName() {
|
||||
return dialectName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get dialect by name
|
||||
*/
|
||||
public static Dialect getByName(String dialectName) {
|
||||
if (dialectName == null) {
|
||||
return null;
|
||||
}
|
||||
for (Dialect dialect : Dialect.values()) {
|
||||
if (dialect.getDialectName().equals(dialectName.toLowerCase())) {
|
||||
return dialect;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser;
|
||||
|
||||
/**
|
||||
* SQL parser context, support additional variable to control parse process.
|
||||
*/
|
||||
public class ParserContext {
|
||||
|
||||
private final ParseDialect parseDialect;
|
||||
|
||||
public ParserContext(ParseDialect parseDialect) {
|
||||
this.parseDialect = parseDialect;
|
||||
}
|
||||
|
||||
public ParseDialect getParserDialect() {
|
||||
return parseDialect;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser.trino;
|
||||
|
||||
import org.apache.doris.nereids.parser.ParserContext;
|
||||
import org.apache.doris.nereids.trees.expressions.Expression;
|
||||
import org.apache.doris.nereids.trees.expressions.functions.Function;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Abstract function transformer, dialect function transformer should extend this.
|
||||
*/
|
||||
public abstract class AbstractFnCallTransformer {
|
||||
|
||||
/**
|
||||
* Check source function signature is the same between function from SQL and
|
||||
* definition in function call transformer.
|
||||
* Check the targetArgs param matches the definition in function call transformer.
|
||||
*/
|
||||
protected abstract boolean check(String sourceFnName,
|
||||
List<Expression> sourceFnTransformedArguments,
|
||||
ParserContext context);
|
||||
|
||||
/**
|
||||
* After check, do transform for function mapping.
|
||||
*/
|
||||
protected abstract Function transform(String sourceFnName,
|
||||
List<Expression> sourceFnTransformedArguments,
|
||||
ParserContext context);
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser.trino;
|
||||
|
||||
/**
|
||||
* Trino complex function transformer
|
||||
*/
|
||||
public abstract class ComplexTrinoFnCallTransformer extends AbstractFnCallTransformer {
|
||||
|
||||
protected abstract String getSourceFnName();
|
||||
}
|
||||
@ -0,0 +1,66 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser.trino;
|
||||
|
||||
import org.apache.doris.nereids.analyzer.UnboundFunction;
|
||||
import org.apache.doris.nereids.parser.ParserContext;
|
||||
import org.apache.doris.nereids.trees.expressions.Expression;
|
||||
import org.apache.doris.nereids.trees.expressions.functions.Function;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* DateDiff complex function transformer
|
||||
*/
|
||||
public class DateDiffFnCallTransformer extends ComplexTrinoFnCallTransformer {
|
||||
|
||||
private static final String SECOND = "second";
|
||||
private static final String HOUR = "hour";
|
||||
private static final String DAY = "day";
|
||||
private static final String MILLI_SECOND = "millisecond";
|
||||
|
||||
@Override
|
||||
public String getSourceFnName() {
|
||||
return "date_diff";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean check(String sourceFnName, List<Expression> sourceFnTransformedArguments,
|
||||
ParserContext context) {
|
||||
return getSourceFnName().equalsIgnoreCase(sourceFnName);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Function transform(String sourceFnName, List<Expression> sourceFnTransformedArguments,
|
||||
ParserContext context) {
|
||||
if (sourceFnTransformedArguments.size() != 3) {
|
||||
return null;
|
||||
}
|
||||
VarcharLiteral diffGranularity = (VarcharLiteral) sourceFnTransformedArguments.get(0);
|
||||
if (SECOND.equals(diffGranularity.getValue())) {
|
||||
return new UnboundFunction(
|
||||
"seconds_diff",
|
||||
ImmutableList.of(sourceFnTransformedArguments.get(1), sourceFnTransformedArguments.get(2)));
|
||||
}
|
||||
// TODO: support other date diff granularity
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,325 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser.trino;
|
||||
|
||||
import org.apache.doris.nereids.analyzer.UnboundAlias;
|
||||
import org.apache.doris.nereids.analyzer.UnboundFunction;
|
||||
import org.apache.doris.nereids.analyzer.UnboundOneRowRelation;
|
||||
import org.apache.doris.nereids.analyzer.UnboundRelation;
|
||||
import org.apache.doris.nereids.analyzer.UnboundResultSink;
|
||||
import org.apache.doris.nereids.analyzer.UnboundSlot;
|
||||
import org.apache.doris.nereids.exceptions.AnalysisException;
|
||||
import org.apache.doris.nereids.exceptions.DialectTransformException;
|
||||
import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant;
|
||||
import org.apache.doris.nereids.parser.ParserContext;
|
||||
import org.apache.doris.nereids.trees.expressions.Cast;
|
||||
import org.apache.doris.nereids.trees.expressions.Expression;
|
||||
import org.apache.doris.nereids.trees.expressions.NamedExpression;
|
||||
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
|
||||
import org.apache.doris.nereids.trees.expressions.functions.Function;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.Literal;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
|
||||
import org.apache.doris.nereids.types.DataType;
|
||||
import org.apache.doris.nereids.types.coercion.CharacterType;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* The actually planBuilder for Trino SQL to Doris logical plan.
|
||||
* It depends on {@link io.trino.sql.tree.AstVisitor}
|
||||
*/
|
||||
public class LogicalPlanTrinoBuilder extends io.trino.sql.tree.AstVisitor<Object, ParserContext> {
|
||||
|
||||
public Object visit(io.trino.sql.tree.Node node, ParserContext context) {
|
||||
return this.process(node, context);
|
||||
}
|
||||
|
||||
public <T> T visit(io.trino.sql.tree.Node node, ParserContext context, Class<T> clazz) {
|
||||
return clazz.cast(this.process(node, context));
|
||||
}
|
||||
|
||||
public <T> List<T> visit(List<? extends io.trino.sql.tree.Node> nodes, ParserContext context, Class<T> clazz) {
|
||||
return nodes.stream()
|
||||
.map(node -> clazz.cast(this.process(node, context)))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public Object processOptional(Optional<? extends io.trino.sql.tree.Node> node, ParserContext context) {
|
||||
return node.map(value -> this.process(value, context)).orElse(null);
|
||||
}
|
||||
|
||||
public <T> T processOptional(Optional<? extends io.trino.sql.tree.Node> node,
|
||||
ParserContext context, Class<T> clazz) {
|
||||
return node.map(value -> clazz.cast(this.process(value, context))).orElse(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected LogicalPlan visitQuery(io.trino.sql.tree.Query node, ParserContext context) {
|
||||
io.trino.sql.tree.QueryBody queryBody = node.getQueryBody();
|
||||
LogicalPlan logicalPlan = (LogicalPlan) visit(queryBody, context);
|
||||
if (!(queryBody instanceof io.trino.sql.tree.QuerySpecification)) {
|
||||
// TODO: need to handle orderBy and limit
|
||||
throw new DialectTransformException("transform querySpecification");
|
||||
}
|
||||
return logicalPlan;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected LogicalPlan visitQuerySpecification(io.trino.sql.tree.QuerySpecification node,
|
||||
ParserContext context) {
|
||||
// from -> where -> group by -> having -> select
|
||||
Optional<io.trino.sql.tree.Relation> from = node.getFrom();
|
||||
LogicalPlan fromPlan = processOptional(from, context, LogicalPlan.class);
|
||||
List<io.trino.sql.tree.SelectItem> selectItems = node.getSelect().getSelectItems();
|
||||
if (from == null || !from.isPresent()) {
|
||||
// TODO: support query values
|
||||
List<NamedExpression> expressions = selectItems.stream()
|
||||
.map(item -> visit(item, context, NamedExpression.class))
|
||||
.collect(ImmutableList.toImmutableList());
|
||||
return new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), expressions);
|
||||
}
|
||||
// TODO: support predicate, aggregate, having, order by, limit
|
||||
// TODO: support distinct
|
||||
boolean isDistinct = node.getSelect().isDistinct();
|
||||
return new UnboundResultSink<>(withProjection(selectItems, fromPlan, isDistinct, context));
|
||||
}
|
||||
|
||||
private LogicalProject withProjection(List<io.trino.sql.tree.SelectItem> selectItems,
|
||||
LogicalPlan input,
|
||||
boolean isDistinct,
|
||||
ParserContext context) {
|
||||
List<NamedExpression> expressions = selectItems.stream()
|
||||
.map(item -> visit(item, context, NamedExpression.class))
|
||||
.collect(Collectors.toList());
|
||||
return new LogicalProject(expressions, ImmutableList.of(), isDistinct, input);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Expression visitSingleColumn(io.trino.sql.tree.SingleColumn node, ParserContext context) {
|
||||
String alias = node.getAlias().map(io.trino.sql.tree.Identifier::getValue).orElse(null);
|
||||
Expression expr = visit(node.getExpression(), context, Expression.class);
|
||||
if (expr instanceof NamedExpression) {
|
||||
return (NamedExpression) expr;
|
||||
} else {
|
||||
return alias == null ? new UnboundAlias(expr) : new UnboundAlias(expr, alias);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitIdentifier(io.trino.sql.tree.Identifier node, ParserContext context) {
|
||||
return new UnboundSlot(ImmutableList.of(node.getValue()));
|
||||
}
|
||||
|
||||
/* ********************************************************************************************
|
||||
* visitFunction
|
||||
* ******************************************************************************************** */
|
||||
|
||||
@Override
|
||||
protected Function visitFunctionCall(io.trino.sql.tree.FunctionCall node, ParserContext context) {
|
||||
List<Expression> exprs = visit(node.getArguments(), context, Expression.class);
|
||||
Function transformedFn =
|
||||
TrinoFnCallTransformers.transform(node.getName().toString(), exprs, context);
|
||||
if (transformedFn == null) {
|
||||
transformedFn = new UnboundFunction(node.getName().toString(), exprs);
|
||||
|
||||
}
|
||||
return transformedFn;
|
||||
}
|
||||
|
||||
/* ********************************************************************************************
|
||||
* visitTable
|
||||
* ******************************************************************************************** */
|
||||
|
||||
@Override
|
||||
protected LogicalPlan visitTable(io.trino.sql.tree.Table node, ParserContext context) {
|
||||
io.trino.sql.tree.QualifiedName name = node.getName();
|
||||
List<String> tableId = name.getParts();
|
||||
// build table
|
||||
return LogicalPlanBuilderAssistant.withCheckPolicy(
|
||||
new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId,
|
||||
ImmutableList.of(), false));
|
||||
}
|
||||
|
||||
/* ********************************************************************************************
|
||||
* visit buildIn function
|
||||
* ******************************************************************************************** */
|
||||
|
||||
@Override
|
||||
protected Expression visitCast(io.trino.sql.tree.Cast node, ParserContext context) {
|
||||
Expression expr = visit(node.getExpression(), context, Expression.class);
|
||||
DataType dataType = mappingType(node.getType());
|
||||
Expression cast = new Cast(expr, dataType);
|
||||
if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) {
|
||||
List<Expression> args = ImmutableList.of(
|
||||
cast,
|
||||
new TinyIntLiteral((byte) 1),
|
||||
Literal.of(((CharacterType) dataType).getLen())
|
||||
);
|
||||
return new UnboundFunction("substr", args);
|
||||
} else {
|
||||
return cast;
|
||||
}
|
||||
}
|
||||
|
||||
/* ********************************************************************************************
|
||||
* visitLiteral
|
||||
* ******************************************************************************************** */
|
||||
|
||||
@Override
|
||||
protected Object visitLiteral(io.trino.sql.tree.Literal node, ParserContext context) {
|
||||
// TODO: support literal transform
|
||||
throw new DialectTransformException("transform literal");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Literal visitLongLiteral(io.trino.sql.tree.LongLiteral node, ParserContext context) {
|
||||
return LogicalPlanBuilderAssistant.handleIntegerLiteral(String.valueOf(node.getValue()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitDoubleLiteral(io.trino.sql.tree.DoubleLiteral node, ParserContext context) {
|
||||
// TODO: support double literal transform
|
||||
throw new DialectTransformException("transform double literal");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitDecimalLiteral(io.trino.sql.tree.DecimalLiteral node, ParserContext context) {
|
||||
// TODO: support decimal literal transform
|
||||
throw new DialectTransformException("transform decimal literal");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitTimestampLiteral(io.trino.sql.tree.TimestampLiteral node, ParserContext context) {
|
||||
try {
|
||||
String value = node.getValue();
|
||||
if (value.length() <= 10) {
|
||||
value += " 00:00:00";
|
||||
}
|
||||
return new DateTimeLiteral(value);
|
||||
} catch (AnalysisException e) {
|
||||
throw new DialectTransformException("transform timestamp literal");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitGenericLiteral(io.trino.sql.tree.GenericLiteral node, ParserContext context) {
|
||||
// TODO: support generic literal transform
|
||||
throw new DialectTransformException("transform generic literal");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitTimeLiteral(io.trino.sql.tree.TimeLiteral node, ParserContext context) {
|
||||
// TODO: support time literal transform
|
||||
throw new DialectTransformException("transform time literal");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitCharLiteral(io.trino.sql.tree.CharLiteral node, ParserContext context) {
|
||||
// TODO: support char literal transform
|
||||
throw new DialectTransformException("transform char literal");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Expression visitStringLiteral(io.trino.sql.tree.StringLiteral node, ParserContext context) {
|
||||
// TODO: add unescapeSQLString.
|
||||
String txt = node.getValue();
|
||||
if (txt.length() <= 1) {
|
||||
return new VarcharLiteral(txt);
|
||||
}
|
||||
return new VarcharLiteral(LogicalPlanBuilderAssistant.escapeBackSlash(txt.substring(0, txt.length())));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitIntervalLiteral(io.trino.sql.tree.IntervalLiteral node, ParserContext context) {
|
||||
// TODO: support interval literal transform
|
||||
throw new DialectTransformException("transform char literal");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitBinaryLiteral(io.trino.sql.tree.BinaryLiteral node, ParserContext context) {
|
||||
// TODO: support binary literal transform
|
||||
throw new DialectTransformException("transform binary literal");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitNullLiteral(io.trino.sql.tree.NullLiteral node, ParserContext context) {
|
||||
return NullLiteral.INSTANCE;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object visitBooleanLiteral(io.trino.sql.tree.BooleanLiteral node, ParserContext context) {
|
||||
return BooleanLiteral.of(node.getValue());
|
||||
}
|
||||
|
||||
private DataType mappingType(io.trino.sql.tree.DataType dataType) {
|
||||
|
||||
if (dataType instanceof io.trino.sql.tree.GenericDataType) {
|
||||
io.trino.sql.tree.GenericDataType genericDataType = (io.trino.sql.tree.GenericDataType) dataType;
|
||||
String typeName = genericDataType.getName().getValue().toLowerCase();
|
||||
List<String> types = Lists.newArrayList(typeName);
|
||||
|
||||
String length = null;
|
||||
String precision = null;
|
||||
String scale = null;
|
||||
List<io.trino.sql.tree.DataTypeParameter> arguments = genericDataType.getArguments();
|
||||
if (!arguments.isEmpty()) {
|
||||
if (arguments.get(0) instanceof io.trino.sql.tree.NumericParameter) {
|
||||
precision = length = ((io.trino.sql.tree.NumericParameter) arguments.get(0)).getValue();
|
||||
}
|
||||
if (arguments.size() > 1 && arguments.get(1) instanceof io.trino.sql.tree.NumericParameter) {
|
||||
scale = ((io.trino.sql.tree.NumericParameter) arguments.get(1)).getValue();
|
||||
}
|
||||
}
|
||||
if ("decimal".equals(typeName)) {
|
||||
if (precision != null) {
|
||||
types.add(precision);
|
||||
}
|
||||
if (scale != null) {
|
||||
types.add(scale);
|
||||
}
|
||||
}
|
||||
if ("varchar".equals(typeName) || "char".equals(typeName)) {
|
||||
if (length != null) {
|
||||
types.add(length);
|
||||
}
|
||||
}
|
||||
|
||||
// unsigned decimal in Trino is longDecimal, not handle now, support it later
|
||||
if (!"decimal".equals(typeName) && typeName.contains("decimal")) {
|
||||
throw new DialectTransformException("transform not standard decimal data type ");
|
||||
}
|
||||
// Trino only support signed, safe unsigned is false here
|
||||
return DataType.convertPrimitiveFromStrings(types, false);
|
||||
} else if (dataType instanceof io.trino.sql.tree.DateTimeDataType) {
|
||||
// TODO: support date data type mapping
|
||||
throw new DialectTransformException("transform date data type");
|
||||
}
|
||||
throw new AnalysisException("Nereids do not support type: " + dataType);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,118 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser.trino;
|
||||
|
||||
import org.apache.doris.nereids.analyzer.PlaceholderExpression;
|
||||
import org.apache.doris.nereids.analyzer.UnboundFunction;
|
||||
import org.apache.doris.nereids.parser.ParserContext;
|
||||
import org.apache.doris.nereids.trees.expressions.Expression;
|
||||
import org.apache.doris.nereids.trees.expressions.functions.Function;
|
||||
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Trino function transformer
|
||||
*/
|
||||
public class TrinoFnCallTransformer extends AbstractFnCallTransformer {
|
||||
private final UnboundFunction targetFunction;
|
||||
private final List<PlaceholderExpression> targetArguments;
|
||||
private final boolean variableArgument;
|
||||
private final int sourceArgumentsNum;
|
||||
|
||||
/**
|
||||
* Trino function transformer, mostly this handle common function.
|
||||
*/
|
||||
public TrinoFnCallTransformer(UnboundFunction targetFunction,
|
||||
boolean variableArgument,
|
||||
int sourceArgumentsNum) {
|
||||
this.targetFunction = targetFunction;
|
||||
this.variableArgument = variableArgument;
|
||||
this.sourceArgumentsNum = sourceArgumentsNum;
|
||||
PlaceholderCollector placeHolderCollector = new PlaceholderCollector(variableArgument);
|
||||
placeHolderCollector.visit(targetFunction, null);
|
||||
this.targetArguments = placeHolderCollector.getPlaceholderExpressions();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean check(String sourceFnName,
|
||||
List<Expression> sourceFnTransformedArguments,
|
||||
ParserContext context) {
|
||||
List<Class<? extends Expression>> sourceFnTransformedArgClazz = sourceFnTransformedArguments.stream()
|
||||
.map(Expression::getClass)
|
||||
.collect(Collectors.toList());
|
||||
if (variableArgument) {
|
||||
if (targetArguments.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
Class<? extends Expression> targetArgumentClazz = targetArguments.get(0).getDelegateClazz();
|
||||
for (Expression argument : sourceFnTransformedArguments) {
|
||||
if (!targetArgumentClazz.isAssignableFrom(argument.getClass())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sourceFnTransformedArguments.size() != sourceArgumentsNum) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < targetArguments.size(); i++) {
|
||||
if (!targetArguments.get(i).getDelegateClazz().isAssignableFrom(sourceFnTransformedArgClazz.get(i))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Function transform(String sourceFnName,
|
||||
List<Expression> sourceFnTransformedArguments,
|
||||
ParserContext context) {
|
||||
return targetFunction.withChildren(sourceFnTransformedArguments);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is the collector for placeholder expression, which placeholder expression
|
||||
* identify the expression that we want to use later but current now is not confirmed.
|
||||
*/
|
||||
public static final class PlaceholderCollector extends DefaultExpressionVisitor<Void, Void> {
|
||||
|
||||
private final List<PlaceholderExpression> placeholderExpressions = new ArrayList<>();
|
||||
private final boolean variableArgument;
|
||||
|
||||
public PlaceholderCollector(boolean variableArgument) {
|
||||
this.variableArgument = variableArgument;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Void visitPlaceholderExpression(PlaceholderExpression placeholderExpression, Void context) {
|
||||
|
||||
if (variableArgument) {
|
||||
placeholderExpressions.add(placeholderExpression);
|
||||
return null;
|
||||
}
|
||||
placeholderExpressions.set(placeholderExpression.getPosition() - 1, placeholderExpression);
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<PlaceholderExpression> getPlaceholderExpressions() {
|
||||
return placeholderExpressions;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,130 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser.trino;
|
||||
|
||||
import org.apache.doris.nereids.analyzer.PlaceholderExpression;
|
||||
import org.apache.doris.nereids.analyzer.UnboundFunction;
|
||||
import org.apache.doris.nereids.parser.ParserContext;
|
||||
import org.apache.doris.nereids.trees.expressions.Expression;
|
||||
import org.apache.doris.nereids.trees.expressions.functions.Function;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableListMultimap;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* The builder and factory for {@link org.apache.doris.nereids.parser.trino.TrinoFnCallTransformer},
|
||||
* and supply transform facade ability.
|
||||
*/
|
||||
public class TrinoFnCallTransformers {
|
||||
|
||||
private static ImmutableListMultimap<String, AbstractFnCallTransformer> TRANSFORMER_MAP;
|
||||
private static ImmutableListMultimap<String, AbstractFnCallTransformer> COMPLEX_TRANSFORMER_MAP;
|
||||
private static final ImmutableListMultimap.Builder<String, AbstractFnCallTransformer> transformerBuilder =
|
||||
ImmutableListMultimap.builder();
|
||||
private static final ImmutableListMultimap.Builder<String, AbstractFnCallTransformer> complexTransformerBuilder =
|
||||
ImmutableListMultimap.builder();
|
||||
|
||||
static {
|
||||
registerTransformers();
|
||||
registerComplexTransformers();
|
||||
}
|
||||
|
||||
private TrinoFnCallTransformers() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Function transform facade
|
||||
*/
|
||||
public static Function transform(String sourceFnName, List<Expression> sourceFnTransformedArguments,
|
||||
ParserContext context) {
|
||||
List<AbstractFnCallTransformer> transformers = getTransformers(sourceFnName);
|
||||
return doTransform(transformers, sourceFnName, sourceFnTransformedArguments, context);
|
||||
}
|
||||
|
||||
private static Function doTransform(List<AbstractFnCallTransformer> transformers,
|
||||
String sourceFnName,
|
||||
List<Expression> sourceFnTransformedArguments,
|
||||
ParserContext context) {
|
||||
for (AbstractFnCallTransformer transformer : transformers) {
|
||||
if (transformer.check(sourceFnName, sourceFnTransformedArguments, context)) {
|
||||
Function transformedFunction =
|
||||
transformer.transform(sourceFnName, sourceFnTransformedArguments, context);
|
||||
if (transformedFunction == null) {
|
||||
continue;
|
||||
}
|
||||
return transformedFunction;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static List<AbstractFnCallTransformer> getTransformers(String sourceFnName) {
|
||||
ImmutableList<AbstractFnCallTransformer> fnCallTransformers =
|
||||
TRANSFORMER_MAP.get(sourceFnName);
|
||||
ImmutableList<AbstractFnCallTransformer> complexFnCallTransformers =
|
||||
COMPLEX_TRANSFORMER_MAP.get(sourceFnName);
|
||||
return ImmutableList.copyOf(Iterables.concat(fnCallTransformers, complexFnCallTransformers));
|
||||
}
|
||||
|
||||
private static void registerTransformers() {
|
||||
registerStringFunctionTransformer();
|
||||
// TODO: add other function transformer
|
||||
// build transformer map in the end
|
||||
TRANSFORMER_MAP = transformerBuilder.build();
|
||||
}
|
||||
|
||||
private static void registerComplexTransformers() {
|
||||
DateDiffFnCallTransformer dateDiffFnCallTransformer = new DateDiffFnCallTransformer();
|
||||
doRegister(dateDiffFnCallTransformer.getSourceFnName(), dateDiffFnCallTransformer);
|
||||
// TODO: add other complex function transformer
|
||||
// build complex transformer map in the end
|
||||
COMPLEX_TRANSFORMER_MAP = complexTransformerBuilder.build();
|
||||
}
|
||||
|
||||
private static void registerStringFunctionTransformer() {
|
||||
doRegister("codepoint", 1, "ascii",
|
||||
Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)), false);
|
||||
// TODO: add other string function transformer
|
||||
}
|
||||
|
||||
private static void doRegister(
|
||||
String sourceFnNme,
|
||||
int sourceFnArgumentsNum,
|
||||
String targetFnName,
|
||||
List<? extends Expression> targetFnArguments,
|
||||
boolean variableArgument) {
|
||||
|
||||
List<Expression> castedTargetFnArguments = targetFnArguments
|
||||
.stream()
|
||||
.map(each -> (Expression) each)
|
||||
.collect(Collectors.toList());
|
||||
transformerBuilder.put(sourceFnNme, new TrinoFnCallTransformer(new UnboundFunction(
|
||||
targetFnName, castedTargetFnArguments), variableArgument, sourceFnArgumentsNum));
|
||||
}
|
||||
|
||||
private static void doRegister(
|
||||
String sourceFnNme,
|
||||
AbstractFnCallTransformer transformer) {
|
||||
complexTransformerBuilder.put(sourceFnNme, transformer);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.nereids.parser.trino;
|
||||
|
||||
/**
|
||||
* Trino Parser, depends on 395 trino-parser, and 4.9.3 antlr-runtime
|
||||
*/
|
||||
public class TrinoParser {
|
||||
private static final io.trino.sql.parser.ParsingOptions PARSING_OPTIONS =
|
||||
new io.trino.sql.parser.ParsingOptions(
|
||||
io.trino.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL);
|
||||
|
||||
public static io.trino.sql.tree.Statement parse(String query) {
|
||||
io.trino.sql.parser.SqlParser sqlParser = new io.trino.sql.parser.SqlParser();
|
||||
return sqlParser.createStatement(query, PARSING_OPTIONS);
|
||||
}
|
||||
}
|
||||
@ -17,6 +17,7 @@
|
||||
|
||||
package org.apache.doris.nereids.trees.expressions.visitor;
|
||||
|
||||
import org.apache.doris.nereids.analyzer.PlaceholderExpression;
|
||||
import org.apache.doris.nereids.analyzer.UnboundAlias;
|
||||
import org.apache.doris.nereids.analyzer.UnboundFunction;
|
||||
import org.apache.doris.nereids.analyzer.UnboundSlot;
|
||||
@ -501,4 +502,12 @@ public abstract class ExpressionVisitor<R, C>
|
||||
public R visitUnboundVariable(UnboundVariable unboundVariable, C context) {
|
||||
return visit(unboundVariable, context);
|
||||
}
|
||||
|
||||
/* ********************************************************************************************
|
||||
* Placeholder expressions
|
||||
* ********************************************************************************************/
|
||||
|
||||
public R visitPlaceholderExpression(PlaceholderExpression placeholderExpression, C context) {
|
||||
return visit(placeholderExpression, context);
|
||||
}
|
||||
}
|
||||
|
||||
@ -28,6 +28,8 @@ import org.apache.doris.common.io.Writable;
|
||||
import org.apache.doris.common.util.TimeUtils;
|
||||
import org.apache.doris.nereids.metrics.Event;
|
||||
import org.apache.doris.nereids.metrics.EventSwitchParser;
|
||||
import org.apache.doris.nereids.parser.ParseDialect;
|
||||
import org.apache.doris.nereids.parser.ParseDialect.Dialect;
|
||||
import org.apache.doris.nereids.rules.RuleType;
|
||||
import org.apache.doris.qe.VariableMgr.VarAttr;
|
||||
import org.apache.doris.thrift.TQueryOptions;
|
||||
@ -39,6 +41,7 @@ import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.json.simple.JSONObject;
|
||||
@ -410,6 +413,8 @@ public class SessionVariable implements Serializable, Writable {
|
||||
|
||||
public static final String FULL_AUTO_ANALYZE_END_TIME = "full_auto_analyze_end_time";
|
||||
|
||||
public static final String SQL_DIALECT = "sql_dialect";
|
||||
|
||||
public static final String EXPAND_RUNTIME_FILTER_BY_INNER_JION = "expand_runtime_filter_by_inner_join";
|
||||
|
||||
public static final String TEST_QUERY_CACHE_HIT = "test_query_cache_hit";
|
||||
@ -1211,6 +1216,10 @@ public class SessionVariable implements Serializable, Writable {
|
||||
flag = VariableMgr.GLOBAL)
|
||||
public String fullAutoAnalyzeEndTime = "02:00:00";
|
||||
|
||||
@VariableMgr.VarAttr(name = SQL_DIALECT, needForward = true, checker = "checkSqlDialect",
|
||||
description = {"解析sql使用的方言", "The dialect used to parse sql."})
|
||||
public String sqlDialect = "doris";
|
||||
|
||||
@VariableMgr.VarAttr(name = ENABLE_UNIQUE_KEY_PARTIAL_UPDATE, needForward = true)
|
||||
public boolean enableUniqueKeyPartialUpdate = false;
|
||||
|
||||
@ -1933,6 +1942,17 @@ public class SessionVariable implements Serializable, Writable {
|
||||
this.enableOrcLazyMat = enableOrcLazyMat;
|
||||
}
|
||||
|
||||
public String getSqlDialect() {
|
||||
return sqlDialect;
|
||||
}
|
||||
|
||||
public ParseDialect.Dialect getSqlParseDialect() {
|
||||
return ParseDialect.Dialect.getByName(sqlDialect);
|
||||
}
|
||||
|
||||
public void setSqlDialect(String sqlDialect) {
|
||||
this.sqlDialect = sqlDialect == null ? null : sqlDialect.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* getInsertVisibleTimeoutMs.
|
||||
@ -2699,4 +2719,16 @@ public class SessionVariable implements Serializable, Writable {
|
||||
public boolean fasterFloatConvert() {
|
||||
return this.fasterFloatConvert;
|
||||
}
|
||||
|
||||
public void checkSqlDialect(String sqlDialect) {
|
||||
if (StringUtils.isEmpty(sqlDialect)) {
|
||||
LOG.warn("sqlDialect value is empty");
|
||||
throw new UnsupportedOperationException("sqlDialect value is empty");
|
||||
}
|
||||
if (Arrays.stream(Dialect.values())
|
||||
.noneMatch(dialect -> dialect.getDialectName().equalsIgnoreCase(sqlDialect))) {
|
||||
LOG.warn("sqlDialect value is invalid, the invalid value is {}", sqlDialect);
|
||||
throw new UnsupportedOperationException("sqlDialect value is invalid, the invalid value is " + sqlDialect);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user