database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Refactor](dialect) Add sql dialect converter plugins (#28890)

Browse Source 
The current logic for SQL dialect conversion is all in the `fe-core` module, which may lead to the following issues:
- Changes to the dialect conversion logic may occur frequently, requiring users to upgrade the Doris version frequently within the fe-core module, leading to a longer change cycle.
- The cost of customized development is high, requiring users to replace the fe-core JAR package.

Turning it into a plugin can address the above issues properly.
This commit is contained in:
Xiangyu Wang
2024-01-08 22:56:17 +08:00
committed by yiguolei
parent 28dca66c06
commit 2ca90b2bf1
54 changed files with 1599 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
View File

@ -27,8 +27,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.parser.ParseDialect;
import org.apache.doris.nereids.parser.spark.SparkSql3LogicalPlanBuilder;
import org.apache.doris.nereids.parser.Dialect;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.thrift.TNullSide;
@ -50,6 +49,8 @@ import java.util.Set;
public class InlineViewRef extends TableRef {
private static final Logger LOG = LogManager.getLogger(InlineViewRef.class);
private static final String DEFAULT_TABLE_ALIAS_FOR_SPARK_SQL = "__auto_generated_subquery_name";
// Catalog or local view that is referenced.
// Null for inline views parsed directly from a query string.
private final View view;
@ -198,12 +199,12 @@ public class InlineViewRef extends TableRef {
if (view == null && !hasExplicitAlias()) {
String dialect = ConnectContext.get().getSessionVariable().getSqlDialect();
ParseDialect.Dialect sqlDialect = ParseDialect.Dialect.getByName(dialect);
if (ParseDialect.Dialect.SPARK_SQL != sqlDialect) {
Dialect sqlDialect = Dialect.getByName(dialect);
if (Dialect.SPARK_SQL != sqlDialect) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DERIVED_MUST_HAVE_ALIAS);
}
hasExplicitAlias = true;
aliases = new String[] { SparkSql3LogicalPlanBuilder.DEFAULT_TABLE_ALIAS };
aliases = new String[] { DEFAULT_TABLE_ALIAS_FOR_SPARK_SQL };
}
// Analyze the inline view query statement with its own analyzer

183
fe/fe-core/src/main/java/org/apache/doris/common/util/SQLDialectUtils.java
View File

@ -1,183 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.util;
import org.apache.doris.common.Config;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.nereids.parser.ParseDialect;
import org.apache.doris.qe.ConnectContext;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import lombok.Data;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.lang.reflect.Type;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
/**
* This class is used to convert sql with different dialects
* using sql convertor service.
* The sql convertor service is a http service which is used to convert sql.
* Request body:
* {
* "version": "v1",
* "sql": "select * from t",
* "from": "presto",
* "to": "doris",
* "source": "text",
* "case_sensitive": "0"
* }
* <p>
* Response body:
* {
* "version": "v1",
* "data": "select * from t",
* "code": 0,
* "message": ""
*/
public class SQLDialectUtils {
private static final Logger LOG = LogManager.getLogger(SQLDialectUtils.class);
public static String convertStmtWithDialect(String originStmt, ConnectContext ctx, MysqlCommand mysqlCommand) {
if (mysqlCommand != MysqlCommand.COM_QUERY) {
return originStmt;
}
if (Config.sql_convertor_service.isEmpty()) {
return originStmt;
}
ParseDialect.Dialect dialect = ctx.getSessionVariable().getSqlParseDialect();
if (dialect == null) {
return originStmt;
}
switch (dialect) {
case PRESTO:
return convertStmtWithPresto(originStmt);
default:
LOG.debug("only support presto dialect now.");
return originStmt;
}
}
private static String convertStmtWithPresto(String originStmt) {
String targetURL = Config.sql_convertor_service;
ConvertRequest convertRequest = new ConvertRequest(originStmt, "presto");
HttpURLConnection connection = null;
try {
URL url = new URL(targetURL);
connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("POST");
connection.setRequestProperty("Content-Type", "application/json");
connection.setUseCaches(false);
connection.setDoOutput(true);
String requestStr = convertRequest.toJson();
try (OutputStream outputStream = connection.getOutputStream()) {
outputStream.write(requestStr.getBytes(StandardCharsets.UTF_8));
}
int responseCode = connection.getResponseCode();
LOG.debug("POST Response Code: {}, post data: {}", responseCode, requestStr);
if (responseCode == HttpURLConnection.HTTP_OK) {
try (InputStreamReader inputStreamReader
= new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8);
BufferedReader in = new BufferedReader(inputStreamReader)) {
String inputLine;
StringBuilder response = new StringBuilder();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
Type type = new TypeToken<ConvertResponse>() {
}.getType();
ConvertResponse result = new Gson().fromJson(response.toString(), type);
LOG.debug("convert response: {}", result);
if (result.code == 0) {
if (!"v1".equals(result.version)) {
LOG.warn("failed to convert sql, response version is not v1: {}", result.version);
return originStmt;
}
return result.data;
} else {
LOG.warn("failed to convert sql, response: {}", result);
return originStmt;
}
}
} else {
LOG.warn("failed to convert sql, response code: {}", responseCode);
return originStmt;
}
} catch (Exception e) {
LOG.warn("failed to convert sql", e);
return originStmt;
} finally {
if (connection != null) {
connection.disconnect();
}
}
}
@Data
private static class ConvertRequest {
private String version; // CHECKSTYLE IGNORE THIS LINE
private String sql_query; // CHECKSTYLE IGNORE THIS LINE
private String from; // CHECKSTYLE IGNORE THIS LINE
private String to; // CHECKSTYLE IGNORE THIS LINE
private String source; // CHECKSTYLE IGNORE THIS LINE
private String case_sensitive; // CHECKSTYLE IGNORE THIS LINE
public ConvertRequest(String originStmt, String dialect) {
this.version = "v1";
this.sql_query = originStmt;
this.from = dialect;
this.to = "doris";
this.source = "text";
this.case_sensitive = "0";
}
public String toJson() {
return new Gson().toJson(this);
}
}
@Data
private static class ConvertResponse {
private String version; // CHECKSTYLE IGNORE THIS LINE
private String data; // CHECKSTYLE IGNORE THIS LINE
private int code; // CHECKSTYLE IGNORE THIS LINE
private String message; // CHECKSTYLE IGNORE THIS LINE
public String toJson() {
return new Gson().toJson(this);
}
@Override
public String toString() {
return toJson();
}
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java
View File

@ -18,8 +18,7 @@
package org.apache.doris.nereids.exceptions;
/**
* DialectTransformException when have not supported transforming for the
* {@link io.trino.sql.tree.Node}.
* DialectTransformException when have not supported transforming for dialect converters.
*/
public class DialectTransformException extends UnsupportedOperationException {

9
fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java
View File

@ -17,17 +17,16 @@
package org.apache.doris.nereids.exceptions;
import org.apache.doris.nereids.parser.ParseDialect;
import org.apache.doris.nereids.parser.Dialect;
/**
* UnsupportedDialectException when not match any in
* {@link org.apache.doris.nereids.parser.ParseDialect}.
* {@link Dialect}.
*/
public class UnsupportedDialectException extends UnsupportedOperationException {
public UnsupportedDialectException(ParseDialect dialect) {
super(String.format("Unsupported dialect name is %s, version is %s",
dialect.getDialect().getDialectName(), dialect.getVersion().getVersionName()));
public UnsupportedDialectException(Dialect dialect) {
super(String.format("Unsupported dialect name is %s", dialect.getDialectName()));
}
public UnsupportedDialectException(String type, String msg) {

2
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ComplexFnCallTransformer.java
View File

@ -18,7 +18,7 @@
package org.apache.doris.nereids.parser;
/**
* Trino complex function transformer
* Complex complex function transformer
*/
public abstract class ComplexFnCallTransformer extends AbstractFnCallTransformer {

101
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/Dialect.java Normal file
View File

@ -0,0 +1,101 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser;
import javax.annotation.Nullable;
/**
* ParseDialect enum, maybe support other dialect.
*/
public enum Dialect {
/**
* Doris parser dialect
*/
DORIS("doris"),
/**
* Trino parser dialect
*/
TRINO("trino"),
/**
* Presto parser dialect
*/
PRESTO("presto"),
/**
* Spark sql parser dialect
*/
SPARK_SQL("spark_sql"),
/**
* Hive parser dialect
*/
HIVE("hive"),
/**
* Alibaba max compute parser dialect
*/
MAX_COMPUTE("max_compute"),
/**
* Mysql parser dialect
*/
MYSQL("mysql"),
/**
* Postgresql parser dialect
*/
POSTGRESQL("postgresql"),
/**
* Sqlserver parser dialect
*/
SQLSERVER("sqlserver"),
/**
* Clickhouse parser dialect
*/
CLICKHOUSE("clickhouse"),
/**
* Sap hana parser dialect
*/
SAP_HANA("sap_hana"),
/**
* OceanBase parser dialect
*/
OCEANBASE("oceanbase");
public static final int MAX_DIALECT_SIZE = Dialect.values().length;
private final String dialectName;
Dialect(String dialectName) {
this.dialectName = dialectName;
}
public String getDialectName() {
return dialectName;
}
/**
* Get dialect by name
*/
public static @Nullable Dialect getByName(String dialectName) {
if (dialectName == null) {
return null;
}
for (Dialect dialect : Dialect.values()) {
if (dialect.getDialectName().equals(dialectName.toLowerCase())) {
return dialect;
}
}
return null;
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java
View File

@ -31,8 +31,7 @@ import java.math.BigInteger;
/**
* Logical plan builder assistant for buildIn dialect and other dialect.
* The same logical in {@link org.apache.doris.nereids.parser.LogicalPlanBuilder}
* and {@link org.apache.doris.nereids.parser.trino.TrinoLogicalPlanBuilder} can be
* extracted to here.
* can be extracted to here.
*/
public class LogicalPlanBuilderAssistant {

47
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
View File

@ -18,20 +18,19 @@
package org.apache.doris.nereids.parser;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.Config;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.DorisLexer;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.parser.spark.SparkSql3LogicalPlanBuilder;
import org.apache.doris.nereids.parser.trino.TrinoParser;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.plugin.DialectConverterPlugin;
import org.apache.doris.plugin.PluginMgr;
import org.apache.doris.qe.SessionVariable;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
@ -67,11 +66,13 @@ public class NereidsParser {
* ParseSQL with dialect.
*/
public List<StatementBase> parseSQL(String sql, SessionVariable sessionVariable) {
@Nullable ParseDialect.Dialect sqlDialect = ParseDialect.Dialect.getByName(sessionVariable.getSqlDialect());
return parseSQLWithDialect(sql, sqlDialect, sessionVariable);
return parseSQLWithDialect(sql, sessionVariable);
}
private List<StatementBase> parseSQL(String originStr, @Nullable LogicalPlanBuilder logicalPlanBuilder) {
/**
* ParseSQL with logicalPlanBuilder.
*/
public List<StatementBase> parseSQL(String originStr, @Nullable LogicalPlanBuilder logicalPlanBuilder) {
List<Pair<LogicalPlan, StatementContext>> logicalPlans = parseMultiple(originStr, logicalPlanBuilder);
List<StatementBase> statementBases = Lists.newArrayList();
for (Pair<LogicalPlan, StatementContext> parsedPlanToContext : logicalPlans) {
@ -81,26 +82,28 @@ public class NereidsParser {
}
private List<StatementBase> parseSQLWithDialect(String sql,
@Nullable ParseDialect.Dialect sqlDialect,
SessionVariable sessionVariable) {
if (!Strings.isNullOrEmpty(Config.sql_convertor_service)) {
// if sql convertor service is enabled, no need to parse sql again by specific dialect.
@Nullable Dialect sqlDialect = Dialect.getByName(sessionVariable.getSqlDialect());
if (sqlDialect == null) {
return parseSQL(sql);
}
switch (sqlDialect) {
case TRINO:
final List<StatementBase> logicalPlans = TrinoParser.parse(sql, sessionVariable);
if (CollectionUtils.isEmpty(logicalPlans)) {
return parseSQL(sql);
PluginMgr pluginMgr = Env.getCurrentEnv().getPluginMgr();
List<DialectConverterPlugin> plugins = pluginMgr.getActiveDialectPluginList(sqlDialect);
for (DialectConverterPlugin plugin : plugins) {
try {
List<StatementBase> statementBases = plugin.parseSqlWithDialect(sql, sessionVariable);
if (CollectionUtils.isNotEmpty(statementBases)) {
return statementBases;
}
return logicalPlans;
case SPARK_SQL:
return parseSQL(sql, new SparkSql3LogicalPlanBuilder());
default:
return parseSQL(sql);
} catch (Throwable throwable) {
LOG.warn("Parse sql with dialect {} failed, plugin: {}, sql: {}.",
sqlDialect, plugin.getClass().getSimpleName(), sql, throwable);
}
}
// fallback if any exception occurs before
return parseSQL(sql);
}
/**

129
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseDialect.java
View File

@ -1,129 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser;
import javax.annotation.Nullable;
/**
* ParseDialect enum, maybe support other dialect.
*/
public enum ParseDialect {
/**
* Trino parser and it's version is 395.
*/
TRINO_395(Dialect.TRINO, Version.TRINO_395),
/**
* Doris parser and it's version is 2.0.0.
*/
DORIS_2_ALL(Dialect.DORIS, Version.DORIS_2_ALL),
/**
* Spark parser and it's version is 3.x.
*/
SPARK_SQL_3_ALL(Dialect.SPARK_SQL, Version.SPARK_SQL_3_ALL);
private final Dialect dialect;
private final Version version;
ParseDialect(Dialect dialect, Version version) {
this.dialect = dialect;
this.version = version;
}
public Version getVersion() {
return version;
}
public Dialect getDialect() {
return dialect;
}
/**
* The version of parse dialect.
*/
public enum Version {
/**
* Trino parser and it's version is 395.
*/
TRINO_395("395"),
/**
* Doris parser and it's version is 2.0.0.
*/
DORIS_2_ALL("2.*"),
/**
* Spark sql parser and it's version is 3.x.
*/
SPARK_SQL_3_ALL("3.*");
private final String version;
Version(String version) {
this.version = version;
}
public String getVersionName() {
return version;
}
}
/**
* The dialect name of parse dialect.
*/
public enum Dialect {
/**
* Trino parser dialect
*/
TRINO("trino"),
/**
* Presto parser dialect
*/
PRESTO("presto"),
/**
* Doris parser dialect
*/
DORIS("doris"),
/**
* Spark sql parser dialect
*/
SPARK_SQL("spark_sql");
private String dialectName;
Dialect(String dialectName) {
this.dialectName = dialectName;
}
public String getDialectName() {
return dialectName;
}
/**
* Get dialect by name
*/
public static @Nullable Dialect getByName(String dialectName) {
if (dialectName == null) {
return null;
}
for (Dialect dialect : Dialect.values()) {
if (dialect.getDialectName().equals(dialectName.toLowerCase())) {
return dialect;
}
}
return null;
}
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserContext.java
View File

@ -22,13 +22,13 @@ package org.apache.doris.nereids.parser;
*/
public class ParserContext {
private final ParseDialect parseDialect;
private final Dialect parseDialect;
public ParserContext(ParseDialect parseDialect) {
public ParserContext(Dialect parseDialect) {
this.parseDialect = parseDialect;
}
public ParseDialect getParserDialect() {
public Dialect getParserDialect() {
return parseDialect;
}
}

81
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/DateTruncFnCallTransformer.java
View File

@ -1,81 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser.spark;
import org.apache.doris.nereids.analyzer.UnboundFunction;
import org.apache.doris.nereids.parser.ComplexFnCallTransformer;
import org.apache.doris.nereids.parser.ParserContext;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.List;
/**
* DateTrunc complex function transformer
*/
public class DateTruncFnCallTransformer extends ComplexFnCallTransformer {
// reference: https://spark.apache.org/docs/latest/api/sql/index.html#trunc
// spark-sql support YEAR/YYYY/YY for year, support MONTH/MON/MM for month
private static final ImmutableSet<String> YEAR = ImmutableSet.<String>builder()
.add("YEAR")
.add("YYYY")
.add("YY")
.build();
private static final ImmutableSet<String> MONTH = ImmutableSet.<String>builder()
.add("MONTH")
.add("MON")
.add("MM")
.build();
@Override
public String getSourceFnName() {
return "trunc";
}
@Override
protected boolean check(String sourceFnName, List<Expression> sourceFnTransformedArguments,
ParserContext context) {
return getSourceFnName().equalsIgnoreCase(sourceFnName) && (sourceFnTransformedArguments.size() == 2);
}
@Override
protected Function transform(String sourceFnName, List<Expression> sourceFnTransformedArguments,
ParserContext context) {
VarcharLiteral fmtLiteral = (VarcharLiteral) sourceFnTransformedArguments.get(1);
if (YEAR.contains(fmtLiteral.getValue().toUpperCase())) {
return new UnboundFunction(
"date_trunc",
ImmutableList.of(sourceFnTransformedArguments.get(0), new VarcharLiteral("YEAR")));
}
if (MONTH.contains(fmtLiteral.getValue().toUpperCase())) {
return new UnboundFunction(
"date_trunc",
ImmutableList.of(sourceFnTransformedArguments.get(0), new VarcharLiteral("MONTH")));
}
return new UnboundFunction(
"date_trunc",
ImmutableList.of(sourceFnTransformedArguments.get(0), sourceFnTransformedArguments.get(1)));
}
}

93
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3FnCallTransformers.java
View File

@ -1,93 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser.spark;
import org.apache.doris.nereids.analyzer.PlaceholderExpression;
import org.apache.doris.nereids.parser.AbstractFnCallTransformers;
import org.apache.doris.nereids.trees.expressions.Expression;
import com.google.common.collect.Lists;
/**
* The builder and factory for spark-sql 3.x FnCallTransformers, supply transform facade ability.
*/
public class SparkSql3FnCallTransformers extends AbstractFnCallTransformers {
private SparkSql3FnCallTransformers() {
}
@Override
protected void registerTransformers() {
// register json functions
registerJsonFunctionTransformers();
// register string functions
registerStringFunctionTransformers();
// register date functions
registerDateFunctionTransformers();
// register numeric functions
registerNumericFunctionTransformers();
// TODO: add other function transformer
}
@Override
protected void registerComplexTransformers() {
DateTruncFnCallTransformer dateTruncFnCallTransformer = new DateTruncFnCallTransformer();
doRegister(dateTruncFnCallTransformer.getSourceFnName(), dateTruncFnCallTransformer);
// TODO: add other complex function transformer
}
private void registerJsonFunctionTransformers() {
doRegister("get_json_object", "json_extract",
Lists.newArrayList(
PlaceholderExpression.of(Expression.class, 1),
PlaceholderExpression.of(Expression.class, 2)));
}
private void registerStringFunctionTransformers() {
doRegister("split", "split_by_string",
Lists.newArrayList(
PlaceholderExpression.of(Expression.class, 1),
PlaceholderExpression.of(Expression.class, 2)));
}
private void registerDateFunctionTransformers() {
// spark-sql support to_date(date_str, fmt) function but doris only support to_date(date_str)
// here try to compat with this situation by using str_to_date(str, fmt),
// this function support the following three formats which can handle the mainly situations:
// 1. yyyyMMdd
// 2. yyyy-MM-dd
// 3. yyyy-MM-dd HH:mm:ss
doRegister("to_date", "str_to_date",
Lists.newArrayList(
PlaceholderExpression.of(Expression.class, 1),
PlaceholderExpression.of(Expression.class, 2)));
}
private void registerNumericFunctionTransformers() {
doRegister("mean", "avg",
Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)));
}
static class SingletonHolder {
private static final SparkSql3FnCallTransformers INSTANCE = new SparkSql3FnCallTransformers();
}
public static SparkSql3FnCallTransformers getSingleton() {
return SingletonHolder.INSTANCE;
}
}

88
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3LogicalPlanBuilder.java
View File

@ -1,88 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser.spark;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.analyzer.UnboundFunction;
import org.apache.doris.nereids.exceptions.ParseException;
import org.apache.doris.nereids.parser.LogicalPlanBuilder;
import org.apache.doris.nereids.parser.ParseDialect;
import org.apache.doris.nereids.parser.ParserContext;
import org.apache.doris.nereids.parser.ParserUtils;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.commons.lang3.StringUtils;
/**
* Extends from {@link org.apache.doris.nereids.parser.LogicalPlanBuilder},
* just focus on the difference between these query syntax.
*/
public class SparkSql3LogicalPlanBuilder extends LogicalPlanBuilder {
// use a default alias name if not exists, keep the same name with spark-sql
public static final String DEFAULT_TABLE_ALIAS = "__auto_generated_subquery_name";
private final ParserContext parserContext;
public SparkSql3LogicalPlanBuilder() {
this.parserContext = new ParserContext(ParseDialect.SPARK_SQL_3_ALL);
}
@Override
public LogicalPlan visitAliasedQuery(DorisParser.AliasedQueryContext ctx) {
LogicalPlan plan = withTableAlias(visitQuery(ctx.query()), ctx.tableAlias());
for (DorisParser.LateralViewContext lateralViewContext : ctx.lateralView()) {
plan = withGenerate(plan, lateralViewContext);
}
return plan;
}
@Override
public Expression visitFunctionCallExpression(DorisParser.FunctionCallExpressionContext ctx) {
Expression expression = super.visitFunctionCallExpression(ctx);
if (!(expression instanceof UnboundFunction)) {
return expression;
}
UnboundFunction sourceFunction = (UnboundFunction) expression;
Function transformedFunction = SparkSql3FnCallTransformers.getSingleton().transform(
sourceFunction.getName(),
sourceFunction.getArguments(),
this.parserContext
);
if (transformedFunction == null) {
return expression;
}
return transformedFunction;
}
private LogicalPlan withTableAlias(LogicalPlan plan, DorisParser.TableAliasContext ctx) {
if (ctx.strictIdentifier() == null) {
return plan;
}
return ParserUtils.withOrigin(ctx.strictIdentifier(), () -> {
String alias = StringUtils.isEmpty(ctx.strictIdentifier().getText())
? DEFAULT_TABLE_ALIAS : ctx.strictIdentifier().getText();
if (null != ctx.identifierList()) {
throw new ParseException("Do not implemented", ctx);
}
return new LogicalSubQueryAlias<>(alias, plan);
});
}
}

64
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java
View File

@ -1,64 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser.trino;
import org.apache.doris.nereids.analyzer.UnboundFunction;
import org.apache.doris.nereids.parser.ComplexFnCallTransformer;
import org.apache.doris.nereids.parser.ParserContext;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* DateDiff complex function transformer
*/
public class DateDiffFnCallTransformer extends ComplexFnCallTransformer {
private static final String SECOND = "second";
private static final String HOUR = "hour";
private static final String DAY = "day";
private static final String MILLI_SECOND = "millisecond";
@Override
public String getSourceFnName() {
return "date_diff";
}
@Override
protected boolean check(String sourceFnName, List<Expression> sourceFnTransformedArguments,
ParserContext context) {
return getSourceFnName().equalsIgnoreCase(sourceFnName) && (sourceFnTransformedArguments.size() == 3);
}
@Override
protected Function transform(String sourceFnName, List<Expression> sourceFnTransformedArguments,
ParserContext context) {
VarcharLiteral diffGranularity = (VarcharLiteral) sourceFnTransformedArguments.get(0);
if (SECOND.equals(diffGranularity.getValue())) {
return new UnboundFunction(
"seconds_diff",
ImmutableList.of(sourceFnTransformedArguments.get(1), sourceFnTransformedArguments.get(2)));
}
// TODO: support other date diff granularity
return null;
}
}

61
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java
View File

@ -1,61 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser.trino;
import org.apache.doris.nereids.analyzer.PlaceholderExpression;
import org.apache.doris.nereids.parser.AbstractFnCallTransformers;
import org.apache.doris.nereids.trees.expressions.Expression;
import com.google.common.collect.Lists;
/**
* The builder and factory for trino function call transformers,
* and supply transform facade ability.
*/
public class TrinoFnCallTransformers extends AbstractFnCallTransformers {
private TrinoFnCallTransformers() {
}
@Override
protected void registerTransformers() {
registerStringFunctionTransformer();
// TODO: add other function transformer
}
@Override
protected void registerComplexTransformers() {
DateDiffFnCallTransformer dateDiffFnCallTransformer = new DateDiffFnCallTransformer();
doRegister(dateDiffFnCallTransformer.getSourceFnName(), dateDiffFnCallTransformer);
// TODO: add other complex function transformer
}
protected void registerStringFunctionTransformer() {
doRegister("codepoint", "ascii",
Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)));
// TODO: add other string function transformer
}
static class SingletonHolder {
private static final TrinoFnCallTransformers INSTANCE = new TrinoFnCallTransformers();
}
public static TrinoFnCallTransformers getSingleton() {
return SingletonHolder.INSTANCE;
}
}

329
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoLogicalPlanBuilder.java
View File

@ -1,329 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser.trino;
import org.apache.doris.nereids.analyzer.UnboundAlias;
import org.apache.doris.nereids.analyzer.UnboundFunction;
import org.apache.doris.nereids.analyzer.UnboundOneRowRelation;
import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.analyzer.UnboundResultSink;
import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.exceptions.DialectTransformException;
import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant;
import org.apache.doris.nereids.parser.ParserContext;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.coercion.CharacterType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* The actually planBuilder for Trino SQL to Doris logical plan.
* It depends on {@link io.trino.sql.tree.AstVisitor}
*/
public class TrinoLogicalPlanBuilder extends io.trino.sql.tree.AstVisitor<Object, ParserContext> {
public Object visit(io.trino.sql.tree.Node node, ParserContext context) {
return this.process(node, context);
}
public <T> T visit(io.trino.sql.tree.Node node, ParserContext context, Class<T> clazz) {
return clazz.cast(this.process(node, context));
}
public <T> List<T> visit(List<? extends io.trino.sql.tree.Node> nodes, ParserContext context, Class<T> clazz) {
return nodes.stream()
.map(node -> clazz.cast(this.process(node, context)))
.collect(Collectors.toList());
}
public Object processOptional(Optional<? extends io.trino.sql.tree.Node> node, ParserContext context) {
return node.map(value -> this.process(value, context)).orElse(null);
}
public <T> T processOptional(Optional<? extends io.trino.sql.tree.Node> node,
ParserContext context, Class<T> clazz) {
return node.map(value -> clazz.cast(this.process(value, context))).orElse(null);
}
@Override
protected LogicalPlan visitQuery(io.trino.sql.tree.Query node, ParserContext context) {
io.trino.sql.tree.QueryBody queryBody = node.getQueryBody();
LogicalPlan logicalPlan = (LogicalPlan) visit(queryBody, context);
if (!(queryBody instanceof io.trino.sql.tree.QuerySpecification)) {
// TODO: need to handle orderBy and limit
throw new DialectTransformException("transform querySpecification");
}
return logicalPlan;
}
@Override
protected LogicalPlan visitQuerySpecification(io.trino.sql.tree.QuerySpecification node,
ParserContext context) {
// from -> where -> group by -> having -> select
Optional<io.trino.sql.tree.Relation> from = node.getFrom();
LogicalPlan fromPlan = processOptional(from, context, LogicalPlan.class);
List<io.trino.sql.tree.SelectItem> selectItems = node.getSelect().getSelectItems();
if (from == null || !from.isPresent()) {
// TODO: support query values
List<NamedExpression> expressions = selectItems.stream()
.map(item -> visit(item, context, NamedExpression.class))
.collect(ImmutableList.toImmutableList());
return new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), expressions);
}
// TODO: support predicate, aggregate, having, order by, limit
// TODO: support distinct
boolean isDistinct = node.getSelect().isDistinct();
return new UnboundResultSink<>(withProjection(selectItems, fromPlan, isDistinct, context));
}
private LogicalProject withProjection(List<io.trino.sql.tree.SelectItem> selectItems,
LogicalPlan input,
boolean isDistinct,
ParserContext context) {
List<NamedExpression> expressions = selectItems.stream()
.map(item -> visit(item, context, NamedExpression.class))
.collect(Collectors.toList());
return new LogicalProject(expressions, ImmutableList.of(), isDistinct, input);
}
@Override
protected Expression visitSingleColumn(io.trino.sql.tree.SingleColumn node, ParserContext context) {
String alias = node.getAlias().map(io.trino.sql.tree.Identifier::getValue).orElse(null);
Expression expr = visit(node.getExpression(), context, Expression.class);
if (expr instanceof NamedExpression) {
return (NamedExpression) expr;
} else {
return alias == null ? new UnboundAlias(expr) : new UnboundAlias(expr, alias);
}
}
@Override
protected Object visitIdentifier(io.trino.sql.tree.Identifier node, ParserContext context) {
return new UnboundSlot(ImmutableList.of(node.getValue()));
}
/* ********************************************************************************************
* visitFunction
* ******************************************************************************************** */
@Override
protected Function visitFunctionCall(io.trino.sql.tree.FunctionCall node, ParserContext context) {
List<Expression> exprs = visit(node.getArguments(), context, Expression.class);
Function transformedFn =
TrinoFnCallTransformers.getSingleton().transform(node.getName().toString(), exprs, context);
if (transformedFn == null) {
transformedFn = new UnboundFunction(node.getName().toString(), exprs);
}
return transformedFn;
}
/* ********************************************************************************************
* visitTable
* ******************************************************************************************** */
@Override
protected LogicalPlan visitTable(io.trino.sql.tree.Table node, ParserContext context) {
io.trino.sql.tree.QualifiedName name = node.getName();
List<String> tableId = name.getParts();
// build table
return LogicalPlanBuilderAssistant.withCheckPolicy(
new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId,
ImmutableList.of(), false));
}
/* ********************************************************************************************
* visit buildIn function
* ******************************************************************************************** */
@Override
protected Expression visitCast(io.trino.sql.tree.Cast node, ParserContext context) {
Expression expr = visit(node.getExpression(), context, Expression.class);
DataType dataType = mappingType(node.getType());
Expression cast = new Cast(expr, dataType);
if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) {
if (dataType.isVarcharType() && ((VarcharType) dataType).isWildcardVarchar()) {
return cast;
}
List<Expression> args = ImmutableList.of(
cast,
new TinyIntLiteral((byte) 1),
Literal.of(((CharacterType) dataType).getLen())
);
return new UnboundFunction("substr", args);
} else {
return cast;
}
}
/* ********************************************************************************************
* visitLiteral
* ******************************************************************************************** */
@Override
protected Object visitLiteral(io.trino.sql.tree.Literal node, ParserContext context) {
// TODO: support literal transform
throw new DialectTransformException("transform literal");
}
@Override
protected Literal visitLongLiteral(io.trino.sql.tree.LongLiteral node, ParserContext context) {
return LogicalPlanBuilderAssistant.handleIntegerLiteral(String.valueOf(node.getValue()));
}
@Override
protected Object visitDoubleLiteral(io.trino.sql.tree.DoubleLiteral node, ParserContext context) {
// TODO: support double literal transform
throw new DialectTransformException("transform double literal");
}
@Override
protected Object visitDecimalLiteral(io.trino.sql.tree.DecimalLiteral node, ParserContext context) {
// TODO: support decimal literal transform
throw new DialectTransformException("transform decimal literal");
}
@Override
protected Object visitTimestampLiteral(io.trino.sql.tree.TimestampLiteral node, ParserContext context) {
try {
String value = node.getValue();
if (value.length() <= 10) {
value += " 00:00:00";
}
return new DateTimeLiteral(value);
} catch (AnalysisException e) {
throw new DialectTransformException("transform timestamp literal");
}
}
@Override
protected Object visitGenericLiteral(io.trino.sql.tree.GenericLiteral node, ParserContext context) {
// TODO: support generic literal transform
throw new DialectTransformException("transform generic literal");
}
@Override
protected Object visitTimeLiteral(io.trino.sql.tree.TimeLiteral node, ParserContext context) {
// TODO: support time literal transform
throw new DialectTransformException("transform time literal");
}
@Override
protected Object visitCharLiteral(io.trino.sql.tree.CharLiteral node, ParserContext context) {
// TODO: support char literal transform
throw new DialectTransformException("transform char literal");
}
@Override
protected Expression visitStringLiteral(io.trino.sql.tree.StringLiteral node, ParserContext context) {
// TODO: add unescapeSQLString.
String txt = node.getValue();
if (txt.length() <= 1) {
return new VarcharLiteral(txt);
}
return new VarcharLiteral(LogicalPlanBuilderAssistant.escapeBackSlash(txt.substring(0, txt.length())));
}
@Override
protected Object visitIntervalLiteral(io.trino.sql.tree.IntervalLiteral node, ParserContext context) {
// TODO: support interval literal transform
throw new DialectTransformException("transform char literal");
}
@Override
protected Object visitBinaryLiteral(io.trino.sql.tree.BinaryLiteral node, ParserContext context) {
// TODO: support binary literal transform
throw new DialectTransformException("transform binary literal");
}
@Override
protected Object visitNullLiteral(io.trino.sql.tree.NullLiteral node, ParserContext context) {
return NullLiteral.INSTANCE;
}
@Override
protected Object visitBooleanLiteral(io.trino.sql.tree.BooleanLiteral node, ParserContext context) {
return BooleanLiteral.of(node.getValue());
}
private DataType mappingType(io.trino.sql.tree.DataType dataType) {
if (dataType instanceof io.trino.sql.tree.GenericDataType) {
io.trino.sql.tree.GenericDataType genericDataType = (io.trino.sql.tree.GenericDataType) dataType;
String typeName = genericDataType.getName().getValue().toLowerCase();
List<String> types = Lists.newArrayList(typeName);
String length = null;
String precision = null;
String scale = null;
List<io.trino.sql.tree.DataTypeParameter> arguments = genericDataType.getArguments();
if (!arguments.isEmpty()) {
if (arguments.get(0) instanceof io.trino.sql.tree.NumericParameter) {
precision = length = ((io.trino.sql.tree.NumericParameter) arguments.get(0)).getValue();
}
if (arguments.size() > 1 && arguments.get(1) instanceof io.trino.sql.tree.NumericParameter) {
scale = ((io.trino.sql.tree.NumericParameter) arguments.get(1)).getValue();
}
}
if ("decimal".equals(typeName)) {
if (precision != null) {
types.add(precision);
}
if (scale != null) {
types.add(scale);
}
}
if ("varchar".equals(typeName) || "char".equals(typeName)) {
if (length != null) {
types.add(length);
}
}
// unsigned decimal in Trino is longDecimal, not handle now, support it later
if (!"decimal".equals(typeName) && typeName.contains("decimal")) {
throw new DialectTransformException("transform not standard decimal data type ");
}
// Trino only support signed, safe unsigned is false here
return DataType.convertPrimitiveFromStrings(types, false);
} else if (dataType instanceof io.trino.sql.tree.DateTimeDataType) {
// TODO: support date data type mapping
throw new DialectTransformException("transform date data type");
}
throw new AnalysisException("Nereids do not support type: " + dataType);
}
}

102
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java
View File

@ -1,102 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.parser.trino;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.exceptions.UnsupportedDialectException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.parser.ParseDialect;
import org.apache.doris.nereids.parser.ParserContext;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.qe.SessionVariable;
import com.google.common.base.Preconditions;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import javax.annotation.Nullable;
/**
* Trino Parser, depends on 395 trino-parser, and 4.9.3 antlr-runtime
*/
public class TrinoParser {
public static final Logger LOG = LogManager.getLogger(TrinoParser.class);
private static final io.trino.sql.parser.ParsingOptions PARSING_OPTIONS = new io.trino.sql.parser.ParsingOptions(
io.trino.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL);
/**
* Parse with trino syntax, return null if parse failed
*/
public static @Nullable List<StatementBase> parse(String sql, SessionVariable sessionVariable) {
final List<StatementBase> logicalPlans = new ArrayList<>();
try {
io.trino.sql.parser.StatementSplitter splitter = new io.trino.sql.parser.StatementSplitter(
addDelimiterIfNeeded(sql));
ParserContext parserContext = new ParserContext(ParseDialect.TRINO_395);
StatementContext statementContext = new StatementContext();
for (io.trino.sql.parser.StatementSplitter.Statement statement : splitter.getCompleteStatements()) {
Object parsedPlan = parseSingle(statement.statement(), parserContext);
logicalPlans.add(parsedPlan == null
? null : new LogicalPlanAdapter((LogicalPlan) parsedPlan, statementContext));
}
if (logicalPlans.isEmpty() || logicalPlans.stream().anyMatch(Objects::isNull)) {
return null;
}
return logicalPlans;
} catch (io.trino.sql.parser.ParsingException | UnsupportedDialectException e) {
LOG.debug("Failed to parse logical plan from trino, sql is :{}", sql, e);
return null;
}
}
private static io.trino.sql.tree.Statement parse(String sql) {
io.trino.sql.parser.SqlParser sqlParser = new io.trino.sql.parser.SqlParser();
return sqlParser.createStatement(sql, PARSING_OPTIONS);
}
/**
* Parse trino dialect sql.
*
* @param sql sql string
* @param parserContext parse context
* @return logical plan
*/
public static <T> T parseSingle(String sql, ParserContext parserContext) {
Preconditions.checkArgument(parserContext.getParserDialect() == ParseDialect.TRINO_395);
io.trino.sql.tree.Statement statement = TrinoParser.parse(sql);
return (T) new TrinoLogicalPlanBuilder().visit(statement, parserContext);
}
/**
* {@link io.trino.sql.parser.StatementSplitter} use ";" as the delimiter if not set
* So add ";" if sql does not end with ";",
* otherwise {@link io.trino.sql.parser.StatementSplitter#getCompleteStatements()} will return empty list
*/
private static String addDelimiterIfNeeded(String sql) {
if (!sql.trim().endsWith(";")) {
return sql + ";";
}
return sql;
}
}

61
fe/fe-core/src/main/java/org/apache/doris/plugin/DialectConverterPlugin.java Normal file
View File

@ -0,0 +1,61 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.plugin;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.nereids.parser.Dialect;
import org.apache.doris.qe.SessionVariable;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import javax.annotation.Nullable;
/**
* Sql dialect adapter interface.
*/
public interface DialectConverterPlugin {
/**
* Set dialect set which this plugin can handle
*/
ImmutableSet<Dialect> acceptDialects();
/**
* <pre>
* Override this method if you want to convert sql before parse.
* Fallback to next dialect plugin if this method returns null, empty string or any exception throws.
* If all plugins fail to convert (returns null, empty string or any exception throws),
* Nereids parser will use the original SQL.
* </pre>
* */
default @Nullable String convertSql(String originSql, SessionVariable sessionVariable) {
return null;
}
/**
* <pre>
* Parse sql with dialect.
* Fallback to next dialect plugin if this method returns null, empty string or any exception throws.
* If all plugins fail to parse (returns null, empty string or any exception throws),
* Nereids parser will fallback to invoke {@link org.apache.doris.nereids.parser.NereidsParser#parseSQL(String)}.
* Use Dialect.getByName(sessionVariable.getSqlDialect()) to extract the dialect parameter.
* </pre>
* */
@Nullable List<StatementBase> parseSqlWithDialect(String sql, SessionVariable sessionVariable);
}

3
fe/fe-core/src/main/java/org/apache/doris/plugin/PluginInfo.java
View File

@ -52,7 +52,8 @@ public class PluginInfo implements Writable {
public enum PluginType {
AUDIT,
IMPORT,
STORAGE;
STORAGE,
DIALECT;
public static int MAX_PLUGIN_TYPE_SIZE = PluginType.values().length;
}

68
fe/fe-core/src/main/java/org/apache/doris/plugin/PluginMgr.java
View File

@ -24,11 +24,13 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.nereids.parser.Dialect;
import org.apache.doris.plugin.PluginInfo.PluginType;
import org.apache.doris.plugin.PluginLoader.PluginStatus;
import org.apache.doris.qe.AuditLogBuilder;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@ -51,15 +53,26 @@ public class PluginMgr implements Writable {
public static final String BUILTIN_PLUGIN_PREFIX = "__builtin_";
private final Map<String, PluginLoader>[] plugins;
// use this for
private final Map<String, DialectConverterPlugin>[] dialectPlugins;
// all dynamic plugins should have unique names,
private final Set<String> dynamicPluginNames;
public PluginMgr() {
plugins = new Map[PluginType.MAX_PLUGIN_TYPE_SIZE];
for (int i = 0; i < PluginType.MAX_PLUGIN_TYPE_SIZE; i++) {
plugins[i] = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
// use synchronized wrapper for thread-safe
plugins[i] = Collections.synchronizedSortedMap(Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER));
}
// use synchronized wrapper for thread-safe
dynamicPluginNames = Collections.synchronizedSortedSet(Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER));
dialectPlugins = new Map[Dialect.MAX_DIALECT_SIZE];
for (int i = 0; i < Dialect.MAX_DIALECT_SIZE; i++) {
// use synchronized wrapper for thread-safe
dialectPlugins[i] = Collections.synchronizedSortedMap(Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER));
}
dynamicPluginNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
}
// create the plugin dir if missing
@ -79,21 +92,15 @@ public class PluginMgr implements Writable {
}
private boolean checkDynamicPluginNameExist(String name) {
synchronized (dynamicPluginNames) {
return dynamicPluginNames.contains(name);
}
return dynamicPluginNames.contains(name);
}
private boolean addDynamicPluginNameIfAbsent(String name) {
synchronized (dynamicPluginNames) {
return dynamicPluginNames.add(name);
}
return dynamicPluginNames.add(name);
}
private boolean removeDynamicPluginName(String name) {
synchronized (dynamicPluginNames) {
return dynamicPluginNames.remove(name);
}
return dynamicPluginNames.remove(name);
}
private void initBuiltinPlugins() {
@ -130,7 +137,11 @@ public class PluginMgr implements Writable {
throw new UserException("plugin " + info.getName() + " has already been installed.");
}
plugins[info.getTypeId()].put(info.getName(), pluginLoader);
// add dialect plugin
Plugin plugin = pluginLoader.getPlugin();
if (plugin instanceof DialectConverterPlugin) {
addDialectPlugin((DialectConverterPlugin) plugin, info);
}
Env.getCurrentEnv().getEditLog().logInstallPlugin(info);
LOG.info("install plugin {}", info.getName());
return info;
@ -140,6 +151,18 @@ public class PluginMgr implements Writable {
}
}
private void addDialectPlugin(DialectConverterPlugin plugin, PluginInfo info) {
for (Dialect dialect : plugin.acceptDialects()) {
dialectPlugins[dialect.ordinal()].put(info.getName(), plugin);
}
}
private void removeDialectPlugin(String name) {
for (int i = 0; i < Dialect.MAX_DIALECT_SIZE; i++) {
dialectPlugins[i].remove(name);
}
}
/**
* Dynamic uninstall plugin.
* If uninstall failed, the plugin should NOT be removed from plugin manager.
@ -153,7 +176,7 @@ public class PluginMgr implements Writable {
if (plugins[i].containsKey(name)) {
PluginLoader loader = plugins[i].get(name);
if (loader == null) {
// this is not a atomic operation, so even if containsKey() is true,
// this is not an atomic operation, so even if containsKey() is true,
// we may still get null object by get() method
continue;
}
@ -171,7 +194,7 @@ public class PluginMgr implements Writable {
plugins[i].remove(name);
loader.setStatus(PluginStatus.UNINSTALLED);
removeDynamicPluginName(name);
removeDialectPlugin(name);
// do not get plugin info by calling loader.getPluginInfo(). That method will try to
// reload the plugin properties from source if this plugin is not installed successfully.
// Here we only need the plugin's name for persisting.
@ -195,7 +218,10 @@ public class PluginMgr implements Writable {
PluginLoader loader = new BuiltinPluginLoader(Config.plugin_dir, pluginInfo, plugin);
PluginLoader checkLoader = plugins[pluginInfo.getTypeId()].putIfAbsent(pluginInfo.getName(), loader);
// add dialect plugin
if (plugin instanceof DialectConverterPlugin) {
addDialectPlugin((DialectConverterPlugin) plugin, pluginInfo);
}
return checkLoader == null;
}
@ -217,6 +243,11 @@ public class PluginMgr implements Writable {
// install plugin
pluginLoader.reload();
pluginLoader.setStatus(PluginStatus.INSTALLED);
// add dialect plugin
Plugin plugin = pluginLoader.getPlugin();
if (plugin instanceof DialectConverterPlugin) {
addDialectPlugin((DialectConverterPlugin) plugin, info);
}
} catch (IOException | UserException e) {
pluginLoader.setStatus(PluginStatus.ERROR, e.getMessage());
throw e;
@ -246,7 +277,12 @@ public class PluginMgr implements Writable {
}
});
return Collections.unmodifiableList(l);
return ImmutableList.copyOf(l);
}
public final List<DialectConverterPlugin> getActiveDialectPluginList(Dialect dialect) {
Map<String, DialectConverterPlugin> m = dialectPlugins[dialect.ordinal()];
return ImmutableList.copyOf(m.values());
}
public final List<PluginInfo> getAllDynamicPluginInfo() {

2
fe/fe-core/src/main/java/org/apache/doris/qe/AuditEventProcessor.java
View File

@ -84,7 +84,7 @@ public class AuditEventProcessor {
AuditEvent auditEvent;
while (!isStopped) {
// update audit plugin list every UPDATE_PLUGIN_INTERVAL_MS.
// because some of plugins may be installed or uninstalled at runtime.
// because some plugins may be installed or uninstalled at runtime.
if (auditPlugins == null || System.currentTimeMillis() - lastUpdateTime > UPDATE_PLUGIN_INTERVAL_MS) {
auditPlugins = pluginMgr.getActivePluginList(PluginType.AUDIT);
lastUpdateTime = System.currentTimeMillis();

31
fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
View File

@ -36,7 +36,6 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.SQLDialectUtils;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.common.util.Util;
@ -50,8 +49,11 @@ import org.apache.doris.mysql.MysqlServerStatusFlag;
import org.apache.doris.nereids.exceptions.NotSupportedException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.minidump.MinidumpUtils;
import org.apache.doris.nereids.parser.Dialect;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.stats.StatsErrorEstimator;
import org.apache.doris.plugin.DialectConverterPlugin;
import org.apache.doris.plugin.PluginMgr;
import org.apache.doris.proto.Data;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.thrift.TMasterOpRequest;
@ -61,6 +63,7 @@ import org.apache.doris.thrift.TUniqueId;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -70,6 +73,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import javax.annotation.Nullable;
/**
* Process one connection, the life cycle is the same as connection
@ -174,8 +178,7 @@ public abstract class ConnectProcessor {
MetricRepo.COUNTER_REQUEST_ALL.increase(1L);
}
String convertedStmt = SQLDialectUtils.convertStmtWithDialect(originStmt, ctx, mysqlCommand);
String convertedStmt = convertOriginStmt(originStmt);
String sqlHash = DigestUtils.md5Hex(convertedStmt);
ctx.setSqlHash(sqlHash);
ctx.getAuditEventBuilder().reset();
@ -284,6 +287,28 @@ public abstract class ConnectProcessor {
}
private String convertOriginStmt(String originStmt) {
String convertedStmt = originStmt;
@Nullable Dialect sqlDialect = Dialect.getByName(ctx.getSessionVariable().getSqlDialect());
if (sqlDialect != null) {
PluginMgr pluginMgr = Env.getCurrentEnv().getPluginMgr();
List<DialectConverterPlugin> plugins = pluginMgr.getActiveDialectPluginList(sqlDialect);
for (DialectConverterPlugin plugin : plugins) {
try {
String convertedSql = plugin.convertSql(originStmt, ctx.getSessionVariable());
if (StringUtils.isNotEmpty(convertedSql)) {
convertedStmt = convertedSql;
break;
}
} catch (Throwable throwable) {
LOG.warn("Convert sql with dialect {} failed, plugin: {}, sql: {}, use origin sql.",
sqlDialect, plugin.getClass().getSimpleName(), originStmt, throwable);
}
}
}
return convertedStmt;
}
// Use a handler for exception to avoid big try catch block which is a little hard to understand
protected void handleQueryException(Throwable throwable, String origStmt,
StatementBase parsedStmt, Data.PQueryStatistics statistics) {

7
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -28,8 +28,7 @@ import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.nereids.metrics.Event;
import org.apache.doris.nereids.metrics.EventSwitchParser;
import org.apache.doris.nereids.parser.ParseDialect;
import org.apache.doris.nereids.parser.ParseDialect.Dialect;
import org.apache.doris.nereids.parser.Dialect;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.planner.GroupCommitBlockSink;
import org.apache.doris.qe.VariableMgr.VarAttr;
@ -2333,8 +2332,8 @@ public class SessionVariable implements Serializable, Writable {
return waitFullBlockScheduleTimes;
}
public ParseDialect.Dialect getSqlParseDialect() {
return ParseDialect.Dialect.getByName(sqlDialect);
public Dialect getSqlParseDialect() {
return Dialect.getByName(sqlDialect);
}
public void setSqlDialect(String sqlDialect) {