database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](catalog) Escape characters for columns in recovery predicate pushdown in SQL (#29854)

Browse Source 
In the previous logic, when we restored the Column in the predicate pushdown based on the logical syntax tree for JdbcScanNode, in order to avoid query errors caused by keywords such as `key`, we added escape characters for it, but before we only Binary predicates are processed, which is imperfect. We should add escape characters to all columns that appear in the predicate to avoid errors with keywords or illegal characters.
This commit is contained in:
zy-kkk
2024-01-15 11:36:18 +08:00
committed by yiguolei
parent 05a65b9f81
commit f6dc6ea13b
10 changed files with 88 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
View File

@ -204,7 +204,7 @@ public class CastExpr extends Expr {
@Override
public String toSqlImpl() {
if (needToMysql) {
if (needExternalSql) {
return getChild(0).toSql();
}
if (isAnalyzed) {

23
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -33,6 +33,8 @@ import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarFunction;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@ -88,7 +90,9 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
public static final String DEFAULT_EXPR_NAME = "expr";
protected boolean disableTableName = false;
protected boolean needToMysql = false;
protected boolean needExternalSql = false;
protected TableType tableType = null;
protected TableIf inputTable = null;
// to be used where we can't come up with a better estimate
public static final double DEFAULT_SELECTIVITY = 0.1;
@ -973,10 +977,13 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
}
public void setNeedToMysql(boolean value) {
needToMysql = value;
public void setExternalContext(boolean needExternalSql, TableType tableType, TableIf inputTable) {
this.needExternalSql = needExternalSql;
this.tableType = tableType;
this.inputTable = inputTable;
for (Expr child : children) {
child.setNeedToMysql(value);
child.setExternalContext(needExternalSql, tableType, inputTable);
}
}
@ -1006,10 +1013,10 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return toSqlImpl();
}
public String toMySql() {
setNeedToMysql(true);
String result = toSql();
setNeedToMysql(false);
public String toExternalSql(TableType tableType, TableIf table) {
setExternalContext(true, tableType, table);
String result = toSql();
setExternalContext(false, null, null);
return result;
}

31
fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
View File

@ -21,8 +21,11 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.JdbcTable;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.io.Text;
@ -246,12 +249,8 @@ public class SlotRef extends Expr {
@Override
public String toSqlImpl() {
if (needToMysql) {
if (col != null) {
return col;
} else {
return "<slot " + Integer.toString(desc.getId().asInt()) + ">";
}
if (needExternalSql) {
return toExternalSqlImpl();
}
if (disableTableName && label != null) {
@ -296,6 +295,26 @@ public class SlotRef extends Expr {
}
}
private String toExternalSqlImpl() {
if (col != null) {
if (tableType.equals(TableType.JDBC_EXTERNAL_TABLE) || tableType.equals(TableType.JDBC) || tableType
.equals(TableType.ODBC)) {
if (inputTable instanceof JdbcTable) {
return ((JdbcTable) inputTable).getProperRealColumnName(
((JdbcTable) inputTable).getJdbcTableType(), col);
} else if (inputTable instanceof OdbcTable) {
return JdbcTable.databaseProperName(((OdbcTable) inputTable).getOdbcTableType(), col);
} else {
return col;
}
} else {
return col;
}
} else {
return "<slot " + Integer.toString(desc.getId().asInt()) + ">";
}
}
public TableName getTableName() {
if (tblName == null) {
Preconditions.checkState(isAnalyzed);

3
fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.external.ExternalScanNode;
import org.apache.doris.statistics.StatisticalType;
@ -144,7 +145,7 @@ public class MysqlScanNode extends ExternalScanNode {
}
ArrayList<Expr> mysqlConjuncts = Expr.cloneList(conjuncts, sMap);
for (Expr p : mysqlConjuncts) {
filters.add(p.toMySql());
filters.add(p.toExternalSql(TableType.MYSQL, null));
}
}

4
fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcFunctionPushDownRule.java vendored
View File

@ -20,6 +20,7 @@ package org.apache.doris.planner.external.jdbc;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.FunctionName;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.thrift.TOdbcTableType;
import com.google.common.base.Preconditions;
@ -108,7 +109,8 @@ public class JdbcFunctionPushDownRule {
Preconditions.checkArgument(!func.isEmpty(), "function can not be empty");
if (checkFunction.test(func)) {
String errMsg = "Unsupported function: " + func + " in expr: " + expr.toMySql()
String errMsg = "Unsupported function: " + func + " in expr: " + expr.toExternalSql(
TableType.JDBC_EXTERNAL_TABLE, null)
+ " in JDBC Table Type: " + tableType;
LOG.warn(errMsg);
errors.add(errMsg);

39
fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java vendored
View File

@ -33,6 +33,8 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.JdbcTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.external.JdbcExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@ -325,7 +327,7 @@ public class JdbcScanNode extends ExternalScanNode {
return !fnExprList.isEmpty();
}
public static String conjunctExprToString(TOdbcTableType tableType, Expr expr, JdbcTable tbl) {
public static String conjunctExprToString(TOdbcTableType tableType, Expr expr, TableIf tbl) {
if (expr instanceof CompoundPredicate) {
StringBuilder result = new StringBuilder();
CompoundPredicate compoundPredicate = (CompoundPredicate) expr;
@ -358,54 +360,37 @@ public class JdbcScanNode extends ExternalScanNode {
if (expr.contains(DateLiteral.class) && expr instanceof BinaryPredicate) {
ArrayList<Expr> children = expr.getChildren();
String filter = children.get(0).toMySql();
String filter = children.get(0).toExternalSql(TableType.JDBC_EXTERNAL_TABLE, tbl);
filter += " " + ((BinaryPredicate) expr).getOp().toString() + " ";
if (tableType.equals(TOdbcTableType.ORACLE)) {
filter += handleOracleDateFormat(children.get(1));
filter += handleOracleDateFormat(children.get(1), tbl);
} else if (tableType.equals(TOdbcTableType.TRINO) || tableType.equals(TOdbcTableType.PRESTO)) {
filter += handleTrinoDateFormat(children.get(1));
filter += handleTrinoDateFormat(children.get(1), tbl);
} else {
filter += children.get(1).toMySql();
filter += children.get(1).toExternalSql(TableType.JDBC_EXTERNAL_TABLE, tbl);
}
return filter;
}
if (expr.contains(SlotRef.class) && expr instanceof BinaryPredicate) {
ArrayList<Expr> children = expr.getChildren();
String filter;
if (children.get(0) instanceof SlotRef) {
if (tbl != null) {
filter = tbl.getProperRealColumnName(tableType, children.get(0).toMySql());
} else {
filter = JdbcTable.databaseProperName(tableType, children.get(0).toMySql());
}
} else {
filter = children.get(0).toMySql();
}
filter += " " + ((BinaryPredicate) expr).getOp().toString() + " ";
filter += children.get(1).toMySql();
return filter;
}
// only for old planner
if (expr.contains(BoolLiteral.class) && "1".equals(expr.getStringValue()) && expr.getChildren().isEmpty()) {
return "1 = 1";
}
return expr.toMySql();
return expr.toExternalSql(TableType.JDBC_EXTERNAL_TABLE, tbl);
}
private static String handleOracleDateFormat(Expr expr) {
private static String handleOracleDateFormat(Expr expr, TableIf tbl) {
if (expr.isConstant()
&& (expr.getType().isDatetime() || expr.getType().isDatetimeV2())) {
return "to_date('" + expr.getStringValue() + "', 'yyyy-mm-dd hh24:mi:ss')";
}
return expr.toMySql();
return expr.toExternalSql(TableType.JDBC_EXTERNAL_TABLE, tbl);
}
private static String handleTrinoDateFormat(Expr expr) {
private static String handleTrinoDateFormat(Expr expr, TableIf tbl) {
if (expr.isConstant()) {
if (expr.getType().isDate() || expr.getType().isDateV2()) {
return "date '" + expr.getStringValue() + "'";
@ -413,6 +398,6 @@ public class JdbcScanNode extends ExternalScanNode {
return "timestamp '" + expr.getStringValue() + "'";
}
}
return expr.toMySql();
return expr.toExternalSql(TableType.JDBC_EXTERNAL_TABLE, tbl);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java vendored
View File

@ -216,7 +216,7 @@ public class OdbcScanNode extends ExternalScanNode {
ArrayList<Expr> odbcConjuncts = Expr.cloneList(conjuncts, sMap);
for (Expr p : odbcConjuncts) {
if (shouldPushDownConjunct(odbcType, p)) {
String filter = JdbcScanNode.conjunctExprToString(odbcType, p, null);
String filter = JdbcScanNode.conjunctExprToString(odbcType, p, tbl);
filters.add(filter);
conjuncts.remove(p);
}