database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[vectorized](bug) fix some open enable_fold_constant_by_be failed cases (#17240)

Browse Source
This commit is contained in:
zhangstar333
2023-03-03 10:30:20 +08:00
committed by GitHub
parent 449f2953c9
commit f5232e5c01
13 changed files with 78 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
View File

@ -51,6 +51,7 @@ import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.thrift.TQueryOptions;
import org.apache.doris.thrift.TUniqueId;
import org.apache.doris.transaction.TransactionState;
import org.apache.doris.transaction.TransactionState.LoadJobSourceType;
@ -226,9 +227,9 @@ public class InsertStmt extends DdlStmt {
}
@Override
public void foldConstant(ExprRewriter rewriter) throws AnalysisException {
public void foldConstant(ExprRewriter rewriter, TQueryOptions tQueryOptions) throws AnalysisException {
Preconditions.checkState(isAnalyzed());
queryStmt.foldConstant(rewriter);
queryStmt.foldConstant(rewriter, tQueryOptions);
}
@Override

2
fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java
View File

@ -98,7 +98,7 @@ public class JsonLiteral extends LiteralExpr {
@Override
public String getStringValue() {
return null;
return value;
}
@Override

5
fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.thrift.TQueryOptions;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@ -517,11 +518,11 @@ public abstract class QueryStmt extends StatementBase implements Queriable {
@Override
public void foldConstant(ExprRewriter rewriter) throws AnalysisException {
public void foldConstant(ExprRewriter rewriter, TQueryOptions tQueryOptions) throws AnalysisException {
Preconditions.checkState(isAnalyzed());
Map<String, Expr> exprMap = new HashMap<>();
collectExprs(exprMap);
rewriter.rewriteConstant(exprMap, analyzer);
rewriter.rewriteConstant(exprMap, analyzer, tQueryOptions);
if (rewriter.changed()) {
putBackExprs(exprMap);
}

3
fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.thrift.TQueryOptions;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@ -197,7 +198,7 @@ public abstract class StatementBase implements ParseNode {
* @throws AnalysisException
* @param rewriter
*/
public void foldConstant(ExprRewriter rewriter) throws AnalysisException {
public void foldConstant(ExprRewriter rewriter, TQueryOptions tQueryOptions) throws AnalysisException {
throw new IllegalStateException(
"foldConstant() not implemented for this stmt: " + getClass().getSimpleName());
}

2
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -905,7 +905,7 @@ public class StmtExecutor implements ProfileWriter {
rewriter.reset();
if (context.getSessionVariable().isEnableFoldConstantByBe()) {
// fold constant expr
parsedStmt.foldConstant(rewriter);
parsedStmt.foldConstant(rewriter, tQueryOptions);
}
// Apply expr and subquery rewrites.

6
fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.analysis.JoinOperator;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.rewrite.mvrewrite.ExprToSlotRefRule;
import org.apache.doris.thrift.TQueryOptions;
import com.google.common.collect.Lists;
@ -182,7 +183,8 @@ public class ExprRewriter {
/**
* FoldConstantsRule rewrite
*/
public void rewriteConstant(Map<String, Expr> exprMap, Analyzer analyzer) throws AnalysisException {
public void rewriteConstant(Map<String, Expr> exprMap, Analyzer analyzer, TQueryOptions tQueryOptions)
throws AnalysisException {
if (exprMap.isEmpty()) {
return;
}
@ -190,7 +192,7 @@ public class ExprRewriter {
// rewrite constant expr
for (ExprRewriteRule rule : rules) {
if (rule instanceof FoldConstantsRule) {
changed = ((FoldConstantsRule) rule).apply(exprMap, analyzer, changed);
changed = ((FoldConstantsRule) rule).apply(exprMap, analyzer, changed, tQueryOptions);
}
}
if (changed) {

43
fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
View File

@ -32,12 +32,14 @@ import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.SysVariableDesc;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.LoadException;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.proto.InternalService;
import org.apache.doris.proto.Types.PScalarType;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.VariableMgr;
import org.apache.doris.rpc.BackendServiceProxy;
@ -47,6 +49,7 @@ import org.apache.doris.thrift.TFoldConstantParams;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.TQueryGlobals;
import org.apache.doris.thrift.TQueryOptions;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
@ -133,7 +136,7 @@ public class FoldConstantsRule implements ExprRewriteRule {
* @return
* @throws AnalysisException
*/
public boolean apply(Map<String, Expr> exprMap, Analyzer analyzer, boolean changed)
public boolean apply(Map<String, Expr> exprMap, Analyzer analyzer, boolean changed, TQueryOptions tQueryOptions)
throws AnalysisException {
// root_expr_id_string:
// child_expr_id_string : texpr
@ -174,7 +177,8 @@ public class FoldConstantsRule implements ExprRewriteRule {
}
if (!paramMap.isEmpty()) {
Map<String, Map<String, Expr>> resultMap = calcConstExpr(paramMap, allConstMap, analyzer.getContext());
Map<String, Map<String, Expr>> resultMap = calcConstExpr(paramMap, allConstMap, analyzer.getContext(),
tQueryOptions);
if (!resultMap.isEmpty()) {
putBackConstExpr(exprMap, resultMap);
@ -195,7 +199,7 @@ public class FoldConstantsRule implements ExprRewriteRule {
*/
// public only for unit test
public void getConstExpr(Expr expr, Map<String, TExpr> constExprMap, Map<String, Expr> oriConstMap,
Analyzer analyzer, Map<String, Expr> sysVarMap, Map<String, Expr> infoFnMap)
Analyzer analyzer, Map<String, Expr> sysVarMap, Map<String, Expr> infoFnMap)
throws AnalysisException {
if (expr.isConstant()) {
// Do not constant fold cast(null as dataType) because we cannot preserve the
@ -338,8 +342,8 @@ public class FoldConstantsRule implements ExprRewriteRule {
* @return
*/
private Map<String, Map<String, Expr>> calcConstExpr(Map<String, Map<String, TExpr>> map,
Map<String, Expr> allConstMap,
ConnectContext context) {
Map<String, Expr> allConstMap,
ConnectContext context, TQueryOptions tQueryOptions) {
TNetworkAddress brpcAddress = null;
Map<String, Map<String, Expr>> resultMap = new HashMap<>();
try {
@ -364,6 +368,8 @@ public class FoldConstantsRule implements ExprRewriteRule {
TFoldConstantParams tParams = new TFoldConstantParams(map, queryGlobals);
tParams.setVecExec(VectorizedUtil.isVectorized());
tParams.setQueryOptions(tQueryOptions);
tParams.setQueryId(context.queryId());
Future<InternalService.PConstantExprResult> future
= BackendServiceProxy.getInstance().foldConstantExpr(brpcAddress, tParams);
@ -375,14 +381,37 @@ public class FoldConstantsRule implements ExprRewriteRule {
Map<String, Expr> tmp = new HashMap<>();
for (Map.Entry<String, InternalService.PExprResult> entry1
: entry.getValue().getMapMap().entrySet()) {
TPrimitiveType type = TPrimitiveType.findByValue(entry1.getValue().getType().getType());
PScalarType scalarType = entry1.getValue().getType();
TPrimitiveType ttype = TPrimitiveType.findByValue(scalarType.getType());
Expr retExpr = null;
if (entry1.getValue().getSuccess()) {
Type type = null;
if (ttype == TPrimitiveType.CHAR) {
Preconditions.checkState(scalarType.hasLen());
type = ScalarType.createCharType(scalarType.getLen());
} else if (ttype == TPrimitiveType.VARCHAR) {
Preconditions.checkState(scalarType.hasLen());
type = ScalarType.createVarcharType(scalarType.getLen());
} else if (ttype == TPrimitiveType.DECIMALV2) {
type = ScalarType.createDecimalType(scalarType.getPrecision(),
scalarType.getScale());
} else if (ttype == TPrimitiveType.DATETIMEV2) {
type = ScalarType.createDatetimeV2Type(scalarType.getScale());
} else if (ttype == TPrimitiveType.DECIMAL32
|| ttype == TPrimitiveType.DECIMAL64
|| ttype == TPrimitiveType.DECIMAL128I) {
type = ScalarType.createDecimalV3Type(scalarType.getPrecision(),
scalarType.getScale());
} else {
type = ScalarType.createType(
PrimitiveType.fromThrift(ttype));
}
retExpr = LiteralExpr.create(entry1.getValue().getContent(),
Type.fromPrimitiveType(PrimitiveType.fromThrift(type)));
type);
} else {
retExpr = allConstMap.get(entry1.getKey());
}
LOG.debug("retExpr: " + retExpr.toString());
tmp.put(entry1.getKey(), retExpr);
}
if (!tmp.isEmpty()) {

11
fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.rewrite.FoldConstantsRule;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TQueryOptions;
import org.apache.doris.utframe.DorisAssert;
import org.apache.doris.utframe.UtFrameUtils;
@ -276,7 +277,9 @@ public class QueryStmtTest {
+ "FROM\n"
+ " (SELECT curdate()) a;";
StatementBase stmt = UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
stmt.foldConstant(new Analyzer(ctx.getEnv(), ctx).getExprRewriter());
SessionVariable sessionVariable = new SessionVariable();
TQueryOptions queryOptions = sessionVariable.getQueryOptionVariables();
stmt.foldConstant(new Analyzer(ctx.getEnv(), ctx).getExprRewriter(), queryOptions);
// reAnalyze
reAnalyze(stmt, ctx);
@ -298,7 +301,7 @@ public class QueryStmtTest {
+ " t2.k2 = @@language\n"
+ ")";
stmt = UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
stmt.foldConstant(new Analyzer(ctx.getEnv(), ctx).getExprRewriter());
stmt.foldConstant(new Analyzer(ctx.getEnv(), ctx).getExprRewriter(), queryOptions);
// reAnalyze
reAnalyze(stmt, ctx);
Assert.assertTrue(stmt.toSql().contains("Apache License, Version 2.0"));
@ -319,7 +322,7 @@ public class QueryStmtTest {
+ " t2.k2 = CONNECTION_ID()\n"
+ ")";
stmt = UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
stmt.foldConstant(new Analyzer(ctx.getEnv(), ctx).getExprRewriter());
stmt.foldConstant(new Analyzer(ctx.getEnv(), ctx).getExprRewriter(), queryOptions);
// reAnalyze
reAnalyze(stmt, ctx);
Assert.assertTrue(stmt.toSql().contains("root''@''%"));
@ -332,7 +335,7 @@ public class QueryStmtTest {
+ " (select USER() k1, CURRENT_USER() k2, SCHEMA() k3) t1,\n"
+ " (select @@license k1, @@version k2) t2\n";
stmt = UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
stmt.foldConstant(new Analyzer(ctx.getEnv(), ctx).getExprRewriter());
stmt.foldConstant(new Analyzer(ctx.getEnv(), ctx).getExprRewriter(), queryOptions);
// reAnalyze
reAnalyze(stmt, ctx);
Assert.assertTrue(stmt.toSql().contains("root''@''%"));