database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids) parse pipe_concat symbol as concat when sql mode set to PIPES_AS_CONCAT (#15775)

Browse Source
This commit is contained in:
谢健
2023-01-11 21:41:14 +08:00
committed by GitHub
parent ea1493d946
commit 88a2088c1d
10 changed files with 115 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
View File

@ -416,7 +416,7 @@ TILDE: '~';
AMPERSAND: '&';
LOGICALAND: '&&';
PIPE: '|';
CONCAT_PIPE: '||';
DOUBLEPIPES: '||';
HAT: '^';
COLON: ':';
ARROW: '->';

13
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View File

@ -245,13 +245,14 @@ expression
;
booleanExpression
: NOT booleanExpression #logicalNot
| EXISTS LEFT_PAREN query RIGHT_PAREN #exist
| (ISNULL | IS_NULL_PRED) LEFT_PAREN valueExpression RIGHT_PAREN #isnull
| IS_NOT_NULL_PRED LEFT_PAREN valueExpression RIGHT_PAREN #is_not_null_pred
| valueExpression predicate? #predicated
: NOT booleanExpression #logicalNot
| EXISTS LEFT_PAREN query RIGHT_PAREN #exist
| (ISNULL | IS_NULL_PRED) LEFT_PAREN valueExpression RIGHT_PAREN #isnull
| IS_NOT_NULL_PRED LEFT_PAREN valueExpression RIGHT_PAREN #is_not_null_pred
| valueExpression predicate? #predicated
| left=booleanExpression operator=(AND | LOGICALAND) right=booleanExpression #logicalBinary
| left=booleanExpression operator=(OR | CONCAT_PIPE) right=booleanExpression #logicalBinary
| left=booleanExpression operator=OR right=booleanExpression #logicalBinary
| left=booleanExpression operator=DOUBLEPIPES right=booleanExpression #doublePipes
;
predicate

52
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.parser;
import org.apache.doris.analysis.ArithmeticExpr.Operator;
import org.apache.doris.analysis.SetType;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.DorisParser.AggClauseContext;
@ -143,7 +144,6 @@ import org.apache.doris.nereids.trees.expressions.ScalarSubquery;
import org.apache.doris.nereids.trees.expressions.Subtract;
import org.apache.doris.nereids.trees.expressions.TVFProperties;
import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
import org.apache.doris.nereids.trees.expressions.VariableDesc;
import org.apache.doris.nereids.trees.expressions.WhenClause;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
@ -171,12 +171,14 @@ import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Interval;
import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.JoinHint;
@ -213,7 +215,13 @@ import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.policy.PolicyTypeEnum;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.SqlModeHelper;
import org.apache.doris.qe.VariableMgr;
import org.apache.doris.qe.VariableVarConverters;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
@ -527,17 +535,29 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
@Override
public Expression visitSystemVariable(SystemVariableContext ctx) {
String name = ctx.identifier().getText();
SessionVariable sessionVariable = ConnectContext.get().getSessionVariable();
Literal literal = null;
if (ctx.kind == null) {
return new VariableDesc(SetType.SESSION, ctx.identifier().getText());
literal = VariableMgr.getLiteral(sessionVariable, name, SetType.DEFAULT);
} else if (ctx.kind.getType() == DorisParser.SESSION) {
literal = VariableMgr.getLiteral(sessionVariable, name, SetType.SESSION);
} else if (ctx.kind.getType() == DorisParser.GLOBAL) {
literal = VariableMgr.getLiteral(sessionVariable, name, SetType.GLOBAL);
}
switch (ctx.kind.getType()) {
case DorisParser.GLOBAL:
return new VariableDesc(SetType.GLOBAL, ctx.identifier().getText());
case DorisParser.SESSION:
return new VariableDesc(SetType.SESSION, ctx.identifier().getText());
default:
throw new ParseException("Unsupported system variable: " + ctx.getText(), ctx);
if (literal == null) {
throw new ParseException("Unsupported system variable: " + ctx.getText(), ctx);
}
if (!Strings.isNullOrEmpty(name) && VariableVarConverters.hasConverter(name)) {
try {
Preconditions.checkArgument(literal instanceof IntegerLikeLiteral);
IntegerLikeLiteral integerLikeLiteral = (IntegerLikeLiteral) literal;
literal = new StringLiteral(VariableVarConverters.decode(name, integerLikeLiteral.getLongValue()));
} catch (DdlException e) {
throw new ParseException(e.getMessage(), ctx);
}
}
return literal;
}
@Override
@ -607,7 +627,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
case DorisParser.AND:
return new And(left, right);
case DorisParser.OR:
case DorisParser.CONCAT_PIPE:
return new Or(left, right);
default:
throw new ParseException("Unsupported logical binary type: " + ctx.operator.getText(), ctx);
@ -808,6 +827,19 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
+ ", supported time unit: YEAR/MONTH/DAY/HOUR/MINUTE/SECOND", ctx);
}
@Override
public Expression visitDoublePipes(DorisParser.DoublePipesContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {
Expression left = getExpression(ctx.left);
Expression right = getExpression(ctx.right);
if (ConnectContext.get().getSessionVariable().getSqlMode() == SqlModeHelper.MODE_PIPES_AS_CONCAT) {
return new UnboundFunction("concat", Lists.newArrayList(left, right));
} else {
return new Or(left, right);
}
});
}
/**
* Create a value based [[CaseWhen]] expression. This has the following SQL form:
* {{{

11
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRuleOnFE.java
View File

@ -40,7 +40,6 @@ import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
import org.apache.doris.nereids.trees.expressions.VariableDesc;
import org.apache.doris.nereids.trees.expressions.WhenClause;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
@ -55,13 +54,10 @@ import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.qe.VariableMgr;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
@ -348,13 +344,6 @@ public class FoldConstantRuleOnFE extends AbstractExpressionRewriteRule {
return Literal.of(isNull.child().nullable());
}
@Override
public Expression visitVariableDesc(VariableDesc variableDesc, ExpressionRewriteContext context) {
Preconditions.checkArgument(variableDesc.isSystemVariable());
return new StringLiteral(VariableMgr.getValue(context.connectContext.getSessionVariable(), variableDesc));
}
@Override
public Expression visitTimestampArithmetic(TimestampArithmetic arithmetic, ExpressionRewriteContext context) {
return ExpressionEvaluator.INSTANCE.eval(arithmetic);

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLikeLiteral.java
View File

@ -34,5 +34,9 @@ public abstract class IntegerLikeLiteral extends Literal {
return getNumber().intValue();
}
public long getLongValue() {
return getNumber().longValue();
}
public abstract Number getNumber();
}

2
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -294,7 +294,7 @@ public class SessionVariable implements Serializable, Writable {
// Set sqlMode to empty string
@VariableMgr.VarAttr(name = SQL_MODE, needForward = true)
public long sqlMode = 0L;
public long sqlMode = SqlModeHelper.MODE_DEFAULT;
@VariableMgr.VarAttr(name = RESOURCE_VARIABLE)
public String resourceGroup = "normal";

14
fe/fe-core/src/main/java/org/apache/doris/qe/SqlModeHelper.java
View File

@ -40,7 +40,8 @@ public class SqlModeHelper {
// of how they works and to be compatible with MySQL, so for now they are not
// really meaningful.
/* Bits for different SQL MODE modes, you can add custom SQL MODE here */
public static final long MODE_REAL_AS_FLOAT = 1L;
/* When a new session is created, its sql mode is set to MODE_DEFAULT */
public static final long MODE_DEFAULT = 1L;
public static final long MODE_PIPES_AS_CONCAT = 2L;
public static final long MODE_ANSI_QUOTES = 4L;
public static final long MODE_IGNORE_SPACE = 8L;
@ -70,9 +71,8 @@ public class SqlModeHelper {
public static final long MODE_TRADITIONAL = 1L << 27;
public static final long MODE_LAST = 1L << 33;
public static final long MODE_REAL_AS_FLOAT = 1L << 34;
/* When a new session is create, its sql mode is set to MODE_DEFAULT */
public static final long MODE_DEFAULT = 0L;
public static final long MODE_ALLOWED_MASK =
(MODE_REAL_AS_FLOAT | MODE_PIPES_AS_CONCAT | MODE_ANSI_QUOTES | MODE_IGNORE_SPACE | MODE_NOT_USED
@ -80,7 +80,8 @@ public class SqlModeHelper {
| MODE_NO_AUTO_VALUE_ON_ZERO | MODE_NO_BACKSLASH_ESCAPES | MODE_STRICT_TRANS_TABLES
| MODE_STRICT_ALL_TABLES | MODE_NO_ZERO_IN_DATE | MODE_NO_ZERO_DATE | MODE_INVALID_DATES
| MODE_ERROR_FOR_DIVISION_BY_ZERO | MODE_HIGH_NOT_PRECEDENCE | MODE_NO_ENGINE_SUBSTITUTION
| MODE_PAD_CHAR_TO_FULL_LENGTH | MODE_TRADITIONAL | MODE_ANSI | MODE_TIME_TRUNCATE_FRACTIONAL);
| MODE_PAD_CHAR_TO_FULL_LENGTH | MODE_TRADITIONAL | MODE_ANSI | MODE_TIME_TRUNCATE_FRACTIONAL
| MODE_DEFAULT);
public static final long MODE_COMBINE_MASK = (MODE_ANSI | MODE_TRADITIONAL);
@ -89,6 +90,7 @@ public class SqlModeHelper {
private static final Map<String, Long> combineModeSet = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
static {
sqlModeSet.put("DEFAULT", MODE_DEFAULT);
sqlModeSet.put("REAL_AS_FLOAT", MODE_REAL_AS_FLOAT);
sqlModeSet.put("PIPES_AS_CONCAT", MODE_PIPES_AS_CONCAT);
sqlModeSet.put("ANSI_QUOTES", MODE_ANSI_QUOTES);
@ -121,8 +123,8 @@ public class SqlModeHelper {
// convert long type SQL MODE to string type that user can read
public static String decode(Long sqlMode) throws DdlException {
// 0 parse to empty string
if (sqlMode == 0) {
if (sqlMode == MODE_DEFAULT) {
//For compatibility with older versions， return empty string
return "";
}
if ((sqlMode & ~MODE_ALLOWED_MASK) != 0) {

51
fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
View File

@ -27,7 +27,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.nereids.trees.expressions.VariableDesc;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.persist.GlobalVarPersistInfo;
import com.google.common.base.Preconditions;
@ -54,6 +54,7 @@ import java.util.Map;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import javax.annotation.Nullable;
/**
* Variable manager, merge session variable and global variable.
@ -442,12 +443,50 @@ public class VariableMgr {
}
// For Nereids optimizer
public static String getValue(SessionVariable var, VariableDesc desc) throws RuntimeException {
try {
return getValue(var, desc.getName(), desc.getSetType());
} catch (AnalysisException e) {
throw new RuntimeException(e);
public static @Nullable Literal getLiteral(SessionVariable var, String name, SetType setType) {
VarContext ctx = ctxByVarName.get(name);
if (ctx == null) {
return null;
}
if (setType == SetType.GLOBAL) {
rlock.lock();
try {
return getLiteral(ctx.getObj(), ctx.getField());
} finally {
rlock.unlock();
}
} else {
return getLiteral(var, ctx.getField());
}
}
private static Literal getLiteral(Object obj, Field field) {
try {
switch (field.getType().getSimpleName()) {
case "boolean":
return Literal.of(field.getBoolean(obj));
case "byte":
return Literal.of(field.getByte(obj));
case "short":
return Literal.of(field.getShort(obj));
case "int":
return Literal.of(field.getInt(obj));
case "long":
return Literal.of(field.getLong(obj));
case "float":
return Literal.of(field.getFloat(obj));
case "double":
return Literal.of(field.getDouble(obj));
case "String":
return Literal.of((String) field.get(obj));
default:
return Literal.of("");
}
} catch (IllegalAccessException e) {
LOG.warn("Access failed.", e);
}
return Literal.of("");
}
private static String getValue(Object obj, Field field) {