database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](meta) Infer the column name when create view if the column is expression (#24990)

Browse Source 
## Proposed changes

Infer the column name when create view if the column is expression

## Further comments
expr column name infer strategy as following:
|      expr       |                example                    |           column name(before)             | Inferred column name(if position is 2)  |
|  -------------  | ---------------------------------------   | ------------------------------            | --------------------------------------  |
| function        | dayofyear()                               | dayofyear()                               | __dayofyear_1                           |
| cast            | cast(1 as bigint)                         | CAST(1 AS BIGINT)                         | __cast_1                                |
| anylyticExpr    | min()                                     | min()                                     | __min_1                                 |
| predicate       | 1 in (1,2,3,4)                            | 1 IN (1, 2, 3, 4)                         | __in_predicate_1                        |
| literal         | 1 or 'string_var_name'                    | 1 or 'string_var_name'                    | __literal_1                             |
| arithmeticExpr  | &                                         | ... & ...                                 | __arithmetic_expr_1                     |
| identifier      | a or b                                    | a or b                                    | a or b                                  |
| case            | CASE WHEN remark = 's' THEN 1 ELSE 2 END  | CASE WHEN remark = 's' THEN 1 ELSE 2 END  | __case_1                                |
| window          | min(timestamp) OVER (...)                 | min(timestamp) OVER(...)                  | __min_1                                 |


SQL for example:
```sql
CREATE VIEW v1 AS 
SELECT 
  error_code,
  1, 
  'string', 
  now(), 
  dayofyear(op_time), 
  cast (source AS BIGINT), 
  min(`timestamp`) OVER (
    ORDER BY 
      op_time DESC ROWS BETWEEN UNBOUNDED PRECEDING
      AND 1 FOLLOWING
  ), 
  1 > 2,
  2 + 3,
  1 IN (1, 2, 3, 4), 
  remark LIKE '%like', 
  CASE WHEN remark = 's' THEN 1 ELSE 2 END,
  TRUE | FALSE 
FROM 
  db_test.table_test1
```

the output column name is as following:
```
error_code
__literal_1
__literal_2
__now_3
__dayofyear_4
__cast_expr_5
__min_6
__binary_predicate_7
__arithmetic_expr_8
__in_predicate_9
__like_predicate_10
__case_expr_11
__arithmetic_expr_12
```
This commit is contained in:
JingDas
2023-10-09 17:14:01 +08:00
committed by GitHub
parent 79fa1d1640
commit 263631e983
22 changed files with 210 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.TreeNode;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.thrift.TExprNode;
import com.google.common.base.Joiner;
@ -144,6 +145,11 @@ public class AnalyticExpr extends Expr {
return window;
}
@Override
protected String getExprName() {
return Utils.normalizeName(getFnCall().getExprName(), DEFAULT_EXPR_NAME);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), fnCall, orderByElements, window);

6
fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.analysis;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.thrift.TColumnRef;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -53,6 +54,11 @@ public class ColumnRefExpr extends Expr {
return columnName;
}
@Override
protected String getExprName() {
return Utils.normalizeName(getName(), DEFAULT_EXPR_NAME);
}
public void setName(String name) {
this.columnName = name;
}

9
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -37,6 +37,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.io.Writable;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.mvrewrite.MVExprEquivalent;
import org.apache.doris.statistics.ExprStats;
@ -82,6 +83,7 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
public static final String AGG_STATE_SUFFIX = "_state";
public static final String AGG_UNION_SUFFIX = "_union";
public static final String AGG_MERGE_SUFFIX = "_merge";
public static final String DEFAULT_EXPR_NAME = "expr";
protected boolean disableTableName = false;
protected boolean needToMysql = false;
@ -292,6 +294,7 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
// Flag to indicate whether to wrap this expr's toSql() in parenthesis. Set by parser.
// Needed for properly capturing expr precedences in the SQL string.
protected boolean printSqlInParens = false;
protected final String exprName = Utils.normalizeName(this.getClass().getSimpleName(), DEFAULT_EXPR_NAME);
protected Expr() {
super();
@ -333,6 +336,12 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
this.id = id;
}
// Name of expr, this is used by generating column name automatically when there is no
// alias or is not slotRef
protected String getExprName() {
return this.exprName;
}
public Type getType() {
return type;
}

6
fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View File

@ -38,6 +38,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -309,6 +310,11 @@ public class FunctionCallExpr extends Expr {
super();
}
@Override
protected String getExprName() {
return Utils.normalizeName(this.getFnName().getFunction(), DEFAULT_EXPR_NAME);
}
public FunctionCallExpr(String functionName, List<Expr> params) {
this(new FunctionName(functionName), new FunctionParams(false, params));
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
View File

@ -357,6 +357,11 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
}
}
@Override
protected String getExprName() {
return "literal";
}
// Port from mysql get_param_length
public static int getParmLen(ByteBuffer data) {
int maxLen = data.remaining();

20
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java
View File

@ -117,8 +117,10 @@ public class SelectListItem {
}
/**
* Return a column label for the select list item.
* Return a column label for the select list item. Without generate column name
* automatically.
*/
@Deprecated
public String toColumnLabel() {
Preconditions.checkState(!isStar());
if (alias != null) {
@ -132,6 +134,22 @@ public class SelectListItem {
return expr.toColumnLabel();
}
/**
* Return a column label for the select list item. Support to generate
* column label automatically when can not get the column label exactly.
* Need the position of selectListItem to generate column label
*/
public String toColumnLabel(int position) {
Preconditions.checkState(!isStar(), "select item should not be star when get column label");
if (alias != null) {
return alias;
}
if (expr instanceof SlotRef) {
return expr.getExprName();
}
return "__" + expr.getExprName() + "_" + position;
}
public void setAlias(String alias) {
this.alias = alias;
}

9
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View File

@ -524,7 +524,9 @@ public class SelectStmt extends QueryStmt {
colLabels.removeIf(exceptCols::contains);
} else {
for (SelectListItem item : selectList.getItems()) {
List<SelectListItem> items = selectList.getItems();
for (int i = 0; i < items.size(); i++) {
SelectListItem item = items.get(i);
if (item.isStar()) {
TableName tblName = item.getTblName();
if (tblName == null) {
@ -541,7 +543,8 @@ public class SelectStmt extends QueryStmt {
throw new AnalysisException("Subquery is not supported in the select list.");
}
resultExprs.add(rewriteQueryExprByMvColumnExpr(item.getExpr(), analyzer));
SlotRef aliasRef = new SlotRef(null, item.toColumnLabel());
String columnLabel = item.toColumnLabel(i);
SlotRef aliasRef = new SlotRef(null, columnLabel);
Expr existingAliasExpr = aliasSMap.get(aliasRef);
if (existingAliasExpr != null && !existingAliasExpr.equals(item.getExpr())) {
// If we have already seen this alias, it refers to more than one column and
@ -549,7 +552,7 @@ public class SelectStmt extends QueryStmt {
ambiguousAliasList.add(aliasRef);
}
aliasSMap.put(aliasRef, item.getExpr().clone());
colLabels.add(item.toColumnLabel());
colLabels.add(columnLabel);
}
}
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
View File

@ -305,6 +305,11 @@ public class SlotRef extends Expr {
return col;
}
@Override
protected String getExprName() {
return toColumnLabel();
}
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.SLOT_REF;

5
fe/fe-core/src/main/java/org/apache/doris/analysis/VirtualSlotRef.java
View File

@ -121,4 +121,9 @@ public class VirtualSlotRef extends SlotRef {
desc = analyzer.registerVirtualColumnRef(super.getColumnName(), type, tupleDescriptor);
numDistinctValues = desc.getStats().getNumDistinctValues();
}
@Override
protected String getExprName() {
return super.getExprName();
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Joiner;
@ -60,6 +61,11 @@ public class UnboundFunction extends Function implements Unbound, PropagateNulla
return name;
}
@Override
protected String getExpressionName() {
return Utils.normalizeName(getName(), DEFAULT_EXPRESSION_NAME);
}
public String getDbName() {
return dbName;
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.trees.plans.AggMode;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
@ -117,6 +118,11 @@ public class AggregateExpression extends Expression implements UnaryExpression {
}
}
@Override
protected String getExpressionName() {
return Utils.normalizeName(function.getName(), DEFAULT_EXPRESSION_NAME);
}
@Override
public boolean equals(Object o) {
if (this == o) {

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AssertNumRowsElement.java
View File

@ -87,6 +87,11 @@ public class AssertNumRowsElement extends Expression implements LeafExpression,
return toString();
}
@Override
protected String getExpressionName() {
return assertion.name().toLowerCase();
}
@Override
public boolean equals(Object o) {
if (this == o) {

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
View File

@ -34,6 +34,7 @@ import org.apache.doris.nereids.types.MapType;
import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@ -49,9 +50,10 @@ import java.util.stream.Collectors;
* Abstract class for all Expression in Nereids.
*/
public abstract class Expression extends AbstractTreeNode<Expression> implements ExpressionTrait {
public static final String DEFAULT_EXPRESSION_NAME = "expression";
// Mask this expression is generated by rule, should be removed.
public boolean isGeneratedIsNotNull = false;
protected final String exprName = Utils.normalizeName(this.getClass().getSimpleName(), DEFAULT_EXPRESSION_NAME);
private final int depth;
private final int width;
@ -92,6 +94,12 @@ public abstract class Expression extends AbstractTreeNode<Expression> implements
return new Alias(this, alias);
}
// Name of expr, this is used by generating column name automatically when there is no
// alias
protected String getExpressionName() {
return this.exprName;
}
/**
* check input data types
*/

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpression.java
View File

@ -56,4 +56,9 @@ public abstract class NamedExpression extends Expression {
public String getQualifiedName() throws UnboundException {
return Utils.qualifiedName(getQualifier(), getName());
}
@Override
protected String getExpressionName() {
return Utils.normalizeName(getName(), DEFAULT_EXPRESSION_NAME);
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java
View File

@ -84,6 +84,11 @@ public abstract class SubqueryExpr extends Expression implements LeafExpression
return "(" + queryPlan + ")";
}
@Override
protected String getExpressionName() {
return "subquery";
}
@Override
public String toString() {
return Utils.toSqlString("SubqueryExpr",

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Suppliers;
@ -54,6 +55,11 @@ public abstract class BoundFunction extends Function implements ComputeSignature
return name;
}
@Override
protected String getExpressionName() {
return Utils.normalizeName(getName(), DEFAULT_EXPRESSION_NAME);
}
public FunctionSignature getSignature() {
return signatureCache.get();
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
View File

@ -131,6 +131,11 @@ public abstract class Literal extends Expression implements LeafExpression, Comp
return toString();
}
@Override
protected String getExpressionName() {
return "literal";
}
@Override
public boolean nullable() throws UnboundException {
return this instanceof NullLiteral;

14
fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import com.google.common.base.CaseFormat;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@ -261,4 +262,17 @@ public class Utils {
public static <T> List<T> copyRequiredMutableList(List<T> list) {
return Lists.newArrayList(Objects.requireNonNull(list, "non-null list is required"));
}
/**
* Normalize the name to lower underscore style, return default name if the name is empty.
*/
public static String normalizeName(String name, String defaultName) {
if (StringUtils.isEmpty(name)) {
return defaultName;
}
if (name.contains("$")) {
name = name.replace("$", "_");
}
return CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, name);
}
}