database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](Materialized-View) support duplicate base column for diffrent aggregate function (#15837)

Browse Source 
support duplicate base column for diffrent aggregate function
This commit is contained in:
Pxl
2023-02-02 18:57:39 +08:00
committed by GitHub
parent e31913faca
commit 0d5b115993
44 changed files with 883 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View File

@ -468,18 +468,14 @@ public class MaterializedViewHandler extends AlterHandler {
}
String mvColumnName = mvColumnItem.getBaseColumnNames().iterator().next();
Column baseColumn = olapTable.getColumn(mvColumnName);
Column mvColumn = mvColumnItem.toMVColumn(olapTable);
if (mvColumnItem.isKey()) {
++numOfKeys;
}
if (baseColumn == null) {
throw new DdlException("The mv column of agg or uniq table cannot be transformed "
+ "from original column[" + String.join(",", mvColumnItem.getBaseColumnNames()) + "]");
}
Preconditions.checkNotNull(baseColumn, "Column[" + mvColumnName + "] does not exist");
AggregateType baseAggregationType = baseColumn.getAggregationType();
AggregateType baseAggregationType = mvColumn.getAggregationType();
AggregateType mvAggregationType = mvColumnItem.getAggregationType();
if (baseColumn.isKey() && !mvColumnItem.isKey()) {
if (mvColumn.isKey() && !mvColumnItem.isKey()) {
throw new DdlException("The column[" + mvColumnName + "] must be the key of materialized view");
}
if (baseAggregationType != mvAggregationType) {

157
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View File

@ -18,30 +18,26 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.UserException;
import org.apache.doris.rewrite.mvrewrite.CountFieldToSum;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Materialized view is performed to materialize the results of query.
@ -60,6 +56,8 @@ public class CreateMaterializedViewStmt extends DdlStmt {
private static final Logger LOG = LogManager.getLogger(CreateMaterializedViewStmt.class);
public static final String MATERIALIZED_VIEW_NAME_PREFIX = "mv_";
public static final String MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX = "mva_";
public static final String MATERIALIZED_VIEW_AGGREGATE_NAME_LINK = "__";
public static final Map<String, MVColumnPattern> FN_NAME_TO_PATTERN;
static {
@ -168,8 +166,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
throw new AnalysisException("The materialized view must contain at least one column");
}
boolean meetAggregate = false;
// TODO(ml): support same column with different aggregation function
Set<String> mvColumnNameSet = Sets.newHashSet();
/**
* 1. The columns of mv must be a single column or a aggregate column without any calculate.
* Also the children of aggregate column must be a single column without any calculate.
@ -209,10 +205,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
List<SlotRef> slots = new ArrayList<>();
functionCallExpr.collect(SlotRef.class, slots);
Preconditions.checkArgument(slots.size() == 1);
String columnName = slots.get(0).getColumnName().toLowerCase();
if (!mvColumnNameSet.add(columnName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, columnName);
}
if (beginIndexOfAggregation == -1) {
beginIndexOfAggregation = i;
@ -358,26 +350,16 @@ public class CreateMaterializedViewStmt extends DdlStmt {
private MVColumnItem buildMVColumnItem(Analyzer analyzer, FunctionCallExpr functionCallExpr)
throws AnalysisException {
String functionName = functionCallExpr.getFnName().getFunction();
List<SlotRef> slots = new ArrayList<>();
functionCallExpr.collect(SlotRef.class, slots);
Preconditions.checkArgument(slots.size() == 1);
SlotRef baseColumnRef = slots.get(0);
String baseColumnName = baseColumnRef.getColumnName().toLowerCase();
Column baseColumn = baseColumnRef.getColumn();
if (baseColumn == null) {
throw new AnalysisException("baseColumn is null");
}
Type baseType = baseColumn.getOriginType();
Expr functionChild0 = functionCallExpr.getChild(0);
String mvColumnName;
AggregateType mvAggregateType;
Expr defineExpr = baseColumnRef;
List<Expr> childs = functionCallExpr.getChildren();
Preconditions.checkArgument(childs.size() == 1);
Expr defineExpr = childs.get(0);
Type baseType = defineExpr.getType();
AggregateType mvAggregateType = null;
Type type;
switch (functionName.toLowerCase()) {
case "sum":
mvColumnName = mvColumnBuilder(baseColumnName);
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
PrimitiveType baseColumnType = baseColumnRef.getType().getPrimitiveType();
PrimitiveType baseColumnType = baseType.getPrimitiveType();
if (baseColumnType == PrimitiveType.TINYINT || baseColumnType == PrimitiveType.SMALLINT
|| baseColumnType == PrimitiveType.INT) {
type = Type.BIGINT;
@ -387,51 +369,44 @@ public class CreateMaterializedViewStmt extends DdlStmt {
type = baseType;
}
if (type != baseType) {
defineExpr = new CastExpr(type, baseColumnRef);
defineExpr.analyze(analyzer);
defineExpr = new CastExpr(type, defineExpr);
if (analyzer != null) {
defineExpr.analyze(analyzer);
}
}
break;
case "min":
case "max":
mvColumnName = mvColumnBuilder(baseColumnName);
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
type = baseType;
break;
case FunctionSet.BITMAP_UNION:
// Compatible aggregation models
if (baseColumnRef.getType().getPrimitiveType() == PrimitiveType.BITMAP) {
mvColumnName = mvColumnBuilder(baseColumnName);
} else {
mvColumnName = mvColumnBuilder(functionName, baseColumnName);
defineExpr = functionChild0;
}
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
type = Type.BITMAP;
if (analyzer != null && !baseType.isBitmapType()) {
throw new AnalysisException(
"BITMAP_UNION need input a bitmap column, but input " + baseType.toString());
}
break;
case FunctionSet.HLL_UNION:
// Compatible aggregation models
if (baseColumnRef.getType().getPrimitiveType() == PrimitiveType.HLL) {
mvColumnName = mvColumnBuilder(baseColumnName);
} else {
mvColumnName = mvColumnBuilder(functionName, baseColumnName);
defineExpr = functionChild0;
}
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
type = Type.HLL;
if (analyzer != null && !baseType.isHllType()) {
throw new AnalysisException("HLL_UNION need input a hll column, but input " + baseType.toString());
}
break;
case FunctionSet.COUNT:
mvColumnName = mvColumnBuilder(functionName, baseColumnName);
mvAggregateType = AggregateType.SUM;
defineExpr = new CaseExpr(null, Lists.newArrayList(new CaseWhenClause(
new IsNullPredicate(baseColumnRef, false),
new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT));
defineExpr.analyze(analyzer);
defineExpr = CountFieldToSum.slotToCaseWhen(defineExpr);
if (analyzer != null) {
defineExpr.analyze(analyzer);
}
type = Type.BIGINT;
break;
default:
throw new AnalysisException("Unsupported function:" + functionName);
}
return new MVColumnItem(mvColumnName, type, mvAggregateType, false, defineExpr, baseColumnName);
if (mvAggregateType == null) {
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
}
return new MVColumnItem(type, mvAggregateType, defineExpr, mvColumnBuilder(defineExpr.toSql()));
}
public Map<String, Expr> parseDefineExprWithoutAnalyze() throws AnalysisException {
@ -439,65 +414,35 @@ public class CreateMaterializedViewStmt extends DdlStmt {
SelectList selectList = selectStmt.getSelectList();
for (SelectListItem selectListItem : selectList.getItems()) {
Expr selectListItemExpr = selectListItem.getExpr();
Expr expr = selectListItemExpr;
String name = MaterializedIndexMeta.normalizeName(expr.toSql());
if (selectListItemExpr instanceof FunctionCallExpr) {
FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItemExpr;
List<SlotRef> slots = new ArrayList<>();
functionCallExpr.collect(SlotRef.class, slots);
Preconditions.checkArgument(slots.size() == 1);
String baseColumnName = slots.get(0).getColumnName();
String functionName = functionCallExpr.getFnName().getFunction();
SlotRef baseSlotRef = slots.get(0);
switch (functionName.toLowerCase()) {
switch (functionCallExpr.getFnName().getFunction().toLowerCase()) {
case "sum":
case "min":
case "max":
result.put(baseColumnName, null);
break;
case FunctionSet.BITMAP_UNION:
if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) {
CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef);
List<Expr> params = Lists.newArrayList();
params.add(castExpr);
FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.TO_BITMAP_WITH_CHECK,
params);
result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr);
} else {
result.put(baseColumnName, null);
}
break;
case FunctionSet.HLL_UNION:
if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) {
CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef);
List<Expr> params = Lists.newArrayList();
params.add(castExpr);
FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.HLL_HASH, params);
result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr);
} else {
result.put(baseColumnName, null);
}
break;
case FunctionSet.COUNT:
Expr defineExpr = new CaseExpr(null, Lists.newArrayList(
new CaseWhenClause(new IsNullPredicate(slots.get(0), false),
new IntLiteral(0, Type.BIGINT))),
new IntLiteral(1, Type.BIGINT));
result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr);
MVColumnItem item = buildMVColumnItem(null, functionCallExpr);
expr = item.getDefineExpr();
name = item.getName();
break;
default:
result.put(mvColumnBuilder(functionCallExpr.toSql()), functionCallExpr);
break;
}
} else {
result.put(mvColumnBuilder(selectListItemExpr.toSql()), selectListItemExpr);
}
result.put(name, expr);
}
return result;
}
// for bitmap_union(to_bitmap(column)) function, we should check value is not negative
// for bitmap_union(to_bitmap(column)) function, we should check value is not
// negative
// in vectorized schema_change mode, so we should rewrite the function to
// bitmap_union(to_bitmap_with_check(column))
private void rewriteToBitmapWithCheck() {
public void rewriteToBitmapWithCheck() {
for (SelectListItem item : selectStmt.getSelectList().getItems()) {
if (item.getExpr() instanceof FunctionCallExpr) {
String functionName = ((FunctionCallExpr) item.getExpr()).getFnName().getFunction();
@ -519,10 +464,34 @@ public class CreateMaterializedViewStmt extends DdlStmt {
.append(sourceColumnName).toString();
}
public static String mvColumnBuilder(AggregateType aggregateType, String sourceColumnName) {
return new StringBuilder().append(MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX).append(aggregateType.toSql())
.append("__")
.append(mvColumnBreaker(sourceColumnName)).toString();
}
public static String mvAggregateColumnBuilder(String functionName, String sourceColumnName) {
return new StringBuilder().append(MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX).append(functionName.toUpperCase())
.append(MATERIALIZED_VIEW_AGGREGATE_NAME_LINK)
.append(sourceColumnName).toString();
}
public static String mvColumnBuilder(String name) {
return new StringBuilder().append(MATERIALIZED_VIEW_NAME_PREFIX).append(name).toString();
}
public static String mvColumnBreaker(String name) {
if (name.startsWith(MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
// mva_SUM__k2 -> k2
return mvColumnBreaker(name.substring(name.indexOf(MATERIALIZED_VIEW_AGGREGATE_NAME_LINK)
+ MATERIALIZED_VIEW_AGGREGATE_NAME_LINK.length()));
} else if (name.startsWith(MATERIALIZED_VIEW_NAME_PREFIX)) {
// mv_k2 -> k2
return mvColumnBreaker(name.substring(MATERIALIZED_VIEW_NAME_PREFIX.length()));
}
return name;
}
@Override
public String toSql() {
return null;

41
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.analysis.ArithmeticExpr.Operator;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Function;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
@ -33,6 +34,7 @@ import org.apache.doris.common.TreeNode;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.mvrewrite.MVExprEquivalent;
import org.apache.doris.statistics.ExprStats;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TExprNode;
@ -71,6 +73,8 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
// supports negation.
private static final String NEGATE_FN = "negate";
protected boolean disableTableName = false;
// to be used where we can't come up with a better estimate
public static final double DEFAULT_SELECTIVITY = 0.1;
@ -907,6 +911,20 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return (printSqlInParens) ? "(" + toSqlImpl() + ")" : toSqlImpl();
}
public void setDisableTableName(boolean value) {
disableTableName = value;
for (Expr child : children) {
child.setDisableTableName(value);
}
}
public String toSqlWithoutTbl() {
setDisableTableName(true);
String result = toSql();
setDisableTableName(false);
return result;
}
public String toDigest() {
return (printSqlInParens) ? "(" + toDigestImpl() + ")" : toDigestImpl();
}
@ -2122,12 +2140,16 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
}
public boolean matchExprs(List<Expr> exprs) {
private boolean matchExprsWithoutAlias(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias) {
for (Expr expr : exprs) {
if (expr == null) {
continue;
}
if (expr.toSql().equals(toSql())) {
if (MaterializedIndexMeta.normalizeName(expr.toSqlWithoutTbl()).equals(CreateMaterializedViewStmt
.mvColumnBreaker(MaterializedIndexMeta.normalizeName(toSqlWithoutTbl())))) {
return true;
}
if (MVExprEquivalent.aggregateArgumentEqual(this, expr)) {
return true;
}
}
@ -2137,13 +2159,26 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
for (Expr expr : getChildren()) {
if (!expr.matchExprs(exprs)) {
if (!expr.matchExprs(exprs, stmt, ignoreAlias)) {
return false;
}
}
return true;
}
public boolean matchExprs(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias) {
if (this instanceof SlotRef && ((SlotRef) this).getColumnName() == null) {
return true; // means this is alias of other expr
}
Expr aliasExpr = stmt.getExprFromAliasSMap(this);
if (!ignoreAlias && aliasExpr != null) {
return aliasExpr.matchExprsWithoutAlias(exprs, stmt, true);
} else {
return matchExprsWithoutAlias(exprs, stmt, ignoreAlias);
}
}
protected Type[] getActualArgTypes(Type[] originType) {
return Arrays.stream(originType).map(
(Type type) -> {

10
fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
View File

@ -472,7 +472,8 @@ public class InsertStmt extends DdlStmt {
}
}
}
if (column.isNameWithPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)) {
if (column.isNameWithPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)
|| column.isNameWithPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
SlotRef refColumn = column.getRefColumn();
if (refColumn == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_FIELD_ERROR,
@ -704,14 +705,7 @@ public class InsertStmt extends DdlStmt {
&& ((OlapTable) targetTable).getSequenceMapCol() != null) {
resultExprs.add(exprByName.get(((OlapTable) targetTable).getSequenceMapCol()));
} else if (col.getDefaultValue() == null) {
/*
The import stmt has been filtered in function checkColumnCoverage when
the default value of column is null and column is not nullable.
So the default value of column may simply be null when column is nullable
*/
Preconditions.checkState(col.isAllowNull());
resultExprs.add(NullLiteral.create(col.getType()));
} else {
if (col.getDefaultValueExprDef() != null) {
resultExprs.add(col.getDefaultValueExpr());

6
fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
View File

@ -33,6 +33,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr> {
private static final Logger LOG = LogManager.getLogger(LiteralExpr.class);
@ -369,4 +370,9 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
data.getInt();
return ret;
}
@Override
public boolean matchExprs(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias) {
return true;
}
}

10
fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java
View File

@ -52,6 +52,12 @@ public class MVColumnItem {
this(name, type, aggregateType, isAggregationTypeImplicit, defineExpr, baseColumnName, null);
}
public MVColumnItem(Type type, AggregateType aggregateType,
Expr defineExpr, String baseColumnName) {
this(CreateMaterializedViewStmt.mvColumnBuilder(aggregateType, baseColumnName), type, aggregateType, false,
defineExpr, baseColumnName, null);
}
public MVColumnItem(String name, Type type, AggregateType aggregateType, boolean isAggregationTypeImplicit,
Expr defineExpr, String baseColumnName, String baseTableName) {
this.name = name;
@ -164,13 +170,11 @@ public class MVColumnItem {
if (result.getType() == null) {
throw new DdlException("base column's type is null");
}
result.setName(name);
result.setIsKey(isKey);
// If the mv column type is inconsistent with the base column type, the daily
// test will core.
// So, I comment this line firstly.
// result.setType(type);
result.setAggregationType(aggregationType, isAggregationTypeImplicit);
} else {
if (type == null) {
throw new DdlException("MVColumnItem type is null");
@ -180,6 +184,8 @@ public class MVColumnItem {
result.setIsAllowNull(defineExpr.isNullable());
}
}
result.setName(name);
result.setAggregationType(aggregationType, isAggregationTypeImplicit);
result.setDefineExpr(defineExpr);
return result;
}

6
fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java
View File

@ -41,11 +41,7 @@ public class MVColumnOneChildPattern implements MVColumnPattern {
if (functionCallExpr.getChildren().size() != 1) {
return false;
}
if (functionCallExpr.getChild(0).unwrapSlotRef() == null) {
return false;
} else {
return true;
}
return true;
}
@Override

6
fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
View File

@ -76,6 +76,8 @@ public abstract class QueryStmt extends StatementBase implements Queriable {
*/
protected ArrayList<Expr> resultExprs = Lists.newArrayList();
protected ArrayList<Expr> originResultExprs = null;
// For a select statement: select list exprs resolved to base tbl refs
// For a union statement: same as resultExprs
/**
@ -548,6 +550,10 @@ public abstract class QueryStmt extends StatementBase implements Queriable {
return false;
}
public Expr getExprFromAliasSMap(Expr expr) {
return aliasSMap.get(expr);
}
// get tables used by this query.
// Set<String> parentViewNameSet contain parent stmt view name
// to make sure query like "with tmp as (select * from db1.table1) " +

5
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View File

@ -210,8 +210,8 @@ public class SelectStmt extends QueryStmt {
if (getAggInfo() != null && getAggInfo().getGroupingExprs() != null) {
exprs.addAll(getAggInfo().getGroupingExprs());
}
if (originSelectList != null) {
exprs.addAll(originSelectList.getExprs());
if (originResultExprs != null) {
exprs.addAll(originResultExprs);
}
if (havingClause != null) {
exprs.add(havingClause);
@ -594,6 +594,7 @@ public class SelectStmt extends QueryStmt {
groupingInfo.substituteGroupingFn(orderingExprNotInSelect, analyzer);
}
}
originResultExprs = Expr.cloneList(resultExprs);
analyzeAggregation(analyzer);
createAnalyticInfo(analyzer);
eliminatingSortNode();

12
fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
View File

@ -111,8 +111,6 @@ public class SlotRef extends Expr {
}
public SlotDescriptor getDesc() {
Preconditions.checkState(isAnalyzed);
Preconditions.checkNotNull(desc);
return desc;
}
@ -229,6 +227,10 @@ public class SlotRef extends Expr {
@Override
public String toSqlImpl() {
if (disableTableName && label != null) {
return label;
}
StringBuilder sb = new StringBuilder();
if (tblName != null) {
@ -245,13 +247,15 @@ public class SlotRef extends Expr {
return label;
}
} else if (desc.getSourceExprs() != null) {
if (ToSqlContext.get() == null || ToSqlContext.get().isNeedSlotRefId()) {
if (!disableTableName && (ToSqlContext.get() == null || ToSqlContext.get().isNeedSlotRefId())) {
if (desc.getId().asInt() != 1) {
sb.append("<slot " + desc.getId().asInt() + ">");
}
}
for (Expr expr : desc.getSourceExprs()) {
sb.append(" ");
if (!disableTableName) {
sb.append(" ");
}
sb.append(expr.toSql());
}
return sb.toString();

2
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View File

@ -208,7 +208,7 @@ public class Column implements Writable, GsonPostProcessable {
}
public String getNameWithoutMvPrefix() {
return this.getNameWithoutPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX);
return CreateMaterializedViewStmt.mvColumnBreaker(name);
}
public String getDisplayName() {

3
fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
View File

@ -918,6 +918,7 @@ public class FunctionSet<T> {
public static final String TO_BITMAP = "to_bitmap";
public static final String TO_BITMAP_WITH_CHECK = "to_bitmap_with_check";
public static final String BITMAP_HASH = "bitmap_hash";
public static final String BITMAP_UNION = "bitmap_union";
public static final String BITMAP_UNION_COUNT = "bitmap_union_count";
public static final String BITMAP_UNION_INT = "bitmap_union_int";
@ -927,6 +928,8 @@ public class FunctionSet<T> {
public static final String ORTHOGONAL_BITMAP_INTERSECT = "orthogonal_bitmap_intersect";
public static final String ORTHOGONAL_BITMAP_INTERSECT_COUNT = "orthogonal_bitmap_intersect_count";
public static final String ORTHOGONAL_BITMAP_UNION_COUNT = "orthogonal_bitmap_union_count";
public static final String APPROX_COUNT_DISTINCT = "approx_count_distinct";
public static final String NDV = "ndv";
public static final String QUANTILE_UNION = "quantile_union";
//TODO(weixiang): is quantile_percent can be replaced by approx_percentile?

1
fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
View File

@ -209,6 +209,7 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
try {
stmt = (CreateMaterializedViewStmt) SqlParserUtils.getStmt(parser, defineStmt.idx);
stmt.setIsReplay(true);
stmt.rewriteToBitmapWithCheck();
Map<String, Expr> columnNameToDefineExpr = stmt.parseDefineExprWithoutAnalyze();
setColumnsDefineExpr(columnNameToDefineExpr);
} catch (Exception e) {

4
fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
View File

@ -80,6 +80,10 @@ public class FeNameFormat {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
columnName, FeNameFormat.COLUMN_NAME_REGEX);
}
if (columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
columnName, FeNameFormat.COLUMN_NAME_REGEX);
}
}
public static void checkLabel(String label) throws AnalysisException {

46
fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.analysis.VirtualSlotRef;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndexMeta;
@ -417,15 +418,18 @@ public class MaterializedViewSelector {
boolean noNeedAggregation = candidateIndexMeta.getKeysType() == KeysType.DUP_KEYS
|| (candidateIndexMeta.getKeysType() == KeysType.UNIQUE_KEYS
&& table.getTableProperty().getEnableUniqueKeyMergeOnWrite());
if (!indexAggColumnExpsList.isEmpty() && selectStmt != null && selectStmt.getAggInfo() != null
&& selectStmt.getAggInfo().getSecondPhaseDistinctAggInfo() != null) {
List<FunctionCallExpr> distinctExprs = selectStmt.getAggInfo().getSecondPhaseDistinctAggInfo()
.getAggregateExprs();
if (!indexAggColumnExpsList.isEmpty() && selectStmt != null && selectStmt.getAggInfo() != null) {
List<FunctionCallExpr> exprs;
if (selectStmt.getAggInfo().getSecondPhaseDistinctAggInfo() != null) {
exprs = selectStmt.getAggInfo().getSecondPhaseDistinctAggInfo()
.getAggregateExprs();
} else {
exprs = selectStmt.getAggInfo().getAggregateExprs();
}
boolean match = false;
for (Expr distinctExpr : distinctExprs) {
for (Expr expr : exprs) {
for (Expr indexExpr : indexAggColumnExpsList) {
if (distinctExpr.toSql() == indexExpr.toSql()) {
if (expr.toSqlWithoutTbl() == indexExpr.toSqlWithoutTbl()) {
match = true;
}
}
@ -476,15 +480,23 @@ public class MaterializedViewSelector {
if (expr == null) {
throw new AnalysisException("match expr input null");
}
String raw = MaterializedIndexMeta.normalizeName(expr.toSql());
if (expr.toSqlWithoutTbl() == null) {
throw new AnalysisException("expr.toSqlWithoutTbl() is null, expr.toSql()=" + expr.toSql());
}
if (expr instanceof VirtualSlotRef) {
continue;
}
String raw = MaterializedIndexMeta.normalizeName(expr.toSqlWithoutTbl());
String withPrefix = CreateMaterializedViewStmt.mvColumnBuilder(raw);
if (indexColumnNames.contains(raw) || indexColumnNames.contains(withPrefix)) {
continue;
}
if (!expr.matchExprs(indexExprs)) {
return false;
if (expr.matchExprs(indexExprs, selectStmt, false)) {
continue;
}
return false;
}
return true;
}
@ -496,6 +508,11 @@ public class MaterializedViewSelector {
if (columnNamesInQueryOutput == null) {
return;
}
Set<String> queryColumnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
List<Expr> exprs = selectStmt.getAllExprs();
columnNamesInQueryOutput
.forEach(name -> queryColumnNames.add(CreateMaterializedViewStmt.mvColumnBreaker(name)));
Iterator<Map.Entry<Long, MaterializedIndexMeta>> iterator = candidateIndexIdToMeta.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<Long, MaterializedIndexMeta> entry = iterator.next();
@ -503,15 +520,14 @@ public class MaterializedViewSelector {
Set<String> indexColumnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
candidateIndexSchema
.forEach(column -> indexColumnNames.add(MaterializedIndexMeta.normalizeName(column.getName())));
.forEach(column -> indexColumnNames.add(CreateMaterializedViewStmt
.mvColumnBreaker(MaterializedIndexMeta.normalizeName(column.getName()))));
List<Expr> indexExprs = new ArrayList<Expr>();
candidateIndexSchema.forEach(column -> indexExprs.add(column.getDefineExpr()));
List<Expr> exprs = selectStmt.getAllExprs();
// The columns in query output must be subset of the columns in SPJ view
if (indexColumnNames.containsAll(columnNamesInQueryOutput)) {
if (indexColumnNames.containsAll(queryColumnNames)) {
continue;
}
if (selectStmt.haveStar() || !matchAllExpr(exprs, indexColumnNames, indexExprs)) {

5
fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -438,7 +438,7 @@ public class OlapScanNode extends ScanNode {
* For example: select count(*) from table (table has a mv named mv1)
* if Optimizer deceide use mv1, we need updateSlotUniqueId.
*/
private void updateSlotUniqueId() {
private void updateSlotUniqueId() throws UserException {
if (!olapTable.getEnableLightSchemaChange() || selectedIndexId == olapTable.getBaseIndexId()) {
return;
}
@ -449,6 +449,9 @@ public class OlapScanNode extends ScanNode {
}
Column baseColumn = slotDescriptor.getColumn();
Column mvColumn = meta.getColumnByName(baseColumn.getName());
if (mvColumn == null) {
throw new UserException("Do not found mvColumn " + baseColumn.getName());
}
slotDescriptor.setColumn(mvColumn);
}
LOG.debug("updateSlotUniqueId() slots: {}", desc.getSlots());

13
fe/fe-core/src/main/java/org/apache/doris/planner/RollupSelector.java
View File

@ -114,8 +114,8 @@ public final class RollupSelector {
return selectedIndexId;
}
private List<Long> selectBestPrefixIndexRollup(List<Expr> conjuncts, boolean isPreAggregation) {
private List<Long> selectBestPrefixIndexRollup(List<Expr> conjuncts, boolean isPreAggregation)
throws UserException {
final List<String> outputColumns = Lists.newArrayList();
for (SlotDescriptor slot : tupleDesc.getMaterializedSlots()) {
Column col = slot.getColumn();
@ -151,11 +151,12 @@ public final class RollupSelector {
}
}
Preconditions.checkArgument(rollupsContainsOutput.size() > 0,
"Can't find candicate rollup contains all output columns.");
if (rollupsContainsOutput.size() == 0) {
throw new UserException("Can't find candicate rollup contains all output columns.");
}
// 2. find all rollups which match the prefix most based on predicates column from containTupleIndices.
// 2. find all rollups which match the prefix most based on predicates column
// from containTupleIndices.
final Set<String> equivalenceColumns = Sets.newHashSet();
final Set<String> unequivalenceColumns = Sets.newHashSet();
collectColumns(conjuncts, equivalenceColumns, unequivalenceColumns);

30
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -1320,8 +1320,14 @@ public class SingleNodePlanner {
if (olapScanNode.getSelectedPartitionIds().size() == 0 && !FeConstants.runningUnitTest) {
continue;
}
// select index by the old Rollup selector
olapScanNode.selectBestRollupByRollupSelector(analyzer);
try {
// select index by the old Rollup selector
olapScanNode.selectBestRollupByRollupSelector(analyzer);
} catch (UserException e) {
LOG.debug("May no rollup index matched");
}
// select index by the new Materialized selector
MaterializedViewSelector.BestIndexInfo bestIndexInfo
= materializedViewSelector.selectBestMV(olapScanNode);
@ -1332,11 +1338,21 @@ public class SingleNodePlanner {
LOG.debug("MV rewriter of tuple [] will be disable", tupleId);
continue;
}
// if the new selected index id is different from the old one, scan node will be updated.
olapScanNode.updateScanRangeInfoByNewMVSelector(bestIndexInfo.getBestIndexId(),
bestIndexInfo.isPreAggregation(), bestIndexInfo.getReasonOfDisable());
if (selectStmt.getAggInfo() != null) {
selectStmt.getAggInfo().updateTypeOfAggregateExprs();
try {
// if the new selected index id is different from the old one, scan node will be
// updated.
olapScanNode.updateScanRangeInfoByNewMVSelector(bestIndexInfo.getBestIndexId(),
bestIndexInfo.isPreAggregation(), bestIndexInfo.getReasonOfDisable());
if (selectStmt.getAggInfo() != null) {
selectStmt.getAggInfo().updateTypeOfAggregateExprs();
}
} catch (Exception e) {
selectFailed = true;
TupleId tupleId = olapScanNode.getTupleId();
selectStmt.updateDisableTuplesMVRewriter(tupleId);
LOG.debug("MV rewriter of tuple [] will be disable", tupleId);
continue;
}
}

15
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -514,6 +514,7 @@ public class StmtExecutor implements ProfileWriter {
Env.getCurrentEnv().getSqlBlockRuleMgr().matchSql(
originStmt.originStmt, context.getSqlHash(), context.getQualifiedUser());
} catch (AnalysisException e) {
e.printStackTrace();
LOG.warn(e.getMessage());
context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
return;
@ -606,18 +607,21 @@ public class StmtExecutor implements ProfileWriter {
context.getState().setError(ErrorCode.ERR_NOT_SUPPORTED_YET, "Do not support this query.");
}
} catch (IOException e) {
e.printStackTrace();
LOG.warn("execute IOException. {}", context.getQueryIdentifier(), e);
// the exception happens when interact with client
// this exception shows the connection is gone
context.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getMessage());
throw e;
} catch (UserException e) {
e.printStackTrace();
// analysis exception only print message, not print the stack
LOG.warn("execute Exception. {}, {}", context.getQueryIdentifier(),
e.getMessage());
context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
context.getState().setErrType(QueryState.ErrType.ANALYSIS_ERR);
} catch (Exception e) {
e.printStackTrace();
LOG.warn("execute Exception. {}", context.getQueryIdentifier(), e);
context.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR,
e.getClass().getSimpleName() + ", msg: " + Util.getRootCauseMessage(e));
@ -634,6 +638,7 @@ public class StmtExecutor implements ProfileWriter {
sessionVariable.setIsSingleSetVar(false);
sessionVariable.clearSessionOriginValue();
} catch (DdlException e) {
e.printStackTrace();
LOG.warn("failed to revert Session value. {}", context.getQueryIdentifier(), e);
context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
}
@ -809,6 +814,7 @@ public class StmtExecutor implements ProfileWriter {
if (parsedStmt instanceof LogicalPlanAdapter) {
throw new NereidsException(new AnalysisException("Unexpected exception: " + e.getMessage(), e));
}
e.printStackTrace();
throw new AnalysisException("Unexpected exception: " + e.getMessage());
} finally {
MetaLockUtils.readUnlockTables(tables);
@ -1030,6 +1036,7 @@ public class StmtExecutor implements ProfileWriter {
SetExecutor executor = new SetExecutor(context, setStmt);
executor.execute();
} catch (DdlException e) {
e.printStackTrace();
// Return error message to client.
context.getState().setError(ErrorCode.ERR_LOCAL_VARIABLE, e.getMessage());
return;
@ -1694,6 +1701,7 @@ public class StmtExecutor implements ProfileWriter {
try {
context.getEnv().changeCatalog(context, switchStmt.getCatalogName());
} catch (DdlException e) {
e.printStackTrace();
context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
return;
}
@ -1727,6 +1735,7 @@ public class StmtExecutor implements ProfileWriter {
}
context.getEnv().changeDb(context, useStmt.getDatabase());
} catch (DdlException e) {
e.printStackTrace();
context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
return;
}
@ -1894,12 +1903,15 @@ public class StmtExecutor implements ProfileWriter {
DdlExecutor.execute(context.getEnv(), (DdlStmt) parsedStmt);
context.getState().setOk();
} catch (QueryStateException e) {
e.printStackTrace();
context.setState(e.getQueryState());
} catch (UserException e) {
e.printStackTrace();
// Return message to info client what happened.
LOG.debug("DDL statement({}) process failed.", originStmt.originStmt, e);
context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
} catch (Exception e) {
e.printStackTrace();
// Maybe our bug
LOG.warn("DDL statement(" + originStmt.originStmt + ") process failed.", e);
context.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, "Unexpected exception: " + e.getMessage());
@ -1913,6 +1925,7 @@ public class StmtExecutor implements ProfileWriter {
context.getEnv().changeCluster(context, enterStmt.getClusterName());
context.setDatabase("");
} catch (DdlException e) {
e.printStackTrace();
context.getState().setError(e.getMysqlErrorCode(), e.getMessage());
return;
}
@ -1931,6 +1944,7 @@ public class StmtExecutor implements ProfileWriter {
DdlExecutor.execute(context.getEnv(), ctasStmt);
context.getState().setOk();
} catch (Exception e) {
e.printStackTrace();
// Maybe our bug
LOG.warn("CTAS create table error, stmt={}", originStmt.originStmt, e);
context.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, "Unexpected exception: " + e.getMessage());
@ -1941,6 +1955,7 @@ public class StmtExecutor implements ProfileWriter {
parsedStmt = ctasStmt.getInsertStmt();
execute();
} catch (Exception e) {
e.printStackTrace();
LOG.warn("CTAS insert data error, stmt={}", ctasStmt.toSql(), e);
// insert error drop table
DropTableStmt dropTableStmt = new DropTableStmt(true, ctasStmt.getCreateTableStmt().getDbTbl(), true);

41
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java
View File

@ -18,14 +18,19 @@
package org.apache.doris.rewrite.mvrewrite;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.CaseExpr;
import org.apache.doris.analysis.CaseWhenClause;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
@ -38,13 +43,19 @@ import com.google.common.collect.Lists;
* For example:
* Table: (k1 int ,k2 varchar)
* MV: (k1 int, mv_count_k2 bigint sum)
* mv_count_k1 = case when k2 is null then 0 else 1
* mv_count_k1 = case when k2 is null then 0 else 1
* Query: select k1, count(k2) from table group by k1
* Rewritten query: select k1, sum(mv_count_k2) from table group by k1
*/
public class CountFieldToSum implements ExprRewriteRule {
public static final ExprRewriteRule INSTANCE = new CountFieldToSum();
public static Expr slotToCaseWhen(Expr expr) throws AnalysisException {
return new CaseExpr(null, Lists.newArrayList(new CaseWhenClause(
new IsNullPredicate(expr, false),
new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT));
}
@Override
public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseType) throws AnalysisException {
// meet condition
@ -74,20 +85,30 @@ public class CountFieldToSum implements ExprRewriteRule {
String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder(FunctionSet.COUNT, queryColumnName);
Column mvColumn = olapTable.getVisibleColumn(mvColumnName);
if (mvColumn == null) {
return expr;
mvColumn = olapTable
.getVisibleColumn(CreateMaterializedViewStmt.mvAggregateColumnBuilder("SUM",
slotToCaseWhen(fnChild0).toSqlWithoutTbl()));
if (mvColumn != null) {
return rewriteExpr(mvColumn, analyzer);
}
} else {
return rewriteExpr(mvColumn, analyzer);
}
// rewrite expr
return rewriteExpr(mvColumn, analyzer);
return expr;
}
private Expr rewriteExpr(Column mvColumn, Analyzer analyzer) {
Preconditions.checkNotNull(mvColumn);
// Notice that we shouldn't set table name field of mvSlotRef here, for we will analyze the new mvSlotRef
// later, if the table name was set here, the Analyzer::registerColumnRef would invoke
// Analyzer::resolveColumnRef(TableName, String) which only try to find the column from the tupleByAlias,
// as at the most time the alias is not equal with the origin table name, so it would cause the unexpected
// exception to Unknown column, because we can't find an alias which named as origin table name that has
// Notice that we shouldn't set table name field of mvSlotRef here, for we will
// analyze the new mvSlotRef
// later, if the table name was set here, the Analyzer::registerColumnRef would
// invoke
// Analyzer::resolveColumnRef(TableName, String) which only try to find the
// column from the tupleByAlias,
// as at the most time the alias is not equal with the origin table name, so it
// would cause the unexpected
// exception to Unknown column, because we can't find an alias which named as
// origin table name that has
// required column.
SlotRef mvSlotRef = new SlotRef(null, mvColumn.getName());
FunctionCallExpr result = new FunctionCallExpr("sum", Lists.newArrayList(mvSlotRef));

179
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/ExprToSlotRefRule.java
View File

@ -24,15 +24,19 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
@ -47,16 +51,154 @@ public class ExprToSlotRefRule implements ExprRewriteRule {
@Override
public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseType) throws AnalysisException {
// todo: support more pattern
if (expr instanceof ArithmeticExpr || expr instanceof FunctionCallExpr) {
return matchExpr(expr, analyzer, clauseType);
if (expr instanceof FunctionCallExpr && ((FunctionCallExpr) expr).isAggregate()) {
return matchAggregateExpr((FunctionCallExpr) expr, analyzer);
} else if (expr instanceof ArithmeticExpr || expr instanceof FunctionCallExpr) {
return matchExpr(expr, analyzer);
} else if (expr instanceof SlotRef) {
return matchSlotRef((SlotRef) expr, analyzer, clauseType);
return matchSlotRef((SlotRef) expr, analyzer);
} else {
return expr;
}
}
private Expr matchExpr(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseType)
private Expr matchAggregateExprCount(FunctionCallExpr expr, Analyzer analyzer, OlapTable olapTable,
TableName tableName)
throws AnalysisException {
if (!expr.isDistinct()) {
return expr;
}
FunctionCallExpr toBitmap = new FunctionCallExpr(FunctionSet.TO_BITMAP, expr.getChildren());
toBitmap.setType(Type.BITMAP);
List<Expr> params = Lists.newArrayList();
params.add(toBitmap);
FunctionCallExpr newCount = new FunctionCallExpr(FunctionSet.BITMAP_UNION_COUNT, params);
Expr result = matchAggregateExprBitmap(newCount, analyzer, olapTable, tableName);
if (result != newCount) {
result.analyzeNoThrow(analyzer);
return result;
}
return expr;
}
private Expr matchAggregateExprCountApprox(FunctionCallExpr expr, Analyzer analyzer, OlapTable olapTable,
TableName tableName)
throws AnalysisException {
FunctionCallExpr hllHash = new FunctionCallExpr(FunctionSet.HLL_HASH, expr.getChildren());
hllHash.setType(Type.HLL);
List<Expr> params = Lists.newArrayList();
params.add(hllHash);
FunctionCallExpr newCount = new FunctionCallExpr(FunctionSet.HLL_UNION_AGG, params);
Expr result = matchAggregateExprHll(newCount, analyzer, olapTable, tableName);
if (result != newCount) {
result.analyzeNoThrow(analyzer);
return result;
}
return expr;
}
private Expr matchAggregateExprBitmap(Expr expr, Analyzer analyzer, OlapTable olapTable, TableName tableName)
throws AnalysisException {
Expr child0 = expr.getChild(0);
if (!child0.getType().isBitmapType()) {
return expr;
}
String mvColumnName = CreateMaterializedViewStmt
.mvColumnBuilder(AggregateType.BITMAP_UNION, child0.toSqlWithoutTbl());
Column mvColumn = olapTable.getVisibleColumn(mvColumnName);
if (mvColumn == null) {
mvColumn = olapTable
.getVisibleColumn(mvColumnName.replace(FunctionSet.TO_BITMAP, FunctionSet.TO_BITMAP_WITH_CHECK));
}
if (mvColumn != null) {
expr = expr.clone();
expr.setChild(0, rewriteExpr(tableName, mvColumn, analyzer));
}
return expr;
}
private Expr matchAggregateExprHll(Expr expr, Analyzer analyzer, OlapTable olapTable, TableName tableName)
throws AnalysisException {
Expr child0 = expr.getChild(0);
if (!child0.getType().isHllType()) {
return expr;
}
String mvColumnName = CreateMaterializedViewStmt
.mvColumnBuilder(AggregateType.HLL_UNION, child0.toSqlWithoutTbl());
Column mvColumn = olapTable.getVisibleColumn(mvColumnName);
if (mvColumn != null) {
expr = expr.clone();
expr.setChild(0, rewriteExpr(tableName, mvColumn, analyzer));
}
return expr;
}
private Expr matchAggregateExpr(FunctionCallExpr expr, Analyzer analyzer)
throws AnalysisException {
List<SlotRef> slots = new ArrayList<>();
expr.collect(SlotRef.class, slots);
if (expr.getChildren().size() != 1 || slots.size() != 1) {
return expr;
}
TableIf table = slots.get(0).getTable();
if (table == null || !(table instanceof OlapTable)) {
return expr;
}
OlapTable olapTable = (OlapTable) table;
String fnName = expr.getFnName().getFunction();
if (fnName.equalsIgnoreCase(FunctionSet.COUNT)) {
Expr result = matchAggregateExprCount(expr, analyzer, olapTable, slots.get(0).getTableName());
if (result != expr) {
return result;
}
}
if (fnName.equalsIgnoreCase(FunctionSet.APPROX_COUNT_DISTINCT) || fnName.equalsIgnoreCase(FunctionSet.NDV)) {
Expr result = matchAggregateExprCountApprox(expr, analyzer, olapTable, slots.get(0).getTableName());
if (result != expr) {
return result;
}
}
if (fnName.equalsIgnoreCase(FunctionSet.BITMAP_UNION)
|| fnName.equalsIgnoreCase(FunctionSet.BITMAP_UNION_COUNT)) {
return matchAggregateExprBitmap(expr, analyzer, olapTable, slots.get(0).getTableName());
}
if (fnName.equalsIgnoreCase(FunctionSet.HLL_UNION)
|| fnName.equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)
|| fnName.equalsIgnoreCase(FunctionSet.HLL_RAW_AGG)) {
return matchAggregateExprHll(expr, analyzer, olapTable, slots.get(0).getTableName());
}
Expr child0 = expr.getChild(0);
Column mvColumn = olapTable.getVisibleColumn(
CreateMaterializedViewStmt.mvAggregateColumnBuilder(expr.getFnName().getFunction(),
child0.toSqlWithoutTbl()));
if (mvColumn == null) {
mvColumn = olapTable.getVisibleColumn(
CreateMaterializedViewStmt.mvAggregateColumnBuilder(expr.getFnName().getFunction(),
CreateMaterializedViewStmt.mvColumnBuilder(child0.toSqlWithoutTbl())));
}
if (mvColumn != null) {
expr = (FunctionCallExpr) expr.clone();
expr.setChild(0, rewriteExpr(slots.get(0).getTableName(), mvColumn, analyzer));
}
return expr;
}
private Expr matchExpr(Expr expr, Analyzer analyzer)
throws AnalysisException {
List<SlotRef> slots = new ArrayList<>();
expr.collect(SlotRef.class, slots);
@ -67,56 +209,55 @@ public class ExprToSlotRefRule implements ExprRewriteRule {
SlotRef queryColumnSlotRef = slots.get(0);
Column column = queryColumnSlotRef.getColumn();
TableIf table = queryColumnSlotRef.getTable();
if (column == null || table == null || !(table instanceof OlapTable)) {
if (table == null || !(table instanceof OlapTable)) {
return expr;
}
OlapTable olapTable = (OlapTable) table;
Column mvColumn = olapTable.getVisibleColumn(expr.toSql());
Column mvColumn = olapTable.getVisibleColumn(expr.toSqlWithoutTbl());
if (mvColumn == null) {
mvColumn = olapTable.getVisibleColumn(
MaterializedIndexMeta.normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql())));
MaterializedIndexMeta
.normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(expr.toSqlWithoutTbl())));
}
if (mvColumn == null) {
mvColumn = olapTable.getVisibleColumn(CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql()));
mvColumn = olapTable.getVisibleColumn(CreateMaterializedViewStmt.mvColumnBuilder(expr.toSqlWithoutTbl()));
}
if (mvColumn == null) {
return expr;
}
return rewriteExpr(queryColumnSlotRef, mvColumn, analyzer);
return rewriteExpr(queryColumnSlotRef.getTableName(), mvColumn, analyzer);
}
private Expr matchSlotRef(SlotRef slot, Analyzer analyzer, ExprRewriter.ClauseType clauseType)
private Expr matchSlotRef(SlotRef slot, Analyzer analyzer)
throws AnalysisException {
Column column = slot.getColumn();
TableIf table = slot.getTable();
if (column == null || table == null || !(table instanceof OlapTable)) {
if (table == null || !(table instanceof OlapTable)) {
return slot;
}
OlapTable olapTable = (OlapTable) table;
Column mvColumn = olapTable.getVisibleColumn(CreateMaterializedViewStmt.mvColumnBuilder(slot.toSql()));
Column mvColumn = olapTable
.getVisibleColumn(CreateMaterializedViewStmt.mvColumnBuilder(slot.toSqlWithoutTbl()));
if (mvColumn == null) {
mvColumn = olapTable.getVisibleColumn(
MaterializedIndexMeta.normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(slot.toSql())));
MaterializedIndexMeta
.normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(slot.toSqlWithoutTbl())));
}
if (mvColumn == null) {
return slot;
}
return rewriteExpr(slot, mvColumn, analyzer);
return rewriteExpr(slot.getTableName(), mvColumn, analyzer);
}
private Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) {
private Expr rewriteExpr(TableName tableName, Column mvColumn, Analyzer analyzer) {
Preconditions.checkNotNull(mvColumn);
Preconditions.checkNotNull(queryColumnSlotRef);
TableName tableName = queryColumnSlotRef.getTableName();
Preconditions.checkNotNull(tableName);
SlotRef mvSlotRef = new SlotRef(tableName, mvColumn.getName());
mvSlotRef.analyzeNoThrow(analyzer);

48
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java
View File

@ -18,6 +18,9 @@
package org.apache.doris.rewrite.mvrewrite;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.common.AnalysisException;
import com.google.common.collect.ImmutableList;
@ -42,4 +45,49 @@ public class MVExprEquivalent {
}
return false;
}
public static boolean aggregateArgumentEqual(Expr expr, Expr mvArgument) {
if (mvArgument == null) {
return false;
}
if (!(expr instanceof FunctionCallExpr)) {
return false;
}
FunctionCallExpr fnExpr = (FunctionCallExpr) expr;
if (fnExpr.getChildren().size() != 1) {
return false;
}
String fnName = fnExpr.getFnName().getFunction();
if (fnName.equalsIgnoreCase(FunctionSet.COUNT)) {
return sumArgumentEqual(fnExpr.getChild(0), mvArgument);
}
if (fnName.equalsIgnoreCase(FunctionSet.BITMAP_UNION)
|| fnName.equalsIgnoreCase(FunctionSet.BITMAP_UNION_COUNT)) {
return bitmapArgumentEqual(fnExpr.getChild(0), mvArgument);
}
return false;
}
// count(k1) = sum(case when ... k1)
public static boolean sumArgumentEqual(Expr expr, Expr mvArgument) {
try {
String lhs = CountFieldToSum.slotToCaseWhen(expr).toSqlWithoutTbl();
String rhs = mvArgument.toSqlWithoutTbl();
return lhs.equalsIgnoreCase(rhs);
} catch (AnalysisException e) {
return false;
}
}
// to_bitmap(k1) = to_bitmap_with_check(k1)
public static boolean bitmapArgumentEqual(Expr expr, Expr mvArgument) {
String lhs = expr.toSqlWithoutTbl().replace(FunctionSet.TO_BITMAP_WITH_CHECK, FunctionSet.TO_BITMAP);
String rhs = mvArgument.toSqlWithoutTbl().replace(FunctionSet.TO_BITMAP_WITH_CHECK, FunctionSet.TO_BITMAP);
return lhs.equalsIgnoreCase(rhs);
}
}

11
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/SlotRefEqualRule.java
View File

@ -17,13 +17,20 @@
package org.apache.doris.rewrite.mvrewrite;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.catalog.MaterializedIndexMeta;
public class SlotRefEqualRule implements MVExprEqualRule {
public static MVExprEqualRule INSTANCE = new SlotRefEqualRule();
private String normalize(String name) {
return MaterializedIndexMeta
.normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(name));
}
@Override
public boolean equal(Expr queryExpr, Expr mvColumnExpr) {
if ((!(queryExpr instanceof SlotRef)) || (!(mvColumnExpr instanceof SlotRef))) {
@ -31,8 +38,10 @@ public class SlotRefEqualRule implements MVExprEqualRule {
}
SlotRef querySlotRef = (SlotRef) queryExpr;
SlotRef mvColumnSlotRef = (SlotRef) mvColumnExpr;
if (querySlotRef.getColumnName() != null
&& querySlotRef.getColumnName().equalsIgnoreCase(mvColumnSlotRef.getColumnName())) {
&& normalize(querySlotRef.getColumnName()).equalsIgnoreCase(
normalize(mvColumnSlotRef.getColumnName()))) {
return true;
}
return false;

106
fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
@ -361,10 +362,6 @@ public class CreateMaterializedViewStmtTest {
result = null;
selectStmt.getOrderByElements();
result = orderByElementList;
slotDescriptor.getColumn();
result = column2;
column2.getOriginType();
result = Type.INT;
slotRef1.toSql();
result = "k1";
}
@ -438,10 +435,6 @@ public class CreateMaterializedViewStmtTest {
selectStmt.analyze(analyzer);
selectStmt.getSelectList();
result = selectList;
slotDescriptor.getColumn();
result = column2;
column2.getOriginType();
result = Type.INT;
slotRef1.toSql();
result = "k1";
}
@ -494,10 +487,6 @@ public class CreateMaterializedViewStmtTest {
result = null;
selectStmt.getOrderByElements();
result = orderByElementList;
slotDescriptor.getColumn();
result = column3;
column3.getOriginType();
result = Type.INT;
slotRef1.toSql();
result = "k1";
slotRef2.toSql();
@ -533,9 +522,8 @@ public class CreateMaterializedViewStmtTest {
selectList.addItem(selectListItem3);
SelectListItem selectListItem4 = new SelectListItem(slotRef4, null);
selectList.addItem(selectListItem4);
TableName tableName = new TableName(internalCtl, "db", "table");
final String columnName5 = "sum_v2";
SlotRef functionChild0 = new SlotRef(tableName, columnName5);
final String columnName5 = "v2";
SlotRef functionChild0 = new SlotRef(null, columnName5);
Deencapsulation.setField(functionChild0, "desc", slotDescriptor);
List<Expr> fn1Children = Lists.newArrayList(functionChild0);
FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", fn1Children);
@ -574,10 +562,6 @@ public class CreateMaterializedViewStmtTest {
result = columnName3;
slotRef4.toSql();
result = columnName4;
functionChild0.getColumn();
result = column5;
column5.getOriginType();
result = Type.INT;
}
};
@ -611,7 +595,8 @@ public class CreateMaterializedViewStmtTest {
MVColumnItem mvColumn4 = mvColumns.get(4);
Assert.assertFalse(mvColumn4.isKey());
Assert.assertFalse(mvColumn4.isAggregationTypeImplicit());
Assert.assertEquals(CreateMaterializedViewStmt.mvColumnBuilder(columnName5), mvColumn4.getName());
Assert.assertEquals(CreateMaterializedViewStmt.mvColumnBuilder(AggregateType.SUM, columnName5),
MaterializedIndexMeta.normalizeName(mvColumn4.getName()));
Assert.assertEquals(AggregateType.SUM, mvColumn4.getAggregationType());
} catch (UserException e) {
Assert.fail(e.getMessage());
@ -1015,9 +1000,8 @@ public class CreateMaterializedViewStmtTest {
selectList.addItem(selectListItem1);
SelectListItem selectListItem2 = new SelectListItem(slotRef2, null);
selectList.addItem(selectListItem2);
TableName tableName = new TableName(internalCtl, "db", "table");
final String columnName3 = "sum_v2";
SlotRef slotRef = new SlotRef(tableName, columnName3);
final String columnName3 = "v2";
SlotRef slotRef = new SlotRef(null, columnName3);
Deencapsulation.setField(slotRef, "desc", slotDescriptor);
List<Expr> children = Lists.newArrayList(slotRef);
FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", children);
@ -1057,10 +1041,6 @@ public class CreateMaterializedViewStmtTest {
result = columnName1;
slotRef2.getColumnName();
result = columnName2;
slotDescriptor.getColumn();
result = column1;
column1.getOriginType();
result = Type.INT;
}
};
@ -1083,7 +1063,8 @@ public class CreateMaterializedViewStmtTest {
MVColumnItem mvColumn2 = mvColumns.get(2);
Assert.assertFalse(mvColumn2.isKey());
Assert.assertFalse(mvColumn2.isAggregationTypeImplicit());
Assert.assertEquals(CreateMaterializedViewStmt.mvColumnBuilder(columnName3), mvColumn2.getName());
Assert.assertEquals(CreateMaterializedViewStmt.mvColumnBuilder(AggregateType.SUM, columnName3),
MaterializedIndexMeta.normalizeName(mvColumn2.getName()));
Assert.assertEquals(AggregateType.SUM, mvColumn2.getAggregationType());
Assert.assertEquals(KeysType.AGG_KEYS, createMaterializedViewStmt.getMVKeysType());
} catch (UserException e) {
@ -1148,69 +1129,44 @@ public class CreateMaterializedViewStmtTest {
@Injectable SlotDescriptor slotDescriptor4) {
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
SlotRef slotRef = new SlotRef(new TableName(internalCtl, "db", "table"), "a");
slotRef.setType(Type.LARGEINT);
List<Expr> params = Lists.newArrayList();
params.add(slotRef);
FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", params);
Deencapsulation.setField(slotRef, "desc", slotDescriptor1);
new Expectations() {
{
slotDescriptor1.getColumn();
result = column1;
column1.getOriginType();
result = Type.LARGEINT;
}
};
MVColumnItem mvColumnItem = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", analyzer,
functionCallExpr);
Assert.assertEquals(Type.LARGEINT, mvColumnItem.getType());
SlotRef slotRef2 = new SlotRef(new TableName(internalCtl, "db", "table"), "a");
slotRef2.setType(Type.BIGINT);
List<Expr> params2 = Lists.newArrayList();
params2.add(slotRef2);
FunctionCallExpr functionCallExpr2 = new FunctionCallExpr("sum", params2);
Deencapsulation.setField(slotRef2, "desc", slotDescriptor2);
new Expectations() {
{
slotDescriptor2.getColumn();
result = column2;
column2.getOriginType();
result = Type.BIGINT;
}
};
MVColumnItem mvColumnItem2 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", analyzer,
functionCallExpr2);
Assert.assertEquals(Type.BIGINT, mvColumnItem2.getType());
SlotRef slotRef3 = new SlotRef(new TableName(internalCtl, "db", "table"), "a");
slotRef3.setType(Type.VARCHAR);
List<Expr> params3 = Lists.newArrayList();
params3.add(slotRef3);
FunctionCallExpr functionCallExpr3 = new FunctionCallExpr("min", params3);
Deencapsulation.setField(slotRef3, "desc", slotDescriptor3);
new Expectations() {
{
slotDescriptor3.getColumn();
result = column3;
column3.getOriginType();
result = Type.VARCHAR;
}
};
MVColumnItem mvColumnItem3 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", analyzer,
functionCallExpr3);
Assert.assertEquals(Type.VARCHAR, mvColumnItem3.getType());
SlotRef slotRef4 = new SlotRef(new TableName(internalCtl, "db", "table"), "a");
slotRef4.setType(Type.DOUBLE);
List<Expr> params4 = Lists.newArrayList();
params4.add(slotRef4);
FunctionCallExpr functionCallExpr4 = new FunctionCallExpr("sum", params4);
Deencapsulation.setField(slotRef4, "desc", slotDescriptor4);
new Expectations() {
{
slotDescriptor4.getColumn();
result = column4;
column4.getOriginType();
result = Type.DOUBLE;
}
};
MVColumnItem mvColumnItem4 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", analyzer,
functionCallExpr4);
Assert.assertEquals(Type.DOUBLE, mvColumnItem4.getType());
@ -1227,53 +1183,35 @@ public class CreateMaterializedViewStmtTest {
@Injectable Column column3) {
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
SlotRef slotRef = new SlotRef(new TableName(internalCtl, "db", "table"), "a");
slotRef.setType(ScalarType.createVarchar(50));
List<Expr> params = Lists.newArrayList();
params.add(slotRef);
FunctionCallExpr functionCallExpr = new FunctionCallExpr("min", params);
Deencapsulation.setField(slotRef, "desc", slotDescriptor1);
new Expectations() {
{
slotDescriptor1.getColumn();
result = column1;
column1.getOriginType();
result = ScalarType.createVarchar(50);
}
};
MVColumnItem mvColumnItem = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", analyzer,
functionCallExpr);
Assert.assertEquals(50, mvColumnItem.getType().getLength());
SlotRef slotRef2 = new SlotRef(new TableName(internalCtl, "db", "table"), "a");
slotRef2.setType(ScalarType.createDecimalType(10, 1));
List<Expr> params2 = Lists.newArrayList();
params2.add(slotRef2);
FunctionCallExpr functionCallExpr2 = new FunctionCallExpr("min", params2);
Deencapsulation.setField(slotRef2, "desc", slotDescriptor2);
new Expectations() {
{
slotDescriptor2.getColumn();
result = column2;
column2.getOriginType();
result = ScalarType.createDecimalType(10, 1);
}
};
MVColumnItem mvColumnItem2 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", analyzer,
functionCallExpr2);
Assert.assertEquals(new Integer(10), mvColumnItem2.getType().getPrecision());
Assert.assertEquals(1, ((ScalarType) mvColumnItem2.getType()).getScalarScale());
SlotRef slotRef3 = new SlotRef(new TableName(internalCtl, "db", "table"), "a");
slotRef3.setType(ScalarType.createChar(5));
List<Expr> params3 = Lists.newArrayList();
params3.add(slotRef3);
FunctionCallExpr functionCallExpr3 = new FunctionCallExpr("min", params3);
Deencapsulation.setField(slotRef3, "desc", slotDescriptor3);
new Expectations() {
{
slotDescriptor3.getColumn();
result = column3;
column3.getOriginType();
result = ScalarType.createChar(5);
}
};
MVColumnItem mvColumnItem3 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", analyzer,
functionCallExpr3);
Assert.assertEquals(5, mvColumnItem3.getType().getLength());

11
fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java
View File

@ -24,7 +24,6 @@ import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.InternalCatalog;
import com.google.common.collect.Lists;
import mockit.Expectations;
import mockit.Injectable;
import org.junit.Assert;
import org.junit.Test;
@ -53,12 +52,6 @@ public class MVColumnOneChildPatternTest {
SlotRef slotRef = new SlotRef(tableName, "c1");
List<Expr> child0Params = Lists.newArrayList();
child0Params.add(slotRef);
new Expectations() {
{
castExpr.unwrapSlotRef();
result = slotRef;
}
};
List<Expr> params = Lists.newArrayList();
params.add(castExpr);
FunctionCallExpr functionCallExpr = new FunctionCallExpr(AggregateType.MIN.name(), params);
@ -89,7 +82,7 @@ public class MVColumnOneChildPatternTest {
Deencapsulation.setField(functionCallExpr, "fn", aggregateFunction);
MVColumnOneChildPattern mvColumnOneChildPattern = new MVColumnOneChildPattern(
AggregateType.SUM.name().toLowerCase());
Assert.assertFalse(mvColumnOneChildPattern.match(functionCallExpr));
Assert.assertTrue(mvColumnOneChildPattern.match(functionCallExpr));
}
@Test
@ -104,6 +97,6 @@ public class MVColumnOneChildPatternTest {
Deencapsulation.setField(functionCallExpr, "fn", aggregateFunction);
MVColumnOneChildPattern mvColumnOneChildPattern = new MVColumnOneChildPattern(
AggregateType.SUM.name().toLowerCase());
Assert.assertFalse(mvColumnOneChildPattern.match(functionCallExpr));
Assert.assertTrue(mvColumnOneChildPattern.match(functionCallExpr));
}
}

1
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java
View File

@ -52,6 +52,7 @@ import java.util.stream.Collectors;
/**
* Tests ported from {@link org.apache.doris.planner.MaterializedViewFunctionTest}
*/
@Disabled("Disabled until nereids support advanced mv")
public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements PatternMatchSupported {
private static final String EMPS_TABLE_NAME = "emps";

25
fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.planner;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.common.FeConstants;
import org.apache.doris.utframe.DorisAssert;
@ -284,8 +283,13 @@ public class MaterializedViewFunctionTest {
+ "from " + EMPS_TABLE_NAME + " group by deptno, commission;";
String query = "select deptno, commission, sum(salary) + 1 from " + EMPS_TABLE_NAME
+ " group by rollup (deptno, commission);";
dorisAssert.withMaterializedView(createMVSql);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
try {
dorisAssert.withMaterializedView(createMVSql);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
/**
@ -549,7 +553,7 @@ public class MaterializedViewFunctionTest {
@Test
public void testMultiMVMultiUsage() throws Exception {
String createEmpsMVSql01 = "create materialized view emp_mv_01 as select deptno, empid, salary "
+ "from " + EMPS_TABLE_NAME + ";";
+ "from " + EMPS_TABLE_NAME + " order by deptno;";
String createEmpsMVSql02 = "create materialized view emp_mv_02 as select deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by deptno;";
String query = "select * from (select deptno, empid from " + EMPS_TABLE_NAME + " where deptno>100) A join "
@ -789,12 +793,11 @@ public class MaterializedViewFunctionTest {
+ "`" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
dorisAssert.withMaterializedView(createUserTagMVSql);
String query = "select `" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + ";";
String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder("" + FunctionSet.HLL_UNION + "", "tag_id");
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName);
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME);
query = "select hll_union_agg(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + ";";
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName);
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME);
query = "select hll_raw_agg(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + ";";
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName);
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME);
}
/*
@ -815,8 +818,7 @@ public class MaterializedViewFunctionTest {
+ "count(tag_id) from " + USER_TAG_TABLE_NAME + " group by user_id;";
dorisAssert.withMaterializedView(createUserTagMVSql);
String query = "select count(tag_id) from " + USER_TAG_TABLE_NAME + ";";
String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder(FunctionSet.COUNT, "tag_id");
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName);
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME);
query = "select user_name, count(tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;";
dorisAssert.query(query).explainWithout(USER_TAG_MV_NAME);
}
@ -857,7 +859,6 @@ public class MaterializedViewFunctionTest {
+ "count(tag_id) from " + USER_TAG_TABLE_NAME + " group by user_id;";
dorisAssert.withMaterializedView(createUserTagMVSql);
String query = "select count(tag_id) from " + USER_TAG_TABLE_NAME + " t ;";
String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder(FunctionSet.COUNT, "tag_id");
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName);
dorisAssert.query(query).explainContains(USER_TAG_MV_NAME);
}
}

4
fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java
View File

@ -201,7 +201,7 @@ public class MaterializedViewSelectorTest {
result = index3Columns;
indexMeta4.getSchema();
result = index4Columns;
slotRef1.toSql();
slotRef1.toSqlWithoutTbl();
result = "c1";
}
};
@ -267,7 +267,7 @@ public class MaterializedViewSelectorTest {
result = index2Columns;
indexMeta3.getSchema();
result = index3Columns;
slotRef1.toSql();
slotRef1.toSqlWithoutTbl();
result = "c1";
}
};