diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index ab8d7e3e45..ad00ff06b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -2200,16 +2200,16 @@ public abstract class Expr extends TreeNode implements ParseNode, Cloneabl } } - public boolean haveMvSlot() { + public boolean haveMvSlot(TupleId tid) { for (Expr expr : getChildren()) { - if (expr.haveMvSlot()) { + if (expr.haveMvSlot(tid)) { return true; } } return false; } - public boolean matchExprs(List exprs, SelectStmt stmt, boolean ignoreAlias, String tableName) + public boolean matchExprs(List exprs, SelectStmt stmt, boolean ignoreAlias, TupleDescriptor tuple) throws AnalysisException { List slots = new ArrayList<>(); collect(SlotRef.class, slots); @@ -2235,7 +2235,7 @@ public abstract class Expr extends TreeNode implements ParseNode, Cloneabl } for (Expr expr : getChildren()) { - if (!expr.matchExprs(exprs, stmt, ignoreAlias, tableName)) { + if (!expr.matchExprs(exprs, stmt, ignoreAlias, tuple)) { return false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 1b39cf094f..311faa5370 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -365,7 +365,7 @@ public abstract class LiteralExpr extends Expr implements Comparable exprs, SelectStmt stmt, boolean ignoreAlias, String tableName) { + public boolean matchExprs(List exprs, SelectStmt stmt, boolean ignoreAlias, TupleDescriptor tuple) { return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index 967785bdfd..aea5ba90c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -107,6 +107,7 @@ public class SlotRef extends Expr { col = other.col; label = other.label; desc = other.desc; + tupleId = other.tupleId; } @Override @@ -369,6 +370,10 @@ public class SlotRef extends Expr { this.tupleId = tupleId; } + TupleId getTupleId() { + return tupleId; + } + @Override public boolean isBoundByTupleIds(List tids) { Preconditions.checkState(desc != null || tupleId != null); @@ -544,13 +549,16 @@ public class SlotRef extends Expr { } @Override - public boolean haveMvSlot() { + public boolean haveMvSlot(TupleId tid) { + if (!isBound(tid)) { + return false; + } String name = MaterializedIndexMeta.normalizeName(toSqlWithoutTbl()); return CreateMaterializedViewStmt.isMVColumn(name); } @Override - public boolean matchExprs(List exprs, SelectStmt stmt, boolean ignoreAlias, String tableName) + public boolean matchExprs(List exprs, SelectStmt stmt, boolean ignoreAlias, TupleDescriptor tuple) throws AnalysisException { Expr originExpr = stmt.getExprFromAliasSMap(this); if (!(originExpr instanceof SlotRef)) { @@ -561,7 +569,7 @@ public class SlotRef extends Expr { if (aliasExpr.getColumnName() == null) { if (desc.getSourceExprs() != null) { for (Expr expr : desc.getSourceExprs()) { - if (!expr.matchExprs(exprs, stmt, ignoreAlias, tableName)) { + if (!expr.matchExprs(exprs, stmt, ignoreAlias, tuple)) { return false; } } @@ -569,9 +577,9 @@ public class SlotRef extends Expr { return true; // means this is alias of other expr. } + String name = MaterializedIndexMeta.normalizeName(aliasExpr.toSqlWithoutTbl()); if (aliasExpr.desc != null) { - TableIf table = aliasExpr.desc.getParent().getTable(); - if (table != null && table.getName() != tableName) { + if (!isBound(tuple.getId()) && !tuple.getColumnNames().contains(name)) { return true; // means this from other scan node. } @@ -580,7 +588,6 @@ public class SlotRef extends Expr { } } - String name = MaterializedIndexMeta.normalizeName(aliasExpr.toSqlWithoutTbl()); for (Expr expr : exprs) { if (CreateMaterializedViewStmt.isMVColumnNormal(name) && MaterializedIndexMeta.normalizeName(expr.toSqlWithoutTbl()).equals(CreateMaterializedViewStmt diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java index 11375e5a4b..4a2d60c9b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java @@ -29,6 +29,8 @@ import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -378,6 +380,16 @@ public class TupleDescriptor { } } + public Set getColumnNames() { + Map> columnNamesInQueryOutput = Maps.newHashMap(); + getTableIdToColumnNames(columnNamesInQueryOutput); + Set columnNames = Sets.newHashSet(); + for (Set names : columnNamesInQueryOutput.values()) { + columnNames.addAll(names); + } + return columnNames; + } + @Override public String toString() { String tblStr = (table == null ? "null" : table.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java index 139abb0ba4..48153bab5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java @@ -36,6 +36,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rewrite.mvrewrite.MVExprEquivalent; +import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -145,13 +146,13 @@ public class MaterializedViewSelector { // Step1: all of predicates is compensating predicates Map candidateIndexIdToMeta = scanNode.getOlapTable().getVisibleIndexIdToMeta(); OlapTable table = scanNode.getOlapTable(); - String tableName = table.getName(); Preconditions.checkState(table != null); long tableId = table.getId(); boolean selectBaseIndex = false; for (Expr expr : selectStmt.getAllExprs()) { - if (selectStmt.isDisableTuplesMVRewriter(selectStmt.getExprFromAliasSMap(expr))) { + if (expr.isBound(scanNode.getTupleId()) + && selectStmt.isDisableTuplesMVRewriter(selectStmt.getExprFromAliasSMap(expr))) { selectBaseIndex = true; } } @@ -159,15 +160,17 @@ public class MaterializedViewSelector { // Step2: check all columns in compensating predicates are available in the view // output checkCompensatingPredicates(columnNamesInPredicates.get(tableId), candidateIndexIdToMeta, selectBaseIndex, - tableName); + scanNode.getTupleId()); // Step3: group by list in query is the subset of group by list in view or view // contains no aggregation - checkGrouping(table, columnNamesInGrouping.get(tableId), candidateIndexIdToMeta, selectBaseIndex); + checkGrouping(table, columnNamesInGrouping.get(tableId), candidateIndexIdToMeta, selectBaseIndex, + scanNode.getTupleId()); // Step4: aggregation functions are available in the view output - checkAggregationFunction(table, aggColumnsInQuery.get(tableId), candidateIndexIdToMeta); + checkAggregationFunction(table, aggColumnsInQuery.get(tableId), candidateIndexIdToMeta, scanNode.getTupleId()); // Step5: columns required to compute output expr are available in the view // output - checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta, selectBaseIndex, tableName); + checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta, selectBaseIndex, + scanNode.getTupleId()); // Step6: if table type is aggregate and the candidateIndexIdToSchema is empty, if ((table.getKeysType() == KeysType.AGG_KEYS || (table.getKeysType() == KeysType.UNIQUE_KEYS && !table.getTableProperty().getEnableUniqueKeyMergeOnWrite())) @@ -191,7 +194,7 @@ public class MaterializedViewSelector { compensateCandidateIndex(candidateIndexIdToMeta, scanNode.getOlapTable().getVisibleIndexIdToMeta(), table); checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta, selectBaseIndex, - tableName); + scanNode.getTupleId()); } Map> result = Maps.newHashMap(); for (Map.Entry entry : candidateIndexIdToMeta.entrySet()) { @@ -299,7 +302,7 @@ public class MaterializedViewSelector { // Step2: check all columns in compensating predicates are available in the view // output private void checkCompensatingPredicates(Set columnsInPredicates, - Map candidateIndexIdToMeta, boolean selectBaseIndex, String tableName) + Map candidateIndexIdToMeta, boolean selectBaseIndex, TupleId tid) throws AnalysisException { Iterator> iterator = candidateIndexIdToMeta.entrySet().iterator(); while (iterator.hasNext()) { @@ -352,7 +355,7 @@ public class MaterializedViewSelector { continue; } - if (!matchAllExpr(predicateExprs, indexExprs, tableName)) { + if (!matchAllExpr(predicateExprs, indexExprs, tid)) { iterator.remove(); } } @@ -376,7 +379,7 @@ public class MaterializedViewSelector { // Step3: group by list in query is the subset of group by list in view or view // contains no aggregation private void checkGrouping(OlapTable table, Set columnsInGrouping, - Map candidateIndexIdToMeta, boolean selectBaseIndex) + Map candidateIndexIdToMeta, boolean selectBaseIndex, TupleId tid) throws AnalysisException { Iterator> iterator = candidateIndexIdToMeta.entrySet().iterator(); while (iterator.hasNext()) { @@ -445,7 +448,7 @@ public class MaterializedViewSelector { continue; } - if (!matchAllExpr(groupingExprs, indexExprs, table.getName())) { + if (!matchAllExpr(groupingExprs, indexExprs, tid)) { iterator.remove(); } } @@ -455,11 +458,11 @@ public class MaterializedViewSelector { // Step4: aggregation functions are available in the view output private void checkAggregationFunction(OlapTable table, Set aggregatedColumnsInQueryOutput, - Map candidateIndexIdToMeta) throws AnalysisException { + Map candidateIndexIdToMeta, TupleId tid) throws AnalysisException { boolean haveMvSlot = false; if (aggregatedColumnsInQueryOutput != null) { for (FunctionCallExpr expr : aggregatedColumnsInQueryOutput) { - if (expr.haveMvSlot()) { + if (expr.haveMvSlot(tid)) { haveMvSlot = true; } } @@ -488,7 +491,7 @@ public class MaterializedViewSelector { continue; } if (aggregatedColumnsInQueryOutput != null - && matchAllExpr(new ArrayList<>(aggregatedColumnsInQueryOutput), indexExprs, table.getName())) { + && matchAllExpr(new ArrayList<>(aggregatedColumnsInQueryOutput), indexExprs, tid)) { continue; } @@ -512,12 +515,8 @@ public class MaterializedViewSelector { + Joiner.on(",").join(candidateIndexIdToMeta.keySet())); } - private boolean matchAllExpr(List exprs, List indexExprs, String tableName) + private boolean matchAllExpr(List exprs, List indexExprs, TupleId tid) throws AnalysisException { - if (exprs.isEmpty()) { - return false; - } - for (Expr expr : exprs) { if (expr == null) { throw new AnalysisException("match expr input null"); @@ -530,7 +529,7 @@ public class MaterializedViewSelector { continue; } - if (expr.matchExprs(indexExprs, selectStmt, false, tableName)) { + if (expr.matchExprs(indexExprs, selectStmt, false, analyzer.getTupleDesc(tid))) { continue; } return false; @@ -541,7 +540,8 @@ public class MaterializedViewSelector { // Step5: columns required to compute output expr are available in the view // output private void checkOutputColumns(Set columnNamesInQueryOutput, - Map candidateIndexIdToMeta, boolean selectBaseIndex, String tableName) + Map candidateIndexIdToMeta, boolean selectBaseIndex, + TupleId tid) throws AnalysisException { if (columnNamesInQueryOutput == null) { return; @@ -554,6 +554,14 @@ public class MaterializedViewSelector { columnNamesInQueryOutput .forEach(name -> queryColumnNames.add(CreateMaterializedViewStmt.mvColumnBreaker(name))); + if (selectBaseIndex) { + for (Expr expr : exprs) { + if (expr.haveMvSlot(tid)) { + throw new MVSelectFailedException("need selectBaseIndex but have mv expr"); + } + } + } + Iterator> iterator = candidateIndexIdToMeta.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); @@ -583,7 +591,7 @@ public class MaterializedViewSelector { continue; } - if (!matchAllExpr(exprs, indexExprs, tableName)) { + if (!matchAllExpr(exprs, indexExprs, tid)) { iterator.remove(); } } diff --git a/regression-test/data/mv_p0/ut/testQueryOnStar/testQueryOnStar.out b/regression-test/data/mv_p0/ut/testQueryOnStar/testQueryOnStar.out index 7782346729..3440fe7517 100644 --- a/regression-test/data/mv_p0/ut/testQueryOnStar/testQueryOnStar.out +++ b/regression-test/data/mv_p0/ut/testQueryOnStar/testQueryOnStar.out @@ -8,3 +8,7 @@ 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +-- !select_mv -- +1 +1 + diff --git a/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy b/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy index 90e98a4dc3..70eb71a1a5 100644 --- a/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy @@ -59,8 +59,8 @@ suite ("multi_slot_k123p") { explain { sql("select lhs.k1,rhs.k2 from d_table as lhs right join d_table as rhs on lhs.k1=rhs.k1;") - notContains "(k123p)" - notContains "`mv_" + contains "(k123p)" + contains "(d_table)" } qt_select_mv "select lhs.k1,rhs.k2 from d_table as lhs right join d_table as rhs on lhs.k1=rhs.k1 order by lhs.k1;" diff --git a/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy b/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy index 8db594a70a..57ddba4d3c 100644 --- a/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy +++ b/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy @@ -49,4 +49,25 @@ suite ("testQueryOnStar") { contains "(emps_mv)" } qt_select_mv "select * from emps where deptno = 1 order by empid;" + + sql """ DROP TABLE IF EXISTS tpch_tiny_region; """ + sql """ + CREATE TABLE IF NOT EXISTS tpch_tiny_region ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) + ) + DUPLICATE KEY(r_regionkey) + DISTRIBUTED BY HASH(r_regionkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + sql """insert into tpch_tiny_region values(1,'a','a');""" + + explain { + sql("select ref_1.`empid` as c0 from tpch_tiny_region as ref_0 left join emps as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;") + contains "(emps_mv)" + } + qt_select_mv "select ref_1.`empid` as c0 from tpch_tiny_region as ref_0 left join emps as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;" }