database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](materialized-view) fix match wrong index on some scan node (#18561)

Browse Source 
fix match wrong index on some scan node
This commit is contained in:
Pxl
2023-04-13 11:50:14 +08:00
committed by GitHub
parent 726402b53b
commit eb46bcb304
8 changed files with 87 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -2200,16 +2200,16 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
}
public boolean haveMvSlot() {
public boolean haveMvSlot(TupleId tid) {
for (Expr expr : getChildren()) {
if (expr.haveMvSlot()) {
if (expr.haveMvSlot(tid)) {
return true;
}
}
return false;
}
public boolean matchExprs(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias, String tableName)
public boolean matchExprs(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias, TupleDescriptor tuple)
throws AnalysisException {
List<SlotRef> slots = new ArrayList<>();
collect(SlotRef.class, slots);
@ -2235,7 +2235,7 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
for (Expr expr : getChildren()) {
if (!expr.matchExprs(exprs, stmt, ignoreAlias, tableName)) {
if (!expr.matchExprs(exprs, stmt, ignoreAlias, tuple)) {
return false;
}
}

2
fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
View File

@ -365,7 +365,7 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
}
@Override
public boolean matchExprs(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias, String tableName) {
public boolean matchExprs(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias, TupleDescriptor tuple) {
return true;
}
}

19
fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
View File

@ -107,6 +107,7 @@ public class SlotRef extends Expr {
col = other.col;
label = other.label;
desc = other.desc;
tupleId = other.tupleId;
}
@Override
@ -369,6 +370,10 @@ public class SlotRef extends Expr {
this.tupleId = tupleId;
}
TupleId getTupleId() {
return tupleId;
}
@Override
public boolean isBoundByTupleIds(List<TupleId> tids) {
Preconditions.checkState(desc != null || tupleId != null);
@ -544,13 +549,16 @@ public class SlotRef extends Expr {
}
@Override
public boolean haveMvSlot() {
public boolean haveMvSlot(TupleId tid) {
if (!isBound(tid)) {
return false;
}
String name = MaterializedIndexMeta.normalizeName(toSqlWithoutTbl());
return CreateMaterializedViewStmt.isMVColumn(name);
}
@Override
public boolean matchExprs(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias, String tableName)
public boolean matchExprs(List<Expr> exprs, SelectStmt stmt, boolean ignoreAlias, TupleDescriptor tuple)
throws AnalysisException {
Expr originExpr = stmt.getExprFromAliasSMap(this);
if (!(originExpr instanceof SlotRef)) {
@ -561,7 +569,7 @@ public class SlotRef extends Expr {
if (aliasExpr.getColumnName() == null) {
if (desc.getSourceExprs() != null) {
for (Expr expr : desc.getSourceExprs()) {
if (!expr.matchExprs(exprs, stmt, ignoreAlias, tableName)) {
if (!expr.matchExprs(exprs, stmt, ignoreAlias, tuple)) {
return false;
}
}
@ -569,9 +577,9 @@ public class SlotRef extends Expr {
return true; // means this is alias of other expr.
}
String name = MaterializedIndexMeta.normalizeName(aliasExpr.toSqlWithoutTbl());
if (aliasExpr.desc != null) {
TableIf table = aliasExpr.desc.getParent().getTable();
if (table != null && table.getName() != tableName) {
if (!isBound(tuple.getId()) && !tuple.getColumnNames().contains(name)) {
return true; // means this from other scan node.
}
@ -580,7 +588,6 @@ public class SlotRef extends Expr {
}
}
String name = MaterializedIndexMeta.normalizeName(aliasExpr.toSqlWithoutTbl());
for (Expr expr : exprs) {
if (CreateMaterializedViewStmt.isMVColumnNormal(name)
&& MaterializedIndexMeta.normalizeName(expr.toSqlWithoutTbl()).equals(CreateMaterializedViewStmt

12
fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java
View File

@ -29,6 +29,8 @@ import com.google.common.base.Joiner;
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -378,6 +380,16 @@ public class TupleDescriptor {
}
}
public Set<String> getColumnNames() {
Map<Long, Set<String>> columnNamesInQueryOutput = Maps.newHashMap();
getTableIdToColumnNames(columnNamesInQueryOutput);
Set<String> columnNames = Sets.newHashSet();
for (Set<String> names : columnNamesInQueryOutput.values()) {
columnNames.addAll(names);
}
return columnNames;
}
@Override
public String toString() {
String tblStr = (table == null ? "null" : table.getName());

52
fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
View File

@ -36,6 +36,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.mvrewrite.MVExprEquivalent;
import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@ -145,13 +146,13 @@ public class MaterializedViewSelector {
// Step1: all of predicates is compensating predicates
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta = scanNode.getOlapTable().getVisibleIndexIdToMeta();
OlapTable table = scanNode.getOlapTable();
String tableName = table.getName();
Preconditions.checkState(table != null);
long tableId = table.getId();
boolean selectBaseIndex = false;
for (Expr expr : selectStmt.getAllExprs()) {
if (selectStmt.isDisableTuplesMVRewriter(selectStmt.getExprFromAliasSMap(expr))) {
if (expr.isBound(scanNode.getTupleId())
&& selectStmt.isDisableTuplesMVRewriter(selectStmt.getExprFromAliasSMap(expr))) {
selectBaseIndex = true;
}
}
@ -159,15 +160,17 @@ public class MaterializedViewSelector {
// Step2: check all columns in compensating predicates are available in the view
// output
checkCompensatingPredicates(columnNamesInPredicates.get(tableId), candidateIndexIdToMeta, selectBaseIndex,
tableName);
scanNode.getTupleId());
// Step3: group by list in query is the subset of group by list in view or view
// contains no aggregation
checkGrouping(table, columnNamesInGrouping.get(tableId), candidateIndexIdToMeta, selectBaseIndex);
checkGrouping(table, columnNamesInGrouping.get(tableId), candidateIndexIdToMeta, selectBaseIndex,
scanNode.getTupleId());
// Step4: aggregation functions are available in the view output
checkAggregationFunction(table, aggColumnsInQuery.get(tableId), candidateIndexIdToMeta);
checkAggregationFunction(table, aggColumnsInQuery.get(tableId), candidateIndexIdToMeta, scanNode.getTupleId());
// Step5: columns required to compute output expr are available in the view
// output
checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta, selectBaseIndex, tableName);
checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta, selectBaseIndex,
scanNode.getTupleId());
// Step6: if table type is aggregate and the candidateIndexIdToSchema is empty,
if ((table.getKeysType() == KeysType.AGG_KEYS || (table.getKeysType() == KeysType.UNIQUE_KEYS
&& !table.getTableProperty().getEnableUniqueKeyMergeOnWrite()))
@ -191,7 +194,7 @@ public class MaterializedViewSelector {
compensateCandidateIndex(candidateIndexIdToMeta, scanNode.getOlapTable().getVisibleIndexIdToMeta(),
table);
checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta, selectBaseIndex,
tableName);
scanNode.getTupleId());
}
Map<Long, List<Column>> result = Maps.newHashMap();
for (Map.Entry<Long, MaterializedIndexMeta> entry : candidateIndexIdToMeta.entrySet()) {
@ -299,7 +302,7 @@ public class MaterializedViewSelector {
// Step2: check all columns in compensating predicates are available in the view
// output
private void checkCompensatingPredicates(Set<String> columnsInPredicates,
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta, boolean selectBaseIndex, String tableName)
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta, boolean selectBaseIndex, TupleId tid)
throws AnalysisException {
Iterator<Map.Entry<Long, MaterializedIndexMeta>> iterator = candidateIndexIdToMeta.entrySet().iterator();
while (iterator.hasNext()) {
@ -352,7 +355,7 @@ public class MaterializedViewSelector {
continue;
}
if (!matchAllExpr(predicateExprs, indexExprs, tableName)) {
if (!matchAllExpr(predicateExprs, indexExprs, tid)) {
iterator.remove();
}
}
@ -376,7 +379,7 @@ public class MaterializedViewSelector {
// Step3: group by list in query is the subset of group by list in view or view
// contains no aggregation
private void checkGrouping(OlapTable table, Set<String> columnsInGrouping,
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta, boolean selectBaseIndex)
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta, boolean selectBaseIndex, TupleId tid)
throws AnalysisException {
Iterator<Map.Entry<Long, MaterializedIndexMeta>> iterator = candidateIndexIdToMeta.entrySet().iterator();
while (iterator.hasNext()) {
@ -445,7 +448,7 @@ public class MaterializedViewSelector {
continue;
}
if (!matchAllExpr(groupingExprs, indexExprs, table.getName())) {
if (!matchAllExpr(groupingExprs, indexExprs, tid)) {
iterator.remove();
}
}
@ -455,11 +458,11 @@ public class MaterializedViewSelector {
// Step4: aggregation functions are available in the view output
private void checkAggregationFunction(OlapTable table, Set<FunctionCallExpr> aggregatedColumnsInQueryOutput,
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta) throws AnalysisException {
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta, TupleId tid) throws AnalysisException {
boolean haveMvSlot = false;
if (aggregatedColumnsInQueryOutput != null) {
for (FunctionCallExpr expr : aggregatedColumnsInQueryOutput) {
if (expr.haveMvSlot()) {
if (expr.haveMvSlot(tid)) {
haveMvSlot = true;
}
}
@ -488,7 +491,7 @@ public class MaterializedViewSelector {
continue;
}
if (aggregatedColumnsInQueryOutput != null
&& matchAllExpr(new ArrayList<>(aggregatedColumnsInQueryOutput), indexExprs, table.getName())) {
&& matchAllExpr(new ArrayList<>(aggregatedColumnsInQueryOutput), indexExprs, tid)) {
continue;
}
@ -512,12 +515,8 @@ public class MaterializedViewSelector {
+ Joiner.on(",").join(candidateIndexIdToMeta.keySet()));
}
private boolean matchAllExpr(List<Expr> exprs, List<Expr> indexExprs, String tableName)
private boolean matchAllExpr(List<Expr> exprs, List<Expr> indexExprs, TupleId tid)
throws AnalysisException {
if (exprs.isEmpty()) {
return false;
}
for (Expr expr : exprs) {
if (expr == null) {
throw new AnalysisException("match expr input null");
@ -530,7 +529,7 @@ public class MaterializedViewSelector {
continue;
}
if (expr.matchExprs(indexExprs, selectStmt, false, tableName)) {
if (expr.matchExprs(indexExprs, selectStmt, false, analyzer.getTupleDesc(tid))) {
continue;
}
return false;
@ -541,7 +540,8 @@ public class MaterializedViewSelector {
// Step5: columns required to compute output expr are available in the view
// output
private void checkOutputColumns(Set<String> columnNamesInQueryOutput,
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta, boolean selectBaseIndex, String tableName)
Map<Long, MaterializedIndexMeta> candidateIndexIdToMeta, boolean selectBaseIndex,
TupleId tid)
throws AnalysisException {
if (columnNamesInQueryOutput == null) {
return;
@ -554,6 +554,14 @@ public class MaterializedViewSelector {
columnNamesInQueryOutput
.forEach(name -> queryColumnNames.add(CreateMaterializedViewStmt.mvColumnBreaker(name)));
if (selectBaseIndex) {
for (Expr expr : exprs) {
if (expr.haveMvSlot(tid)) {
throw new MVSelectFailedException("need selectBaseIndex but have mv expr");
}
}
}
Iterator<Map.Entry<Long, MaterializedIndexMeta>> iterator = candidateIndexIdToMeta.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<Long, MaterializedIndexMeta> entry = iterator.next();
@ -583,7 +591,7 @@ public class MaterializedViewSelector {
continue;
}
if (!matchAllExpr(exprs, indexExprs, tableName)) {
if (!matchAllExpr(exprs, indexExprs, tid)) {
iterator.remove();
}
}

4
regression-test/data/mv_p0/ut/testQueryOnStar/testQueryOnStar.out
View File

@ -8,3 +8,7 @@
2020-01-01 1 a 1 1 1
2020-01-01 1 a 1 1 1
-- !select_mv --
1
1

4
regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy
View File

@ -59,8 +59,8 @@ suite ("multi_slot_k123p") {
explain {
sql("select lhs.k1,rhs.k2 from d_table as lhs right join d_table as rhs on lhs.k1=rhs.k1;")
notContains "(k123p)"
notContains "`mv_"
contains "(k123p)"
contains "(d_table)"
}
qt_select_mv "select lhs.k1,rhs.k2 from d_table as lhs right join d_table as rhs on lhs.k1=rhs.k1 order by lhs.k1;"

21
regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy
View File

@ -49,4 +49,25 @@ suite ("testQueryOnStar") {
contains "(emps_mv)"
}
qt_select_mv "select * from emps where deptno = 1 order by empid;"
sql """ DROP TABLE IF EXISTS tpch_tiny_region; """
sql """
CREATE TABLE IF NOT EXISTS tpch_tiny_region (
r_regionkey INTEGER NOT NULL,
r_name CHAR(25) NOT NULL,
r_comment VARCHAR(152)
)
DUPLICATE KEY(r_regionkey)
DISTRIBUTED BY HASH(r_regionkey) BUCKETS 3
PROPERTIES (
"replication_num" = "1"
)
"""
sql """insert into tpch_tiny_region values(1,'a','a');"""
explain {
sql("select ref_1.`empid` as c0 from tpch_tiny_region as ref_0 left join emps as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;")
contains "(emps_mv)"
}
qt_select_mv "select ref_1.`empid` as c0 from tpch_tiny_region as ref_0 left join emps as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;"
}