database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enchancement](materialized-view) enchance materialized view where clause match (#18179)

Browse Source 
enchance materialized view where clause match
This commit is contained in:
Pxl
2023-03-30 13:02:21 +08:00
committed by GitHub
parent c8ea5bff1d
commit c8ad62a3cd
10 changed files with 76 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
View File

@ -277,4 +277,31 @@ public class CompoundPredicate extends Predicate {
public String toString() {
return toSqlImpl();
}
@Override
public boolean containsSubPredicate(Expr subExpr) throws AnalysisException {
if (op.equals(Operator.AND)) {
for (Expr child : children) {
if (child.containsSubPredicate(subExpr)) {
return true;
}
}
}
return super.containsSubPredicate(subExpr);
}
@Override
public Expr replaceSubPredicate(Expr subExpr) throws AnalysisException {
if (op.equals(Operator.AND)) {
Expr lhs = children.get(0);
Expr rhs = children.get(1);
if (lhs.replaceSubPredicate(subExpr) == null) {
return rhs;
}
if (rhs.replaceSubPredicate(subExpr) == null) {
return lhs;
}
}
return this;
}
}

14
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -2226,6 +2226,20 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return true;
}
public boolean containsSubPredicate(Expr subExpr) throws AnalysisException {
if (toSqlWithoutTbl().equals(subExpr.toSqlWithoutTbl())) {
return true;
}
return false;
}
public Expr replaceSubPredicate(Expr subExpr) throws AnalysisException {
if (toSqlWithoutTbl().equals(subExpr.toSqlWithoutTbl())) {
return null;
}
return this;
}
protected Type[] getActualArgTypes(Type[] originType) {
return Arrays.stream(originType).map(
(Type type) -> {

4
fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java
View File

@ -325,8 +325,8 @@ public class MaterializedViewSelector {
}
if (entry.getValue().getWhereClause() != null) {
if (selectStmt.getOriginalWhereClause() == null || !entry.getValue().getWhereClause().toSqlWithoutTbl()
.equals(selectStmt.getOriginalWhereClause().toSqlWithoutTbl())) {
if (selectStmt.getOriginalWhereClause() == null || !selectStmt.getOriginalWhereClause()
.containsSubPredicate(entry.getValue().getWhereClause())) {
iterator.remove();
}
continue;

11
fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -333,8 +333,11 @@ public class OlapScanNode extends ScanNode {
return selectedIndexId;
}
public void ignoreConjuncts() {
vconjunct = null;
public void ignoreConjuncts(Expr whereExpr) throws AnalysisException {
if (whereExpr == null) {
return;
}
vconjunct = vconjunct.replaceSubPredicate(whereExpr);
}
/**
@ -1103,8 +1106,8 @@ public class OlapScanNode extends ScanNode {
if (useTopnOpt) {
output.append(prefix).append("TOPN OPT\n");
}
if (!conjuncts.isEmpty()) {
output.append(prefix).append("PREDICATES: ").append(getExplainString(conjuncts)).append("\n");
if (vconjunct != null) {
output.append(prefix).append("PREDICATES: ").append(vconjunct.toSql()).append("\n");
}
if (!runtimeFilters.isEmpty()) {
output.append(prefix).append("runtime filters: ");

7
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -1363,12 +1363,9 @@ public class SingleNodePlanner {
} else {
try {
// mv index have where clause, so where expr on scan node is unused.
Expr whereExpr = olapScanNode.getOlapTable()
olapScanNode.ignoreConjuncts(olapScanNode.getOlapTable()
.getIndexMetaByIndexId(bestIndexInfo.getBestIndexId())
.getWhereClause();
if (whereExpr != null) {
olapScanNode.ignoreConjuncts();
}
.getWhereClause());
// if the new selected index id is different from the old one, scan node will be
// updated.

2
fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
View File

@ -455,7 +455,7 @@ public class PlannerTest extends TestWithFeService {
stmtExecutor.execute();
Planner planner = stmtExecutor.planner();
String plan = planner.getExplainString(new ExplainOptions(false, false));
Assertions.assertTrue(plan.contains("PREDICATES: `k1` = 1, `k2` = 1\n"));
Assertions.assertTrue(plan.contains("PREDICATES: `k1` = 1 AND `k2` = 1\n"));
}
@Test

16
fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View File

@ -1654,7 +1654,7 @@ public class QueryPlanTest extends TestWithFeService {
//default format
String sql = "select * from test1 where from_unixtime(query_time) > '2021-03-02 10:01:28'";
String explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` <= 253402271999, `query_time` > 1614650488"));
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` <= 253402271999 AND `query_time` > 1614650488"));
}
@Test
@ -1829,7 +1829,7 @@ public class QueryPlanTest extends TestWithFeService {
// (false or expr1) and (false or expr2) ==> expr1 and expr2
String sql9 = "select * from test.test1 where (-2=2 or query_time=2) and (-2=2 or stmt_id=2);";
String explainString9 = getSQLPlanOrErrorMsg("EXPLAIN " + sql9);
Assert.assertTrue(explainString9.contains("PREDICATES: `query_time` = 2, `stmt_id` = 2"));
Assert.assertTrue(explainString9.contains("PREDICATES: `query_time` = 2 AND `stmt_id` = 2"));
// false or (expr and true) ==> expr
String sql10 = "select * from test.test1 where (2=-2) OR (query_time=0 AND 1=1);";
@ -1865,7 +1865,7 @@ public class QueryPlanTest extends TestWithFeService {
Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07',"
+ " `date` <= '2021-10-11'"));
} else {
Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07 00:00:00',"
Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07 00:00:00' AND"
+ " `date` <= '2021-10-11 00:00:00'"));
}
}
@ -2210,15 +2210,15 @@ public class QueryPlanTest extends TestWithFeService {
sql = "SELECT * from test1 where (query_time = 1 or query_time = 2) and query_time in (3, 4)";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql);
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2), `query_time` IN (3, 4)\n"));
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) AND `query_time` IN (3, 4)\n"));
sql = "SELECT * from test1 where (query_time = 1 or query_time = 2 or scan_bytes = 2) and scan_bytes in (2, 3)";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql);
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) OR `scan_bytes` = 2, `scan_bytes` IN (2, 3)\n"));
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) OR `scan_bytes` = 2 AND `scan_bytes` IN (2, 3)\n"));
sql = "SELECT * from test1 where (query_time = 1 or query_time = 2) and (scan_bytes = 2 or scan_bytes = 3)";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql);
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2), `scan_bytes` IN (2, 3)\n"));
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) AND `scan_bytes` IN (2, 3)\n"));
sql = "SELECT * from test1 where query_time = 1 or query_time = 2 or query_time = 3 or query_time = 1";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql);
@ -2236,7 +2236,7 @@ public class QueryPlanTest extends TestWithFeService {
sql = "SELECT * from test1 where (query_time = 1 or query_time = 2) and query_time in (3, 4)";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql);
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2), `query_time` IN (3, 4)\n"));
Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) AND `query_time` IN (3, 4)\n"));
//test we can handle `!=` and `not in`
sql = "select * from test1 where (query_time = 1 or query_time = 2 or query_time!= 3 or query_time not in (5, 6))";
@ -2269,7 +2269,7 @@ public class QueryPlanTest extends TestWithFeService {
sql = "select * from test1 where (stmt_id=1 and state='a') or (stmt_id=2 and state='b')";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql);
Assert.assertTrue(explainString.contains(
"PREDICATES: `state` IN ('a', 'b'), `stmt_id` IN (1, 2),"
"PREDICATES: `state` IN ('a', 'b') AND `stmt_id` IN (1, 2) AND"
+ " `stmt_id` = 1 AND `state` = 'a' OR `stmt_id` = 2 AND `state` = 'b'\n"
));
}

11
fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java
View File

@ -172,7 +172,7 @@ public class PolicyTest extends TestWithFeService {
createPolicy("CREATE ROW POLICY test_row_policy4 ON test.table1 AS PERMISSIVE TO test_policy USING (k2 = 1)");
String queryStr = "EXPLAIN select * from test.table1";
String explainString = getSQLPlanOrErrorMsg(queryStr);
Assertions.assertTrue(explainString.contains("`k1` = 1, `k2` = 1, `k2` = 2 OR `k2` = 1"));
Assertions.assertTrue(explainString.contains("`k1` = 1 AND `k2` = 1 AND `k2` = 2 OR `k2` = 1"));
dropPolicy("DROP ROW POLICY test_row_policy1 ON test.table1");
dropPolicy("DROP ROW POLICY test_row_policy2 ON test.table1");
dropPolicy("DROP ROW POLICY test_row_policy3 ON test.table1");
@ -184,14 +184,13 @@ public class PolicyTest extends TestWithFeService {
createPolicy("CREATE ROW POLICY test_row_policy1 ON test.table1 AS RESTRICTIVE TO test_policy USING (k1 = 1)");
createPolicy("CREATE ROW POLICY test_row_policy2 ON test.table1 AS RESTRICTIVE TO test_policy USING (k2 = 1)");
String joinSql = "select * from table1 join table2 on table1.k1=table2.k1";
System.out.println(getSQLPlanOrErrorMsg(joinSql));
Assertions.assertTrue(getSQLPlanOrErrorMsg(joinSql).contains("PREDICATES: `k1` = 1, `k2` = 1"));
Assertions.assertTrue(getSQLPlanOrErrorMsg(joinSql).contains("PREDICATES: `k1` = 1 AND `k2` = 1"));
String unionSql = "select * from table1 union select * from table2";
Assertions.assertTrue(getSQLPlanOrErrorMsg(unionSql).contains("PREDICATES: `k1` = 1, `k2` = 1"));
Assertions.assertTrue(getSQLPlanOrErrorMsg(unionSql).contains("PREDICATES: `k1` = 1 AND `k2` = 1"));
String subQuerySql = "select * from table2 where k1 in (select k1 from table1)";
Assertions.assertTrue(getSQLPlanOrErrorMsg(subQuerySql).contains("PREDICATES: `k1` = 1, `k2` = 1"));
Assertions.assertTrue(getSQLPlanOrErrorMsg(subQuerySql).contains("PREDICATES: `k1` = 1 AND `k2` = 1"));
String aliasSql = "select * from table1 t1 join table2 t2 on t1.k1=t2.k1";
Assertions.assertTrue(getSQLPlanOrErrorMsg(aliasSql).contains("PREDICATES: `t1`.`k1` = 1, `t1`.`k2` = 1"));
Assertions.assertTrue(getSQLPlanOrErrorMsg(aliasSql).contains("PREDICATES: `t1`.`k1` = 1 AND `t1`.`k2` = 1"));
dropPolicy("DROP ROW POLICY test_row_policy1 ON test.table1");
dropPolicy("DROP ROW POLICY test_row_policy2 ON test.table1");
}

4
regression-test/data/mv_p0/where/k123/k123.out
View File

@ -35,3 +35,7 @@
1 2
1 2
-- !select_mv --
2 4
2 4

6
regression-test/suites/mv_p0/where/k123/k123.groovy
View File

@ -80,4 +80,10 @@ suite ("k123p") {
contains "(d_table)"
}
qt_select_mv "select k1,k2+k3 from d_table where k4 = 'a' order by k1;"
explain {
sql("""select k1,k2+k3 from d_table where k1 = 2 and k4 = "b";""")
contains "(k123p4w)"
}
qt_select_mv """select k1,k2+k3 from d_table where k1 = 2 and k4 = "b" order by k1;"""
}