From c8ad62a3cda378c40ff4163f08de0c470ea73c32 Mon Sep 17 00:00:00 2001 From: Pxl Date: Thu, 30 Mar 2023 13:02:21 +0800 Subject: [PATCH] [Enchancement](materialized-view) enchance materialized view where clause match (#18179) enchance materialized view where clause match --- .../doris/analysis/CompoundPredicate.java | 27 +++++++++++++++++++ .../java/org/apache/doris/analysis/Expr.java | 14 ++++++++++ .../planner/MaterializedViewSelector.java | 4 +-- .../apache/doris/planner/OlapScanNode.java | 11 +++++--- .../doris/planner/SingleNodePlanner.java | 7 ++--- .../org/apache/doris/planner/PlannerTest.java | 2 +- .../apache/doris/planner/QueryPlanTest.java | 16 +++++------ .../org/apache/doris/policy/PolicyTest.java | 11 ++++---- .../data/mv_p0/where/k123/k123.out | 4 +++ .../suites/mv_p0/where/k123/k123.groovy | 6 +++++ 10 files changed, 76 insertions(+), 26 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java index d711114db4..b98b982eb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java @@ -277,4 +277,31 @@ public class CompoundPredicate extends Predicate { public String toString() { return toSqlImpl(); } + + @Override + public boolean containsSubPredicate(Expr subExpr) throws AnalysisException { + if (op.equals(Operator.AND)) { + for (Expr child : children) { + if (child.containsSubPredicate(subExpr)) { + return true; + } + } + } + return super.containsSubPredicate(subExpr); + } + + @Override + public Expr replaceSubPredicate(Expr subExpr) throws AnalysisException { + if (op.equals(Operator.AND)) { + Expr lhs = children.get(0); + Expr rhs = children.get(1); + if (lhs.replaceSubPredicate(subExpr) == null) { + return rhs; + } + if (rhs.replaceSubPredicate(subExpr) == null) { + return lhs; + } + } + return this; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index c39616fc6d..a2d03c50ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -2226,6 +2226,20 @@ public abstract class Expr extends TreeNode implements ParseNode, Cloneabl return true; } + public boolean containsSubPredicate(Expr subExpr) throws AnalysisException { + if (toSqlWithoutTbl().equals(subExpr.toSqlWithoutTbl())) { + return true; + } + return false; + } + + public Expr replaceSubPredicate(Expr subExpr) throws AnalysisException { + if (toSqlWithoutTbl().equals(subExpr.toSqlWithoutTbl())) { + return null; + } + return this; + } + protected Type[] getActualArgTypes(Type[] originType) { return Arrays.stream(originType).map( (Type type) -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java index 0996ff30b2..139abb0ba4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java @@ -325,8 +325,8 @@ public class MaterializedViewSelector { } if (entry.getValue().getWhereClause() != null) { - if (selectStmt.getOriginalWhereClause() == null || !entry.getValue().getWhereClause().toSqlWithoutTbl() - .equals(selectStmt.getOriginalWhereClause().toSqlWithoutTbl())) { + if (selectStmt.getOriginalWhereClause() == null || !selectStmt.getOriginalWhereClause() + .containsSubPredicate(entry.getValue().getWhereClause())) { iterator.remove(); } continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index f07adf882c..cfc42fd8ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -333,8 +333,11 @@ public class OlapScanNode extends ScanNode { return selectedIndexId; } - public void ignoreConjuncts() { - vconjunct = null; + public void ignoreConjuncts(Expr whereExpr) throws AnalysisException { + if (whereExpr == null) { + return; + } + vconjunct = vconjunct.replaceSubPredicate(whereExpr); } /** @@ -1103,8 +1106,8 @@ public class OlapScanNode extends ScanNode { if (useTopnOpt) { output.append(prefix).append("TOPN OPT\n"); } - if (!conjuncts.isEmpty()) { - output.append(prefix).append("PREDICATES: ").append(getExplainString(conjuncts)).append("\n"); + if (vconjunct != null) { + output.append(prefix).append("PREDICATES: ").append(vconjunct.toSql()).append("\n"); } if (!runtimeFilters.isEmpty()) { output.append(prefix).append("runtime filters: "); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 1883cc9094..9b7e716b9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -1363,12 +1363,9 @@ public class SingleNodePlanner { } else { try { // mv index have where clause, so where expr on scan node is unused. - Expr whereExpr = olapScanNode.getOlapTable() + olapScanNode.ignoreConjuncts(olapScanNode.getOlapTable() .getIndexMetaByIndexId(bestIndexInfo.getBestIndexId()) - .getWhereClause(); - if (whereExpr != null) { - olapScanNode.ignoreConjuncts(); - } + .getWhereClause()); // if the new selected index id is different from the old one, scan node will be // updated. diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java index 730aab5674..97812cb0d0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java @@ -455,7 +455,7 @@ public class PlannerTest extends TestWithFeService { stmtExecutor.execute(); Planner planner = stmtExecutor.planner(); String plan = planner.getExplainString(new ExplainOptions(false, false)); - Assertions.assertTrue(plan.contains("PREDICATES: `k1` = 1, `k2` = 1\n")); + Assertions.assertTrue(plan.contains("PREDICATES: `k1` = 1 AND `k2` = 1\n")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 206456e188..a057325d10 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -1654,7 +1654,7 @@ public class QueryPlanTest extends TestWithFeService { //default format String sql = "select * from test1 where from_unixtime(query_time) > '2021-03-02 10:01:28'"; String explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `query_time` <= 253402271999, `query_time` > 1614650488")); + Assert.assertTrue(explainString.contains("PREDICATES: `query_time` <= 253402271999 AND `query_time` > 1614650488")); } @Test @@ -1829,7 +1829,7 @@ public class QueryPlanTest extends TestWithFeService { // (false or expr1) and (false or expr2) ==> expr1 and expr2 String sql9 = "select * from test.test1 where (-2=2 or query_time=2) and (-2=2 or stmt_id=2);"; String explainString9 = getSQLPlanOrErrorMsg("EXPLAIN " + sql9); - Assert.assertTrue(explainString9.contains("PREDICATES: `query_time` = 2, `stmt_id` = 2")); + Assert.assertTrue(explainString9.contains("PREDICATES: `query_time` = 2 AND `stmt_id` = 2")); // false or (expr and true) ==> expr String sql10 = "select * from test.test1 where (2=-2) OR (query_time=0 AND 1=1);"; @@ -1865,7 +1865,7 @@ public class QueryPlanTest extends TestWithFeService { Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07'," + " `date` <= '2021-10-11'")); } else { - Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07 00:00:00'," + Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07 00:00:00' AND" + " `date` <= '2021-10-11 00:00:00'")); } } @@ -2210,15 +2210,15 @@ public class QueryPlanTest extends TestWithFeService { sql = "SELECT * from test1 where (query_time = 1 or query_time = 2) and query_time in (3, 4)"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2), `query_time` IN (3, 4)\n")); + Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) AND `query_time` IN (3, 4)\n")); sql = "SELECT * from test1 where (query_time = 1 or query_time = 2 or scan_bytes = 2) and scan_bytes in (2, 3)"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) OR `scan_bytes` = 2, `scan_bytes` IN (2, 3)\n")); + Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) OR `scan_bytes` = 2 AND `scan_bytes` IN (2, 3)\n")); sql = "SELECT * from test1 where (query_time = 1 or query_time = 2) and (scan_bytes = 2 or scan_bytes = 3)"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2), `scan_bytes` IN (2, 3)\n")); + Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) AND `scan_bytes` IN (2, 3)\n")); sql = "SELECT * from test1 where query_time = 1 or query_time = 2 or query_time = 3 or query_time = 1"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); @@ -2236,7 +2236,7 @@ public class QueryPlanTest extends TestWithFeService { sql = "SELECT * from test1 where (query_time = 1 or query_time = 2) and query_time in (3, 4)"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); - Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2), `query_time` IN (3, 4)\n")); + Assert.assertTrue(explainString.contains("PREDICATES: `query_time` IN (1, 2) AND `query_time` IN (3, 4)\n")); //test we can handle `!=` and `not in` sql = "select * from test1 where (query_time = 1 or query_time = 2 or query_time!= 3 or query_time not in (5, 6))"; @@ -2269,7 +2269,7 @@ public class QueryPlanTest extends TestWithFeService { sql = "select * from test1 where (stmt_id=1 and state='a') or (stmt_id=2 and state='b')"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "EXPLAIN " + sql); Assert.assertTrue(explainString.contains( - "PREDICATES: `state` IN ('a', 'b'), `stmt_id` IN (1, 2)," + "PREDICATES: `state` IN ('a', 'b') AND `stmt_id` IN (1, 2) AND" + " `stmt_id` = 1 AND `state` = 'a' OR `stmt_id` = 2 AND `state` = 'b'\n" )); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java index 71084fb429..b53bbe46ef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java @@ -172,7 +172,7 @@ public class PolicyTest extends TestWithFeService { createPolicy("CREATE ROW POLICY test_row_policy4 ON test.table1 AS PERMISSIVE TO test_policy USING (k2 = 1)"); String queryStr = "EXPLAIN select * from test.table1"; String explainString = getSQLPlanOrErrorMsg(queryStr); - Assertions.assertTrue(explainString.contains("`k1` = 1, `k2` = 1, `k2` = 2 OR `k2` = 1")); + Assertions.assertTrue(explainString.contains("`k1` = 1 AND `k2` = 1 AND `k2` = 2 OR `k2` = 1")); dropPolicy("DROP ROW POLICY test_row_policy1 ON test.table1"); dropPolicy("DROP ROW POLICY test_row_policy2 ON test.table1"); dropPolicy("DROP ROW POLICY test_row_policy3 ON test.table1"); @@ -184,14 +184,13 @@ public class PolicyTest extends TestWithFeService { createPolicy("CREATE ROW POLICY test_row_policy1 ON test.table1 AS RESTRICTIVE TO test_policy USING (k1 = 1)"); createPolicy("CREATE ROW POLICY test_row_policy2 ON test.table1 AS RESTRICTIVE TO test_policy USING (k2 = 1)"); String joinSql = "select * from table1 join table2 on table1.k1=table2.k1"; - System.out.println(getSQLPlanOrErrorMsg(joinSql)); - Assertions.assertTrue(getSQLPlanOrErrorMsg(joinSql).contains("PREDICATES: `k1` = 1, `k2` = 1")); + Assertions.assertTrue(getSQLPlanOrErrorMsg(joinSql).contains("PREDICATES: `k1` = 1 AND `k2` = 1")); String unionSql = "select * from table1 union select * from table2"; - Assertions.assertTrue(getSQLPlanOrErrorMsg(unionSql).contains("PREDICATES: `k1` = 1, `k2` = 1")); + Assertions.assertTrue(getSQLPlanOrErrorMsg(unionSql).contains("PREDICATES: `k1` = 1 AND `k2` = 1")); String subQuerySql = "select * from table2 where k1 in (select k1 from table1)"; - Assertions.assertTrue(getSQLPlanOrErrorMsg(subQuerySql).contains("PREDICATES: `k1` = 1, `k2` = 1")); + Assertions.assertTrue(getSQLPlanOrErrorMsg(subQuerySql).contains("PREDICATES: `k1` = 1 AND `k2` = 1")); String aliasSql = "select * from table1 t1 join table2 t2 on t1.k1=t2.k1"; - Assertions.assertTrue(getSQLPlanOrErrorMsg(aliasSql).contains("PREDICATES: `t1`.`k1` = 1, `t1`.`k2` = 1")); + Assertions.assertTrue(getSQLPlanOrErrorMsg(aliasSql).contains("PREDICATES: `t1`.`k1` = 1 AND `t1`.`k2` = 1")); dropPolicy("DROP ROW POLICY test_row_policy1 ON test.table1"); dropPolicy("DROP ROW POLICY test_row_policy2 ON test.table1"); } diff --git a/regression-test/data/mv_p0/where/k123/k123.out b/regression-test/data/mv_p0/where/k123/k123.out index fb9659afad..55bd6c538c 100644 --- a/regression-test/data/mv_p0/where/k123/k123.out +++ b/regression-test/data/mv_p0/where/k123/k123.out @@ -35,3 +35,7 @@ 1 2 1 2 +-- !select_mv -- +2 4 +2 4 + diff --git a/regression-test/suites/mv_p0/where/k123/k123.groovy b/regression-test/suites/mv_p0/where/k123/k123.groovy index d80c59b1e2..09201fe31c 100644 --- a/regression-test/suites/mv_p0/where/k123/k123.groovy +++ b/regression-test/suites/mv_p0/where/k123/k123.groovy @@ -80,4 +80,10 @@ suite ("k123p") { contains "(d_table)" } qt_select_mv "select k1,k2+k3 from d_table where k4 = 'a' order by k1;" + + explain { + sql("""select k1,k2+k3 from d_table where k1 = 2 and k4 = "b";""") + contains "(k123p4w)" + } + qt_select_mv """select k1,k2+k3 from d_table where k1 = 2 and k4 = "b" order by k1;""" }