From 2d19f2fbfe628077eab7fc8ce78f2c53abf3b7b2 Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:34:53 +0800 Subject: [PATCH] [fix](planner)need call materializeSrcExpr for materialized slots in join node (#25204) --- .../apache/doris/analysis/SlotDescriptor.java | 1 + .../apache/doris/planner/JoinNodeBase.java | 7 +- .../apache/doris/planner/QueryPlanTest.java | 22 +++--- .../test_inlineview_with_project.out | 3 + .../test_inlineview_with_project.groovy | 73 +++++++++++++++++++ 5 files changed, 88 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 919e285034..ede85e74d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -92,6 +92,7 @@ public class SlotDescriptor { this.isAgg = false; this.stats = src.stats; this.type = src.type; + this.sourceExprs.add(new SlotRef(src)); } public boolean isMultiRef() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java index 11e9c31e8a..a460ea0f75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java @@ -168,9 +168,6 @@ public abstract class JoinNodeBase extends PlanNode { boolean needSetToNullable = getChild(0) instanceof JoinNodeBase && analyzer.isOuterJoined(leftTupleDesc.getId()); for (SlotDescriptor leftSlotDesc : leftTupleDesc.getSlots()) { - if (!isMaterializedByChild(leftSlotDesc, getChild(0).getOutputSmap())) { - continue; - } SlotDescriptor outputSlotDesc = analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, leftSlotDesc); if (leftNullable) { @@ -191,9 +188,6 @@ public abstract class JoinNodeBase extends PlanNode { boolean needSetToNullable = getChild(1) instanceof JoinNodeBase && analyzer.isOuterJoined(rightTupleDesc.getId()); for (SlotDescriptor rightSlotDesc : rightTupleDesc.getSlots()) { - if (!isMaterializedByChild(rightSlotDesc, getChild(1).getOutputSmap())) { - continue; - } SlotDescriptor outputSlotDesc = analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, rightSlotDesc); if (rightNullable) { @@ -226,6 +220,7 @@ public abstract class JoinNodeBase extends PlanNode { rSlotRef.getDesc().setIsMaterialized(lSlotRef.getDesc().isMaterialized()); } else { rSlotRef.getDesc().setIsMaterialized(true); + rSlotRef.materializeSrcExpr(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 55304b5a3d..7bfa8a9227 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -731,7 +731,7 @@ public class QueryPlanTest extends TestWithFeService { + "left join join2 on join1.id = join2.id\n" + "and join1.id > 1;"; String explainString = getSQLPlanOrErrorMsg("explain " + sql); - Assert.assertTrue(explainString.contains("other join predicates: > 1")); + Assert.assertTrue(explainString.contains("other join predicates: > 1")); Assert.assertFalse(explainString.contains("PREDICATES: `join1`.`id` > 1")); /* @@ -818,7 +818,7 @@ public class QueryPlanTest extends TestWithFeService { + "left anti join join2 on join1.id = join2.id\n" + "and join1.id > 1;"; explainString = getSQLPlanOrErrorMsg("explain " + sql); - Assert.assertTrue(explainString.contains("other join predicates: > 1")); + Assert.assertTrue(explainString.contains("other join predicates: > 1")); Assert.assertFalse(explainString.contains("PREDICATES: `join1`.`id` > 1")); // test semi join, left table join predicate, only push to left table @@ -1541,7 +1541,6 @@ public class QueryPlanTest extends TestWithFeService { public void testEmptyNode() throws Exception { connectContext.setDatabase("default_cluster:test"); String emptyNode = "EMPTYSET"; - String denseRank = "dense_rank"; List sqls = Lists.newArrayList(); sqls.add("explain select * from baseall limit 0"); @@ -1560,7 +1559,6 @@ public class QueryPlanTest extends TestWithFeService { for (String sql : sqls) { String explainString = getSQLPlanOrErrorMsg(sql); Assert.assertTrue(explainString.contains(emptyNode)); - Assert.assertFalse(explainString.contains(denseRank)); } } @@ -1651,25 +1649,25 @@ public class QueryPlanTest extends TestWithFeService { connectContext.setDatabase("default_cluster:test"); //valid date String sql = "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ a.aid, b.bid FROM (SELECT 3 AS aid) a right outer JOIN (SELECT 4 AS bid) b ON (a.aid=b.bid)"; - assertSQLPlanOrErrorMsgContains(sql, "OUTPUT EXPRS:\n" + " \n" + " "); + assertSQLPlanOrErrorMsgContains(sql, "OUTPUT EXPRS:\n" + " 3\n" + " 4"); sql = "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ a.aid, b.bid FROM (SELECT 3 AS aid) a left outer JOIN (SELECT 4 AS bid) b ON (a.aid=b.bid)"; - assertSQLPlanOrErrorMsgContains(sql, "OUTPUT EXPRS:\n" + " \n" + " "); + assertSQLPlanOrErrorMsgContains(sql, "OUTPUT EXPRS:\n" + " 3\n" + " 4"); sql = "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ a.aid, b.bid FROM (SELECT 3 AS aid) a full outer JOIN (SELECT 4 AS bid) b ON (a.aid=b.bid)"; - assertSQLPlanOrErrorMsgContains(sql, "OUTPUT EXPRS:\n" + " \n" + " "); + assertSQLPlanOrErrorMsgContains(sql, "OUTPUT EXPRS:\n" + " 3\n" + " 4"); sql = "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ a.aid, b.bid FROM (SELECT 3 AS aid) a JOIN (SELECT 4 AS bid) b ON (a.aid=b.bid)"; - assertSQLPlanOrErrorMsgContains(sql, "OUTPUT EXPRS:\n" + " \n" + " "); + assertSQLPlanOrErrorMsgContains(sql, "OUTPUT EXPRS:\n" + " 3\n" + " 4"); sql = "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ a.k1, b.k2 FROM (SELECT k1 from baseall) a LEFT OUTER JOIN (select k1, 999 as k2 from baseall) b ON (a.k1=b.k1)"; - assertSQLPlanOrErrorMsgContains(sql, "\n" + " "); + assertSQLPlanOrErrorMsgContains(sql, " `k1`\n" + " 999"); sql = "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ a.k1, b.k2 FROM (SELECT 1 as k1 from baseall) a RIGHT OUTER JOIN (select k1, 999 as k2 from baseall) b ON (a.k1=b.k1)"; - assertSQLPlanOrErrorMsgContains(sql, "\n" + " "); + assertSQLPlanOrErrorMsgContains(sql, " 1\n" + " 999"); sql = "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ a.k1, b.k2 FROM (SELECT 1 as k1 from baseall) a FULL JOIN (select k1, 999 as k2 from baseall) b ON (a.k1=b.k1)"; - assertSQLPlanOrErrorMsgContains(sql, "\n" + " "); + assertSQLPlanOrErrorMsgContains(sql, " 1\n" + " 999"); } @Test @@ -2064,7 +2062,7 @@ public class QueryPlanTest extends TestWithFeService { Assert.assertFalse(explainString.contains("OUTPUT EXPRS:\n 3\n 4")); System.out.println(explainString); Assert.assertTrue(explainString.contains( - "OUTPUT EXPRS:\n" + " CAST( AS INT)\n" + " CAST( AS INT)")); + "OUTPUT EXPRS:\n" + " CAST( 3 AS INT)\n" + " CAST( 4 AS INT)")); } @Test diff --git a/regression-test/data/correctness_p0/test_inlineview_with_project.out b/regression-test/data/correctness_p0/test_inlineview_with_project.out index 009611dd53..238ba7ef7e 100644 --- a/regression-test/data/correctness_p0/test_inlineview_with_project.out +++ b/regression-test/data/correctness_p0/test_inlineview_with_project.out @@ -7,3 +7,6 @@ -- !select3 -- +-- !select4 -- +0.0 + diff --git a/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy b/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy index a155514041..58f46f7901 100644 --- a/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy +++ b/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy @@ -382,4 +382,77 @@ suite("test_inlineview_with_project") { sql """ drop table if exists cir2824_table; """ + + sql """ + drop table if exists dws_mf_wms_t1; + """ + + sql """ + drop table if exists dws_mf_wms_t2; + """ + + sql """ + drop table if exists dws_mf_wms_t3; + """ + + sql """ + CREATE TABLE `dws_mf_wms_t1` ( + `id` varchar(20) NOT NULL COMMENT '', + `final_weight` double NULL COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`id`) + COMMENT '' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + CREATE TABLE `dws_mf_wms_t2` ( + `plate_id` varchar(32) NULL COMMENT '', + `entry_time` datetime NULL COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`plate_id`) + COMMENT '' + DISTRIBUTED BY HASH(`plate_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + CREATE TABLE `dws_mf_wms_t3` ( + `material_id` varchar(50) NULL, + `out_time` datetime NULL COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`material_id`) + COMMENT ' ' + DISTRIBUTED BY HASH(`material_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """insert into dws_mf_wms_t1 values( '1', 1.0);""" + sql """insert into dws_mf_wms_t2 values( '1', '2020-02-02 22:22:22');""" + sql """insert into dws_mf_wms_t3 values( '1', '2020-02-02 22:22:22');""" + + qt_select4 """select cur_final_weight from ( + SELECT + round(`t1`.`final_weight` / 1000 , 2) AS `cur_final_weight`, + coalesce(`t5`.`avg_inv_hours`, 0) AS `avg_inv_hours`, + coalesce(`t5`.`max_inv_hours`, 0) AS `max_inv_hours` + FROM + `dws_mf_wms_t1` t1 + LEFT OUTER JOIN ( + SELECT + round(avg(timestampdiff(SECOND, `t1`.`entry_time`, `t2`.`out_time`)) / 3600.0, 1) AS `avg_inv_hours`, + round(max(timestampdiff(SECOND, `t1`.`entry_time`, `t2`.`out_time`)) / 3600.0, 1) AS `max_inv_hours` + FROM + `dws_mf_wms_t2` t1 + LEFT OUTER JOIN `dws_mf_wms_t3` t2 ON + `t1`.`plate_id` = `t2`.`material_id`) t5 ON + 1 = 1 + )res;""" }