From cc11e50200f092006c287684384c2e85f7d94995 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Fri, 17 May 2024 11:39:39 +0800 Subject: [PATCH] [fix](mtmv)Fix slot desc wrong in query rewrite by materialized view when query is complex (#34904) --- .../java/org/apache/doris/catalog/MTMV.java | 8 +- .../mv/AbstractMaterializedViewRule.java | 2 +- .../mv/InitMaterializationContextHook.java | 5 +- .../mv/MaterializationContext.java | 13 +- .../exploration/mv/MaterializedViewUtils.java | 7 +- .../trees/plans/visitor/TableCollector.java | 14 +- .../mv/nested/ddl/customer_create.sql | 15 + .../mv/nested/ddl/customer_delete.sql | 1 + .../mv/nested/ddl/date_create.sql | 24 + .../mv/nested/ddl/date_delete.sql | 1 + .../mv/nested/ddl/lineorder_create.sql | 24 + .../mv/nested/ddl/lineorder_delete.sql | 1 + .../mv/nested/ddl/lineorder_flat_create.sql | 45 ++ .../mv/nested/ddl/lineorder_flat_delete.sql | 1 + .../mv/nested/ddl/part_create.sql | 16 + .../mv/nested/ddl/part_delete.sql | 1 + .../mv/nested/ddl/supplier_create.sql | 14 + .../mv/nested/ddl/supplier_delete.sql | 1 + .../mv/nested/nested_materialized_view.groovy | 750 +++++++++++++++++- 19 files changed, 920 insertions(+), 23 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_delete.sql diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index a3050bc3ac..fb32741afa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -40,6 +40,7 @@ import org.apache.doris.mtmv.MTMVRefreshSnapshot; import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVStatus; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -267,12 +268,15 @@ public class MTMV extends OlapTable { } } - public MTMVCache getOrGenerateCache() throws AnalysisException { + /** + * Called when in query, Should use one connection context in query + */ + public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws AnalysisException { if (cache == null) { writeMvLock(); try { if (cache == null) { - this.cache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this)); + this.cache = MTMVCache.from(this, connectionContext); } } finally { writeMvUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index a88362e669..2274ad441f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -323,9 +323,9 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac continue; } recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext); + rewriteResults.add(rewrittenPlan); // if rewrite successfully, try to regenerate mv scan because it maybe used again materializationContext.tryReGenerateMvScanPlan(cascadesContext); - rewriteResults.add(rewrittenPlan); } return rewriteResults; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 311932fa1b..d91fa5cb1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -63,6 +63,9 @@ public class InitMaterializationContextHook implements PlannerHook { } Plan rewritePlan = cascadesContext.getRewritePlan(); TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), true); + // Keep use one connection context when in query, if new connect context, + // the ConnectionContext.get() will change + collectorContext.setConnectContext(cascadesContext.getConnectContext()); rewritePlan.accept(TableCollector.INSTANCE, collectorContext); Set collectedTables = collectorContext.getCollectedTables(); if (collectedTables.isEmpty()) { @@ -80,7 +83,7 @@ public class InitMaterializationContextHook implements PlannerHook { for (MTMV materializedView : availableMTMVs) { MTMVCache mtmvCache = null; try { - mtmvCache = materializedView.getOrGenerateCache(); + mtmvCache = materializedView.getOrGenerateCache(cascadesContext.getConnectContext()); } catch (AnalysisException e) { LOG.warn("MaterializationContext init mv cache generate fail", e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 2f0d04e114..c2c2448b24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -134,15 +134,14 @@ public abstract class MaterializationContext { * Try to generate scan plan for materialization * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later * query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output - * should be different + * should be different. + * This method should be called when query rewrite successfully */ public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) { - if (!this.matchedSuccessGroups.isEmpty()) { - this.mvScanPlan = doGenerateMvPlan(cascadesContext); - // mv output expression shuttle, this will be used to expression rewrite - this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, - this.mvScanPlan.getExpressions()); - } + this.mvScanPlan = doGenerateMvPlan(cascadesContext); + // mv output expression shuttle, this will be used to expression rewrite + this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, + this.mvScanPlan.getOutput()); } public void addSlotMappingToCache(RelationMapping relationMapping, SlotMapping slotMapping) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 608f3b5f5f..f1a90fb940 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -184,12 +184,13 @@ public class MaterializedViewUtils { LogicalOlapScan mvScan = new LogicalOlapScan( cascadesContext.getStatementContext().getNextRelationId(), materializedView, - ImmutableList.of(materializedView.getQualifiedDbName()), + materializedView.getFullQualifiers(), + ImmutableList.of(), + materializedView.getBaseIndexId(), + PreAggStatus.on(), // this must be empty, or it will be used to sample ImmutableList.of(), - ImmutableList.of(), Optional.empty()); - mvScan = mvScan.withMaterializedIndexSelected(PreAggStatus.on(), materializedView.getBaseIndexId()); List mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast) .collect(Collectors.toList()); return new LogicalProject(mvProjects, mvScan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index 94b1eb89c2..5ab6b7ef01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; +import org.apache.doris.qe.ConnectContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -72,7 +73,9 @@ public class TableCollector extends DefaultPlanVisitor targetTableTypes; // if expand the mv or not private final boolean expand; + private ConnectContext connectContext; public TableCollectorContext(Set targetTableTypes, boolean expand) { this.targetTableTypes = targetTableTypes; @@ -102,5 +106,13 @@ public class TableCollector extends DefaultPlanVisitor + sql "SET enable_materialized_view_rewrite=false" + def origin_res = sql stmt + logger.info("origin_res: " + origin_res) + sql "SET enable_materialized_view_rewrite=true" + def mv_origin_res = sql stmt + logger.info("mv_origin_res: " + mv_origin_res) + assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size())) + for (int row = 0; row < mv_origin_res.size(); row++) { + assertTrue(mv_origin_res[row].size() == origin_res[row].size()) + for (int col = 0; col < mv_origin_res[row].size(); col++) { + assertTrue(mv_origin_res[row][col] == origin_res[row][col]) + } + } + } + + // ssb_sf1_p1 is writted to test unique key table merge correctly. + // It creates unique key table and sets bucket num to 1 in order to make sure that + // many rowsets will be created during loading and then the merge process will be triggered. + + def tables = ["customer", "lineorder", "part", "date", "supplier"] + def columns = ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", + """lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""", + """p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy""", + """d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", + """s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy"""] + + for (String table in tables) { + sql new File("""${context.file.parent}/ddl/${table}_create.sql""").text + sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text + } + def i = 0 + for (String tableName in tables) { + streamLoad { + // a default db 'regression_test' is specified in + // ${DORIS_HOME}/conf/regression-conf.groovy + table tableName + + // default label is UUID: + // set 'label' UUID.randomUUID().toString() + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', columns[i] + + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + i++ + } + sql """ sync """ + + String db = context.config.getDbNameByFile(context.file) sql "use ${db}" sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" + sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite = true" def create_mtmv = { db_name, mv_name, mv_sql -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" sql""" - CREATE MATERIALIZED VIEW ${mv_name} + CREATE MATERIALIZED VIEW ${mv_name} BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ('replication_num' = '1') + PROPERTIES ('replication_num' = '1') AS ${mv_sql} """ @@ -50,8 +130,8 @@ suite("nested_materialized_view") { o_orderstatus CHAR(1) NOT NULL, o_totalprice DECIMALV3(15,2) NOT NULL, o_orderdate DATE NOT NULL, - o_orderpriority CHAR(15) NOT NULL, - o_clerk CHAR(15) NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, o_shippriority INTEGER NOT NULL, o_comment VARCHAR(79) NOT NULL ) @@ -104,7 +184,7 @@ suite("nested_materialized_view") { ps_suppkey INTEGER NOT NULL, ps_availqty INTEGER NOT NULL, ps_supplycost DECIMALV3(15,2) NOT NULL, - ps_comment VARCHAR(199) NOT NULL + ps_comment VARCHAR(199) NOT NULL ) DUPLICATE KEY(ps_partkey, ps_suppkey) DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 @@ -130,7 +210,7 @@ suite("nested_materialized_view") { (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), - (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); """ sql """ @@ -192,7 +272,661 @@ suite("nested_materialized_view") { sql("${query1_0}") contains("mv1_0_inner_mv(mv1_0_inner_mv)") } - order_qt_query1_1_after "${query1_0}" + + // complex nest mv rewrite + create_mtmv(db, "mv1_a", """ + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19930423 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate;""") + + create_mtmv(db, "mv2_a", """ + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19930423 + group by + lo_custkey, + lo_orderdate;""") + + create_mtmv(db, "mv4_a", """ + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19930423 + group by + lo_partkey, + lo_orderdate;""") + + create_mtmv(db, "mv_all_6_a", """ + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value4, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum +from + mv1_a t1 + left join mv2_a t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join mv4_a t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join mv4_a t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring'; + """) + + create_mtmv(db, "mv1_b", """ +select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19980421 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate;""") + + create_mtmv(db, "mv2_b", """ + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19980421 + group by + lo_custkey, + lo_orderdate;""") + + create_mtmv(db, "mv4_b", """ + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19980421 + group by + lo_partkey, + lo_orderdate;""") + + create_mtmv(db, "mv_all_6_b", """ + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value4, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum +from + mv1_b t1 + left join mv2_b t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join mv4_b t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join mv4_b t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring'; + """) + + + create_mtmv(db, "mv1_c", """ + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940413 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate;""") + + create_mtmv(db, "mv2_c", """ + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940413 + group by + lo_custkey, + lo_orderdate;""") + + create_mtmv(db, "mv4_c", """ + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940413 + group by + lo_partkey, + lo_orderdate;""") + + create_mtmv(db, "mv_all_6_c", """ + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value4, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum +from + mv1_c t1 + left join mv2_c t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join mv4_c t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join mv4_c t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring'; + """) + + + create_mtmv(db, "mv1_d", """ + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940218 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate;""") + + create_mtmv(db, "mv2_d", """ + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940218 + group by + lo_custkey, + lo_orderdate;""") + + create_mtmv(db, "mv4_d", """ + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940218 + group by + lo_partkey, + lo_orderdate;""") + + create_mtmv(db, "mv_all_6_d", """ + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value4, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum +from + mv1_d t1 + left join mv2_d t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join mv4_d t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join mv4_d t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring'; + """) + + def query2_0 = """ +select * from ( + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value3, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum + from + ( + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19930423 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate + ) t1 + left join ( + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19930423 + group by + lo_custkey, + lo_orderdate + ) t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value3 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19930423 + group by + lo_partkey, + lo_orderdate + ) t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19930423 + group by + lo_partkey, + lo_orderdate + ) t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring' + union all + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value3, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum + from + ( + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19980421 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate + ) t1 + left join ( + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19980421 + group by + lo_custkey, + lo_orderdate + ) t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value3 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19980421 + group by + lo_partkey, + lo_orderdate + ) t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19980421 + group by + lo_partkey, + lo_orderdate + ) t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring' + union ALL + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value3, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum + from + ( + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940413 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate + ) t1 + left join ( + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940413 + group by + lo_custkey, + lo_orderdate + ) t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value3 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940413 + group by + lo_partkey, + lo_orderdate + ) t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940413 + group by + lo_partkey, + lo_orderdate + ) t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring' + UNION ALL + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value3, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum + from + ( + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940218 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate + ) t1 + left join ( + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940218 + group by + lo_custkey, + lo_orderdate + ) t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value3 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940218 + group by + lo_partkey, + lo_orderdate + ) t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940218 + group by + lo_partkey, + lo_orderdate + ) t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring' +) t order by 1,2,3,4,5,6,7,8,9; + """ + + sql "SET enable_materialized_view_rewrite= true" + sql "SET enable_materialized_view_nest_rewrite = true" + explain { + sql("${query2_0}") + check {result -> + result.contains("mv_all_6_a(mv_all_6_a)") && result.contains("mv_all_6_b(mv_all_6_b)") + && result.contains("mv_all_6_c(mv_all_6_c)") && result.contains("mv_all_6_d(mv_all_6_d)") + } + } + // Compare result when before and after mv rewrite + compare_res(query2_0) }