database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](materialized-view) fix divide double can not match mv (#23504)

Browse Source 
* fix divide double can not match mv

* fix

* fix
This commit is contained in:
Pxl
2023-08-28 18:01:08 +08:00
committed by GitHub
parent 392437008c
commit 8e4c0d1e81
10 changed files with 95 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
View File

@ -148,7 +148,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
protected static boolean containsAllColumn(Expression expression, Set<String> mvColumnNames) {
if (mvColumnNames.contains(Column.getNameWithoutMvPrefix(expression.toSql()))) {
if (mvColumnNames.contains(expression.toSql())) {
return true;
}
if (expression.children().isEmpty()) {
@ -400,7 +400,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
for (Slot mvSlot : mvPlan.getOutputByIndex(mvPlan.getSelectedIndexId())) {
boolean isPushed = false;
for (Slot baseSlot : mvPlan.getOutput()) {
if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumnAggregate(mvSlot.getName())) {
if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) {
continue;
}
if (baseSlot.toSql().equalsIgnoreCase(
@ -412,7 +412,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
}
if (!isPushed) {
if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumnAggregate(mvSlot.getName())) {
if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) {
mvNameToMvSlot.put(normalizeName(
org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(mvSlot.getName())),
mvSlot);
@ -570,6 +570,9 @@ public abstract class AbstractSelectMaterializedIndexRule {
if (baseSlotToMvSlot.containsKey(slotReference)) {
return baseSlotToMvSlot.get(slotReference);
}
if (mvNameToMvSlot.containsKey(slotReference.toSql())) {
return mvNameToMvSlot.get(slotReference.toSql());
}
return slotReference;
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java
View File

@ -23,6 +23,8 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DoubleType;
import java.text.NumberFormat;
/**
* Double literal
*/
@ -49,4 +51,11 @@ public class DoubleLiteral extends Literal {
public LiteralExpr toLegacyLiteral() {
return new FloatLiteral(value, Type.DOUBLE);
}
@Override
public String toString() {
NumberFormat nf = NumberFormat.getInstance();
nf.setGroupingUsed(false);
return nf.format(value);
}
}

1
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
View File

@ -231,7 +231,6 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
}
public LogicalProject<Plan> readFromJson(JSONObject logicalProject) {
return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())),
null, null, isDistinct);
}

18
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java
View File

@ -36,7 +36,7 @@ public class SimplifyArithmeticRuleTest extends ExpressionRewriteTestHelper {
assertRewriteAfterTypeCoercion("IA", "IA");
assertRewriteAfterTypeCoercion("IA + 1", "IA + 1");
assertRewriteAfterTypeCoercion("IA + IB", "IA + IB");
assertRewriteAfterTypeCoercion("1 * 3 / IA", "(3.0 / cast(IA as DOUBLE))");
assertRewriteAfterTypeCoercion("1 * 3 / IA", "(3 / cast(IA as DOUBLE))");
assertRewriteAfterTypeCoercion("1 - IA", "1 - IA");
assertRewriteAfterTypeCoercion("1 + 1", "2");
assertRewriteAfterTypeCoercion("IA + 2 - 1", "IA + 1");
@ -44,12 +44,12 @@ public class SimplifyArithmeticRuleTest extends ExpressionRewriteTestHelper {
assertRewriteAfterTypeCoercion("IA + 2 - ((1 - IB) - (3 + IC))", "IA + IB + IC + 4");
assertRewriteAfterTypeCoercion("IA * IB + 2 - IC * 2", "(IA * IB) - (IC * 2) + 2");
assertRewriteAfterTypeCoercion("IA * IB", "IA * IB");
assertRewriteAfterTypeCoercion("IA * IB / 2 * 2", "cast((IA * IB) as DOUBLE) / 1.0");
assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4.0");
assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4.0");
assertRewriteAfterTypeCoercion("IA * (IB / 2) * 2)", "cast(IA as DOUBLE) * cast(IB as DOUBLE) / 1.0");
assertRewriteAfterTypeCoercion("IA * (IB / 2) * (IC + 1))", "cast(IA as DOUBLE) * cast(IB as DOUBLE) * cast((IC + 1) as DOUBLE) / 2.0");
assertRewriteAfterTypeCoercion("IA * IB / 2 / IC * 2 * ID / 4", "(((cast((IA * IB) as DOUBLE) / cast(IC as DOUBLE)) * cast(ID as DOUBLE)) / 4.0)");
assertRewriteAfterTypeCoercion("IA * IB / 2 * 2", "cast((IA * IB) as DOUBLE) / 1");
assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4");
assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4");
assertRewriteAfterTypeCoercion("IA * (IB / 2) * 2)", "cast(IA as DOUBLE) * cast(IB as DOUBLE) / 1");
assertRewriteAfterTypeCoercion("IA * (IB / 2) * (IC + 1))", "cast(IA as DOUBLE) * cast(IB as DOUBLE) * cast((IC + 1) as DOUBLE) / 2");
assertRewriteAfterTypeCoercion("IA * IB / 2 / IC * 2 * ID / 4", "(((cast((IA * IB) as DOUBLE) / cast(IC as DOUBLE)) * cast(ID as DOUBLE)) / 4)");
}
@Test
@ -86,8 +86,8 @@ public class SimplifyArithmeticRuleTest extends ExpressionRewriteTestHelper {
assertRewriteAfterTypeCoercion("IA + 1 > IB", "cast(IA as BIGINT) > (cast(IB as BIGINT) - 1)");
assertRewriteAfterTypeCoercion("IA + 1 > IB * IC", "cast(IA as BIGINT) > ((IB * IC) - 1)");
assertRewriteAfterTypeCoercion("IA * ID > IB * IC", "IA * ID > IB * IC");
assertRewriteAfterTypeCoercion("IA * ID / 2 > IB * IC", "cast((IA * ID) as DOUBLE) > cast((IB * IC) as DOUBLE) * 2.0");
assertRewriteAfterTypeCoercion("IA * ID / -2 > IB * IC", "cast((IB * IC) as DOUBLE) * -2.0 > cast((IA * ID) as DOUBLE)");
assertRewriteAfterTypeCoercion("IA * ID / 2 > IB * IC", "cast((IA * ID) as DOUBLE) > cast((IB * IC) as DOUBLE) * 2");
assertRewriteAfterTypeCoercion("IA * ID / -2 > IB * IC", "cast((IB * IC) as DOUBLE) * -2 > cast((IA * ID) as DOUBLE)");
}
}

13
regression-test/data/mv_p0/test_dup_mv_div/test_dup_mv_div.out Normal file
View File

@ -0,0 +1,13 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select_star --
-4 -4 -4 4
1 1 1 1
2 2 2 b
3 -3 \N c
-- !select_mv --
-4 -4.0
1 1.0
2 2.0
3 -3.0

2
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out
View File

@ -42,7 +42,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------PhysicalDistribute
--------PhysicalTopN
----------PhysicalProject
------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)(CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0.0 END > CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0.0 END)
------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)(CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0 END > CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0 END)
--------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id)
----------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id)
------------------PhysicalDistribute

2
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out
View File

@ -3,7 +3,7 @@
PhysicalCteAnchor ( cteId=CTEId#0 )
--PhysicalCteProducer ( cteId=CTEId#0 )
----PhysicalProject
------filter((CASE WHEN (mean = 0.0) THEN 0.0 ELSE (stdev / mean) END > 1.0))
------filter((CASE WHEN (mean = 0) THEN 0 ELSE (stdev / mean) END > 1))
--------hashAgg[GLOBAL]
----------PhysicalDistribute
------------hashAgg[LOCAL]

2
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out
View File

@ -25,7 +25,7 @@ PhysicalResultSink
------------------------------------PhysicalOlapScan[date_dim]
----------------------------PhysicalDistribute
------------------------------PhysicalProject
--------------------------------filter(((cast(hd_buy_potential as VARCHAR(*)) = '501-1000') OR (cast(hd_buy_potential as VARCHAR(*)) = 'Unknown'))(household_demographics.hd_vehicle_count > 0)(CASE WHEN (hd_vehicle_count > 0) THEN (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) ELSE NULL END > 1.0))
--------------------------------filter(((cast(hd_buy_potential as VARCHAR(*)) = '501-1000') OR (cast(hd_buy_potential as VARCHAR(*)) = 'Unknown'))(household_demographics.hd_vehicle_count > 0)(CASE WHEN (hd_vehicle_count > 0) THEN (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) ELSE NULL END > 1))
----------------------------------PhysicalOlapScan[household_demographics]
--------------------------PhysicalDistribute
----------------------------PhysicalProject

6
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out
View File

@ -42,12 +42,12 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------PhysicalDistribute
--------PhysicalTopN
----------PhysicalProject
------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id)(CASE WHEN (year_total > 0.0) THEN (year_total / year_total) ELSE NULL END > CASE WHEN (year_total > 0.0) THEN (year_total / year_total) ELSE NULL END)
------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id)(CASE WHEN (year_total > 0) THEN (year_total / year_total) ELSE NULL END > CASE WHEN (year_total > 0) THEN (year_total / year_total) ELSE NULL END)
--------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)
----------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id)
------------------PhysicalDistribute
--------------------PhysicalProject
----------------------filter((t_s_firstyear.year = 1999)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0.0))
----------------------filter((t_s_firstyear.year = 1999)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0))
------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
------------------PhysicalDistribute
--------------------PhysicalProject
@ -59,6 +59,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
----------------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------------PhysicalDistribute
----------------PhysicalProject
------------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0.0)(t_w_firstyear.sale_type = 'w'))
------------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0)(t_w_firstyear.sale_type = 'w'))
--------------------PhysicalCteConsumer ( cteId=CTEId#0 )

52
regression-test/suites/mv_p0/test_dup_mv_div/test_dup_mv_div.groovy Normal file
View File

@ -0,0 +1,52 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
suite ("test_dup_mv_div") {
sql """set enable_nereids_planner=true"""
sql """SET enable_fallback_to_original_planner=false"""
sql """ DROP TABLE IF EXISTS d_table; """
sql """
create table d_table(
k1 int null,
k2 int not null,
k3 bigint null,
k4 varchar(100) null
)
duplicate key (k1,k2,k3)
distributed BY hash(k1) buckets 3
properties("replication_num" = "1");
"""
sql "insert into d_table select 1,1,1,'1';"
sql "insert into d_table select 2,2,2,'b';"
sql "insert into d_table select 3,-3,null,'c';"
createMV ("create materialized view kdiv as select k1,k2/1 from d_table;")
sql "insert into d_table select -4,-4,-4,'4';"
qt_select_star "select * from d_table order by k1;"
explain {
sql("select k1,k2/1 from d_table order by k1;")
contains "(kdiv)"
}
qt_select_mv "select k1,k2/1 from d_table order by k1;"
}