From 32a32808a70738d8d819b6b544e3dd9d80efa74d Mon Sep 17 00:00:00 2001 From: feiniaofeiafei <53502832+feiniaofeiafei@users.noreply.github.com> Date: Fri, 30 Aug 2024 13:39:37 +0800 Subject: [PATCH] [improvement](nereids) support convert_tz in partition prune (#40047) (#40127) cherry-pick #40047 to branch-2.1 --- .../rules/OneRangePartitionEvaluator.java | 14 +++ .../functions/scalar/ConvertTz.java | 19 +++- .../partition_prune/test_convert_tz.groovy | 99 +++++++++++++++++++ 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/nereids_rules_p0/partition_prune/test_convert_tz.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index 9de31fde6d..c3345c3469 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz; import org.apache.doris.nereids.trees.expressions.functions.scalar.Date; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -638,6 +639,19 @@ public class OneRangePartitionEvaluator return computeMonotonicFunctionRange(result); } + @Override + public EvaluateRangeResult visitConvertTz(ConvertTz convertTz, EvaluateRangeInput context) { + EvaluateRangeResult result = super.visitConvertTz(convertTz, context); + if (!(result.result instanceof ConvertTz)) { + return result; + } + Expression converTzChild = convertTz.child(0); + if (partitionSlotContainsNull.containsKey(converTzChild)) { + partitionSlotContainsNull.put(convertTz, true); + } + return computeMonotonicFunctionRange(result); + } + private boolean isPartitionSlot(Slot slot) { return slotToType.containsKey(slot); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java index 75270fa022..11b44d8ace 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; @@ -39,7 +41,7 @@ import java.util.List; * ScalarFunction 'convert_tz'. This class is generated by GenerateFunction. */ public class ConvertTz extends ScalarFunction - implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + implements TernaryExpression, ExplicitlyCastableSignature, AlwaysNullable, Monotonic { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT) @@ -84,4 +86,19 @@ public class ConvertTz extends ScalarFunction public R accept(ExpressionVisitor visitor, C context) { return visitor.visitConvertTz(this, context); } + + @Override + public boolean isPositive() { + return true; + } + + @Override + public int getMonotonicFunctionChildIndex() { + return 0; + } + + @Override + public Expression withConstantArgs(Literal literal) { + return new ConvertTz(literal, child(1), child(2)); + } } diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_convert_tz.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_convert_tz.groovy new file mode 100644 index 0000000000..c309d10d06 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_convert_tz.groovy @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_convert_tz") { + sql "set enable_fallback_to_original_planner=false" + sql "drop table if exists test_convert_tz;" + sql """CREATE TABLE test_convert_tz + ( + timestamp DATETIME NOT NULL + ) + ENGINE = olap + PARTITION BY range (timestamp) + ( + PARTITION `p1` VALUES LESS THAN ('2021-01-01'), + PARTITION `p2` VALUES LESS THAN ('2021-02-01'), + PARTITION `p3` VALUES LESS THAN ('2021-03-01') + ) DISTRIBUTED BY HASH (timestamp) + PROPERTIES( + "storage_format" = "DEFAULT", + "replication_num" = "1");""" + sql """INSERT INTO test_convert_tz (timestamp) + VALUES ('2020-12-31'), + ('2021-01-05'), + ('2021-01-15'), + ('2021-02-05'), + ('2021-02-15');""" + + explain { + sql "SELECT * FROM test_convert_tz WHERE convert_tz(timestamp, 'Asia/Shanghai', 'Europe/Paris') < '2021-01-01';" + contains("partitions=2/3 (p1,p2)") + } + explain { + sql "SELECT * FROM test_convert_tz WHERE convert_tz(timestamp, 'Asia/Shanghai', 'Europe/Paris') > '2021-01-01';"; + contains("partitions=2/3 (p2,p3)") + } + + explain { + sql """SELECT * FROM test_convert_tz WHERE convert_tz(timestamp, 'Asia/Shanghai', 'Europe/Paris') < '2021-02-24' + and convert_tz(timestamp, 'Asia/Shanghai', 'Europe/Paris') > '2021-01-01';""" + contains("partitions=2/3 (p2,p3)") + } + + explain { + sql """SELECT * FROM test_convert_tz WHERE convert_tz(timestamp, 'Asia/Shanghai', 'Europe/Paris') < '2021-02-24' + or convert_tz(timestamp, 'Asia/Shanghai', 'Europe/Paris') > '2021-01-01';""" + contains("partitions=3/3 (p1,p2,p3)") + } + + explain { + sql "SELECT * FROM test_convert_tz WHERE convert_tz(timestamp, 'Asia/Beijing', 'Europe/Paris') is null;"; + contains("partitions=3/3 (p1,p2,p3)") + } + + explain { + sql "SELECT * FROM test_convert_tz WHERE convert_tz(timestamp, 'Asia/Beijing', 'Europe/Paris') is not null;"; + contains("partitions=3/3 (p1,p2,p3)") + } + + explain { + sql "SELECT * FROM test_convert_tz WHERE date_trunc(convert_tz(timestamp, 'Asia/Beijing', 'Europe/Paris'), 'month') <'2021-01-01';"; + contains("partitions=3/3 (p1,p2,p3)") + } + + explain { + sql "SELECT * FROM test_convert_tz WHERE date_trunc(convert_tz(timestamp, 'Asia/Shanghai', 'Europe/Paris'), 'month') <'2021-01-01';"; + contains("partitions=2/3 (p1,p2)") + } + + explain { + sql "SELECT * FROM test_convert_tz WHERE convert_tz(date_trunc(timestamp, 'month'), 'Asia/Shanghai', 'Europe/Paris') <'2021-01-01';"; + contains("partitions=2/3 (p1,p2)") + } + for (int i = 0; i < 2; i++) { + if (i == 0) { + sql "set disable_nereids_rules = 'REWRITE_FILTER_EXPRESSION'" + } else { + sql "set disable_nereids_rules = ''" + } + explain { + sql "SELECT * FROM test_convert_tz WHERE not convert_tz(timestamp, 'Asia/Shanghai', 'Europe/Paris') <= '2021-01-01';"; + contains("partitions=2/3 (p2,p3)") + } + } +} \ No newline at end of file