From cfb110c90552441a372d0d1e5cc542bc097df52d Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Wed, 11 Jan 2023 17:18:44 +0800 Subject: [PATCH] [fix](nereids) fix some nereids bugs (#15714) 1. remove forcing nullable for slot on EmptySetNode. 2. order by xxx desc should use nulls last as default order. 3. don't create runtime filter if runtime filter mode is OFF. 4. group by constant value need check the corresponding expr shouldn't have any aggregation functions. 5. fix two left outer join reorder bug( A left join B left join C). 6. fix semi join and left outer join reorder bug.( A left join B semi join C ). 7. fix group by NULL bug. 8. change ceil and floor function to correct signature. 9. add literal comparasion for string and date type. 10. fix the getOnClauseUsedSlots method may not return valid value. 11. the tightness common type of string and date should be date. 12. the nullability of set operation node's result exprs is not set correctly. 13. Sort node should remove redundent ordering exprs. --- .../translator/PhysicalPlanTranslator.java | 3 - .../jobs/batch/NereidsRewriteJobExecutor.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 2 +- .../processor/post/PlanPostProcessors.java | 5 +- .../nereids/rules/analysis/CheckAnalysis.java | 7 + .../ResolveOrdinalInOrderByAndGroupBy.java | 10 +- .../join/OuterJoinLAsscomProject.java | 79 ++++++--- .../join/SemiJoinLogicalJoinTranspose.java | 16 +- .../SemiJoinLogicalJoinTransposeProject.java | 19 ++- .../ApplyPullFilterOnProjectUnderAgg.java | 4 +- .../logical/EliminateGroupByConstant.java | 2 +- .../rewrite/logical/NormalizeToSlot.java | 8 +- .../expressions/functions/scalar/Ceil.java | 3 +- .../expressions/functions/scalar/Floor.java | 3 +- .../trees/expressions/literal/Literal.java | 10 +- .../trees/plans/logical/LogicalAggregate.java | 33 +++- .../apache/doris/nereids/util/JoinUtils.java | 11 +- .../doris/nereids/util/TypeCoercionUtils.java | 4 + .../doris/planner/SetOperationNode.java | 3 + .../org/apache/doris/planner/SortNode.java | 8 +- .../join/OuterJoinLAsscomProjectTest.java | 39 +---- .../data/nereids_syntax_p0/function.out | 5 + .../nereids_syntax_p0/group_by_constant.out | 17 ++ .../data/nereids_syntax_p0/join_order.out | 4 + .../data/nereids_syntax_p0/set_operation.out | 4 + .../data/nereids_syntax_p0/type_cast.out | 7 + .../data/query_p0/keyword/order_group.out | 2 +- .../suites/nereids_syntax_p0/function.groovy | 8 + .../group_by_constant.groovy | 19 +++ .../suites/nereids_syntax_p0/join.groovy | 155 ++++++++++++++++++ .../nereids_syntax_p0/join_order.groovy | 96 +++++++++++ .../nereids_syntax_p0/set_operation.groovy | 16 +- .../suites/nereids_syntax_p0/type_cast.groovy | 41 +++++ 33 files changed, 539 insertions(+), 106 deletions(-) create mode 100644 regression-test/data/nereids_syntax_p0/join_order.out create mode 100644 regression-test/data/nereids_syntax_p0/type_cast.out create mode 100644 regression-test/suites/nereids_syntax_p0/join_order.groovy create mode 100644 regression-test/suites/nereids_syntax_p0/type_cast.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index d563e18eb7..d84720ee9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -361,9 +361,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor output = emptyRelation.getOutput(); TupleDescriptor tupleDescriptor = generateTupleDesc(output, null, context); for (int i = 0; i < output.size(); i++) { - SlotDescriptor slotDescriptor = tupleDescriptor.getSlots().get(i); - slotDescriptor.setIsNullable(true); // we should set to nullable, or else BE would core - Slot slot = output.get(i); SlotRef slotRef = context.findSlotRef(slot.getExprId()); slotRef.setLabel(slot.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java index 17f0e72e7a..275901dff7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java @@ -86,6 +86,7 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob { .add(topDownBatch(ImmutableList.of(new ExpressionNormalization(cascadesContext.getConnectContext())))) .add(topDownBatch(ImmutableList.of(new ExpressionOptimization()))) .add(topDownBatch(ImmutableList.of(new ExtractSingleTableExpressionFromDisjunction()))) + .add(topDownBatch(ImmutableList.of(new EliminateGroupByConstant()))) .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) .add(visitorJob(RuleType.INFER_PREDICATES, new InferPredicates())) @@ -107,7 +108,6 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob { .add(topDownBatch(ImmutableList.of(new CountDistinctRewrite()))) // we need to execute this rule at the end of rewrite // to avoid two consecutive same project appear when we do optimization. - .add(topDownBatch(ImmutableList.of(new EliminateGroupByConstant()))) .add(topDownBatch(ImmutableList.of(new EliminateOrderByConstant()))) .add(topDownBatch(ImmutableList.of(new EliminateUnnecessaryProject()))) .add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithAggregate()))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 951fbc7cc8..56260c999b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1108,7 +1108,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { public OrderKey visitSortItem(SortItemContext ctx) { return ParserUtils.withOrigin(ctx, () -> { boolean isAsc = ctx.DESC() == null; - boolean isNullFirst = ctx.LAST() == null; + boolean isNullFirst = ctx.FIRST() != null || (ctx.LAST() == null && isAsc); Expression expression = typedVisit(ctx.expression()); return new OrderKey(expression, isAsc, isNullFirst); }); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java index 269dc94be1..090f523b18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.processor.post; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TRuntimeFilterMode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -57,7 +58,9 @@ public class PlanPostProcessors { public List getProcessors() { // add processor if we need Builder builder = ImmutableList.builder(); - if (cascadesContext.getConnectContext().getSessionVariable().isEnableNereidsRuntimeFilter()) { + if (cascadesContext.getConnectContext().getSessionVariable().isEnableNereidsRuntimeFilter() + && !cascadesContext.getConnectContext().getSessionVariable().getRuntimeFilterMode() + .toUpperCase().equals(TRuntimeFilterMode.OFF.name())) { builder.add(new RuntimeFilterGenerator()); if (ConnectContext.get().getSessionVariable().enableRuntimeFilterPrune) { builder.add(new RuntimeFilterPruner()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java index a4e14bfc80..8d45087b84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java @@ -104,5 +104,12 @@ public class CheckAnalysis implements AnalysisRuleFactory { throw new AnalysisException( "The query contains multi count distinct or sum distinct, each can't have multi columns"); } + Optional expr = aggregate.getGroupByExpressions().stream() + .filter(expression -> expression.containsType(AggregateFunction.class)).findFirst(); + if (expr.isPresent()) { + throw new AnalysisException( + "GROUP BY expression must not contain aggregate functions: " + + expr.get().toSql()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java index b04f085cef..5ee1a8fad6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -63,7 +64,7 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { }) )) .add(RuleType.RESOLVE_ORDINAL_IN_GROUP_BY.build( - logicalAggregate().then(agg -> { + logicalAggregate().whenNot(agg -> agg.isOrdinalIsResolved()).then(agg -> { List aggOutput = agg.getOutputExpressions(); List groupByWithoutOrd = new ArrayList<>(); boolean ordExists = false; @@ -74,6 +75,9 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { int ord = i.getIntValue(); checkOrd(ord, aggOutput.size()); Expression aggExpr = aggOutput.get(ord - 1); + if (aggExpr instanceof Alias) { + aggExpr = ((Alias) aggExpr).child(); + } groupByWithoutOrd.add(aggExpr); ordExists = true; } else { @@ -81,11 +85,11 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { } } if (ordExists) { - return new LogicalAggregate(groupByWithoutOrd, agg.getOutputExpressions(), agg.child()); + return new LogicalAggregate(groupByWithoutOrd, agg.getOutputExpressions(), true, + agg.child()); } else { return agg; } - }))).build(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java index a356a71dea..d9ed3ffe04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; import org.apache.doris.nereids.util.PlanUtils; @@ -75,26 +76,25 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { GroupPlan a = bottomJoin.left(); GroupPlan b = bottomJoin.right(); GroupPlan c = topJoin.right(); - Set bOutputSet = b.getOutputSet(); + Set aOutputSet = a.getOutputSet(); Set cOutputSet = c.getOutputSet(); /* ********** Split projects ********** */ Map> projectExprsMap = projects.stream() .collect(Collectors.partitioningBy(projectExpr -> { Set usedSlots = projectExpr.collect(SlotReference.class::isInstance); - return bOutputSet.containsAll(usedSlots); + return aOutputSet.containsAll(usedSlots); })); - List newLeftProjects = projectExprsMap.get(Boolean.FALSE); - List newRightProjects = projectExprsMap.get(Boolean.TRUE); - - Set bExprIdSet = InnerJoinLAsscomProject.getExprIdSetForB(bottomJoin.right(), - newRightProjects); + List newLeftProjects = projectExprsMap.get(Boolean.TRUE); + List newRightProjects = projectExprsMap.get(Boolean.FALSE); + Set aExprIdSet = getExprIdSetForA(bottomJoin.left(), + newLeftProjects); /* ********** split Conjuncts ********** */ - Map> splitHashJoinConjuncts - = InnerJoinLAsscomProject.splitConjunctsWithAlias( - topJoin.getHashJoinConjuncts(), bottomJoin.getHashJoinConjuncts(), bExprIdSet); - List newTopHashJoinConjuncts = splitHashJoinConjuncts.get(true); + Map> newHashJoinConjuncts + = createNewConjunctsWithAlias( + topJoin.getHashJoinConjuncts(), bottomJoin.getHashJoinConjuncts(), aExprIdSet); + List newTopHashJoinConjuncts = newHashJoinConjuncts.get(true); Preconditions.checkState(!newTopHashJoinConjuncts.isEmpty(), "LAsscom newTopHashJoinConjuncts join can't empty"); // When newTopHashJoinConjuncts.size() != bottomJoin.getHashJoinConjuncts().size() @@ -103,25 +103,21 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { && newTopHashJoinConjuncts.size() != bottomJoin.getHashJoinConjuncts().size()) { return null; } - List newBottomHashJoinConjuncts = splitHashJoinConjuncts.get(false); + List newBottomHashJoinConjuncts = newHashJoinConjuncts.get(false); if (newBottomHashJoinConjuncts.size() == 0) { return null; } - Map> splitOtherJoinConjuncts - = InnerJoinLAsscomProject.splitConjunctsWithAlias( + Map> newOtherJoinConjuncts + = createNewConjunctsWithAlias( topJoin.getOtherJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), - bExprIdSet); - List newTopOtherJoinConjuncts = splitOtherJoinConjuncts.get(true); - // When topJoin type differ from bottomJoin type (LOJ-inner or inner LOJ), - // we just can exchange topJoin and bottomJoin. like: - // Failed in: (A LOJ B on A.id = B.id) join C on c.id = A.id & c.id = B.id (top contain c.id = B.id) - // If type is same like LOJ(LOJ()), we can LAsscom. - if (topJoin.getJoinType() != bottomJoin.getJoinType() - && newTopOtherJoinConjuncts.size() != bottomJoin.getOtherJoinConjuncts().size()) { + aExprIdSet); + List newTopOtherJoinConjuncts = newOtherJoinConjuncts.get(true); + List newBottomOtherJoinConjuncts = newOtherJoinConjuncts.get(false); + if (newBottomOtherJoinConjuncts.size() != topJoin.getOtherJoinConjuncts().size() + || newTopOtherJoinConjuncts.size() != bottomJoin.getOtherJoinConjuncts().size()) { return null; } - List newBottomOtherJoinConjuncts = splitOtherJoinConjuncts.get(false); /* ********** replace Conjuncts by projects ********** */ Map inputToOutput = new HashMap<>(); @@ -158,10 +154,10 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { return usedSlotRefs.stream(); }) .filter(slot -> !cOutputSet.contains(slot)) - .collect(Collectors.partitioningBy(slot -> bExprIdSet.contains(slot.getExprId()), + .collect(Collectors.partitioningBy(slot -> aExprIdSet.contains(slot.getExprId()), Collectors.toSet())); - Set aUsedSlots = abOnUsedSlots.get(Boolean.FALSE); - Set bUsedSlots = abOnUsedSlots.get(Boolean.TRUE); + Set aUsedSlots = abOnUsedSlots.get(Boolean.TRUE); + Set bUsedSlots = abOnUsedSlots.get(Boolean.FALSE); JoinUtils.addSlotsUsedByOn(bUsedSlots, newRightProjects); JoinUtils.addSlotsUsedByOn(aUsedSlots, newLeftProjects); @@ -200,4 +196,35 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { return PlanUtils.project(new ArrayList<>(topJoin.getOutput()), newTopJoin).get(); }).toRule(RuleType.LOGICAL_OUTER_JOIN_LASSCOM_PROJECT); } + + private Map> createNewConjunctsWithAlias(List topConjuncts, + List bottomConjuncts, Set bExprIdSet) { + // if top join's conjuncts are all related to A, we can do reorder + Map> splitOn = new HashMap<>(); + splitOn.put(true, new ArrayList<>()); + if (topConjuncts.stream().allMatch(topHashOn -> { + Set usedSlots = topHashOn.getInputSlots(); + Set usedSlotsId = usedSlots.stream().map(NamedExpression::getExprId) + .collect(Collectors.toSet()); + + return ExpressionUtils.isIntersecting(bExprIdSet, usedSlotsId); + })) { + // do reorder, create new bottom join conjuncts + splitOn.put(false, new ArrayList<>(topConjuncts)); + } else { + // can't reorder, return empty list + splitOn.put(false, new ArrayList<>()); + } + + List newTopHashJoinConjuncts = splitOn.get(true); + newTopHashJoinConjuncts.addAll(bottomConjuncts); + + return splitOn; + } + + private Set getExprIdSetForA(GroupPlan a, List aProject) { + return Stream.concat( + a.getOutput().stream().map(NamedExpression::getExprId), + aProject.stream().map(NamedExpression::getExprId)).collect(Collectors.toSet()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java index 0f4c57a09f..c601b8720d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java @@ -54,9 +54,10 @@ public class SemiJoinLogicalJoinTranspose extends OneExplorationRuleFactory { @Override public Rule build() { return logicalJoin(logicalJoin(), group()) - .when(topJoin -> topJoin.getJoinType() == JoinType.LEFT_SEMI_JOIN - || topJoin.getJoinType() == JoinType.LEFT_ANTI_JOIN - || topJoin.getJoinType() == JoinType.NULL_AWARE_LEFT_ANTI_JOIN) + .when(topJoin -> (topJoin.getJoinType().isLeftSemiOrAntiJoin() + && (topJoin.left().getJoinType().isInnerJoin() + || topJoin.left().getJoinType().isLeftOuterJoin() + || topJoin.left().getJoinType().isRightOuterJoin()))) .whenNot(topJoin -> topJoin.left().getJoinType().isSemiOrAntiJoin()) .when(this::conditionChecker) .whenNot(topJoin -> topJoin.hasJoinHint() || topJoin.left().hasJoinHint()) @@ -75,6 +76,8 @@ public class SemiJoinLogicalJoinTranspose extends OneExplorationRuleFactory { lasscom = ExpressionUtils.isIntersecting(usedSlot, aOutputSet) || lasscom; } + JoinType newTopJoinType = JoinType.INNER_JOIN; + if (lasscom) { /* * topSemiJoin newTopJoin @@ -85,9 +88,10 @@ public class SemiJoinLogicalJoinTranspose extends OneExplorationRuleFactory { */ LogicalJoin newBottomSemiJoin = new LogicalJoin<>( topSemiJoin.getJoinType(), - topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), JoinHint.NONE, + topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), + JoinHint.NONE, a, c); - return new LogicalJoin<>(bottomJoin.getJoinType(), bottomJoin.getHashJoinConjuncts(), + return new LogicalJoin<>(newTopJoinType, bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), JoinHint.NONE, newBottomSemiJoin, b); } else { /* @@ -102,7 +106,7 @@ public class SemiJoinLogicalJoinTranspose extends OneExplorationRuleFactory { topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), JoinHint.NONE, b, c); - return new LogicalJoin<>(bottomJoin.getJoinType(), bottomJoin.getHashJoinConjuncts(), + return new LogicalJoin<>(newTopJoinType, bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), JoinHint.NONE, a, newBottomSemiJoin); } }).toRule(RuleType.LOGICAL_SEMI_JOIN_LOGICAL_JOIN_TRANSPOSE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java index b517c47f3c..20e2176f00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java @@ -56,9 +56,10 @@ public class SemiJoinLogicalJoinTransposeProject extends OneExplorationRuleFacto @Override public Rule build() { return logicalJoin(logicalProject(logicalJoin()), group()) - .when(topJoin -> topJoin.getJoinType() == JoinType.LEFT_SEMI_JOIN - || topJoin.getJoinType() == JoinType.LEFT_ANTI_JOIN - || topJoin.getJoinType() == JoinType.NULL_AWARE_LEFT_ANTI_JOIN) + .when(topJoin -> (topJoin.getJoinType().isLeftSemiOrAntiJoin() + && (topJoin.left().child().getJoinType().isInnerJoin() + || topJoin.left().child().getJoinType().isLeftOuterJoin() + || topJoin.left().child().getJoinType().isRightOuterJoin()))) .whenNot(topJoin -> topJoin.left().child().getJoinType().isSemiOrAntiJoin()) .whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint()) .when(join -> JoinReorderCommon.checkProject(join.left())) @@ -80,6 +81,8 @@ public class SemiJoinLogicalJoinTransposeProject extends OneExplorationRuleFacto lasscom = ExpressionUtils.isIntersecting(usedSlot, aOutputSet) || lasscom; } + JoinType newTopJoinType = JoinType.INNER_JOIN; + if (lasscom) { /*- * topSemiJoin project @@ -94,8 +97,9 @@ public class SemiJoinLogicalJoinTransposeProject extends OneExplorationRuleFacto topSemiJoin.getJoinType(), topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), JoinHint.NONE, a, c); - LogicalJoin newTopJoin = new LogicalJoin<>(bottomJoin.getJoinType(), - bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), JoinHint.NONE, + LogicalJoin newTopJoin = new LogicalJoin<>(newTopJoinType, + bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), + JoinHint.NONE, newBottomSemiJoin, b); return new LogicalProject<>(new ArrayList<>(topSemiJoin.getOutput()), newTopJoin); @@ -113,8 +117,9 @@ public class SemiJoinLogicalJoinTransposeProject extends OneExplorationRuleFacto topSemiJoin.getJoinType(), topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), JoinHint.NONE, b, c); - LogicalJoin newTopJoin = new LogicalJoin<>(bottomJoin.getJoinType(), - bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), JoinHint.NONE, + LogicalJoin newTopJoin = new LogicalJoin<>(newTopJoinType, + bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), + JoinHint.NONE, a, newBottomSemiJoin); return new LogicalProject<>(new ArrayList<>(topSemiJoin.getOutput()), newTopJoin); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java index 4c66fd422c..232d5d3190 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java @@ -76,8 +76,8 @@ public class ApplyPullFilterOnProjectUnderAgg extends OneRewriteRuleFactory { LogicalProject newProject = new LogicalProject<>(newProjects, filter.child()); LogicalFilter newFilter = new LogicalFilter<>(filter.getConjuncts(), newProject); - LogicalAggregate newAgg = new LogicalAggregate<>( - agg.getGroupByExpressions(), agg.getOutputExpressions(), newFilter); + LogicalAggregate newAgg = new LogicalAggregate<>(agg.getGroupByExpressions(), + agg.getOutputExpressions(), agg.isOrdinalIsResolved(), newFilter); return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), apply.getCorrelationFilter(), apply.left(), newAgg); }).toRule(RuleType.APPLY_PULL_FILTER_ON_PROJECT_UNDER_AGG); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java index 35c51485ed..1d52871a77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java @@ -59,7 +59,7 @@ public class EliminateGroupByConstant extends OneRewriteRuleFactory { lit = expression; } } - if (slotGroupByExprs.isEmpty() && lit != null) { + if (slotGroupByExprs.isEmpty() && lit != null && aggregate.getAggregateFunctions().isEmpty()) { slotGroupByExprs.add(lit); } return aggregate.withGroupByAndOutput(ImmutableList.copyOf(slotGroupByExprs), outputExprs); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeToSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeToSlot.java index 34686534e3..d4a53b8ac0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeToSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeToSlot.java @@ -21,6 +21,9 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.types.TinyIntType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -133,7 +136,10 @@ public interface NormalizeToSlot { } Alias alias = new Alias(expression, expression.toSql()); - return new NormalizeToSlotTriplet(expression, alias.toSlot(), alias); + SlotReference slot = (SlotReference) alias.toSlot(); + // BE will create a nullable uint8 column to expand NullLiteral when necessary + return new NormalizeToSlotTriplet(expression, expression instanceof NullLiteral ? slot.withDataType( + TinyIntType.INSTANCE) : slot, alias); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ceil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ceil.java index f2423c6b20..7359aa2ca4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ceil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ceil.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSi import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DoubleType; import com.google.common.base.Preconditions; @@ -39,7 +38,7 @@ public class Ceil extends ScalarFunction implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, ComputePrecisionForRound { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BigIntType.INSTANCE).args(DoubleType.INSTANCE) + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Floor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Floor.java index 6d5a47259c..380bb33431 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Floor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Floor.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSi import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DoubleType; import com.google.common.base.Preconditions; @@ -39,7 +38,7 @@ public class Floor extends ScalarFunction implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, ComputePrecisionForRound { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BigIntType.INSTANCE).args(DoubleType.INSTANCE) + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 7f5fad5af6..812436218d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.trees.expressions.shape.LeafExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.StringType; import org.apache.commons.lang3.StringUtils; @@ -161,16 +160,15 @@ public abstract class Literal extends Expression implements LeafExpression, Comp return Float.compare((float) getValue(), (float) other.getValue()); } else if (type.isDoubleType()) { return Double.compare((double) getValue(), (double) other.getValue()); - } else if (type.isDecimalV2Type()) { - return Long.compare((Long) getValue(), (Long) other.getValue()); } else if (type.isDateLikeType()) { - // todo process date + return Long.compare((Long) getValue(), (Long) other.getValue()); } else if (type.isDecimalV2Type()) { return ((BigDecimal) getValue()).compareTo((BigDecimal) other.getValue()); - } else if (type instanceof StringType) { + } else if (type.isStringLikeType()) { return StringUtils.compare((String) getValue(), (String) other.getValue()); + } else { + throw new RuntimeException(String.format("Literal {} is not supported!", type.toString())); } - return -1; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 1557145392..8aeb5df235 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -60,6 +60,8 @@ public class LogicalAggregate extends LogicalUnary sourceRepeat; + private final boolean ordinalIsResolved; + /** * Desc: Constructor for LogicalAggregate. */ @@ -71,6 +73,12 @@ public class LogicalAggregate extends LogicalUnary groupByExpressions, + List outputExpressions, boolean ordinalIsResolved, CHILD_TYPE child) { + this(groupByExpressions, outputExpressions, false, ordinalIsResolved, Optional.empty(), + Optional.empty(), Optional.empty(), child); + } + /** * Desc: Constructor for LogicalAggregate. * Generated from LogicalRepeat. @@ -89,7 +97,7 @@ public class LogicalAggregate extends LogicalUnary sourceRepeat, CHILD_TYPE child) { - this(groupByExpressions, outputExpressions, normalized, sourceRepeat, + this(groupByExpressions, outputExpressions, normalized, false, sourceRepeat, Optional.empty(), Optional.empty(), child); } @@ -100,6 +108,7 @@ public class LogicalAggregate extends LogicalUnary groupByExpressions, List outputExpressions, boolean normalized, + boolean ordinalIsResolved, Optional sourceRepeat, Optional groupExpression, Optional logicalProperties, @@ -108,6 +117,7 @@ public class LogicalAggregate extends LogicalUnary extends LogicalUnary extends LogicalUnary withChildren(List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, - normalized, sourceRepeat, children.get(0)); + normalized, ordinalIsResolved, sourceRepeat, Optional.empty(), Optional.empty(), children.get(0)); } @Override public LogicalAggregate withGroupExpression(Optional groupExpression) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, - normalized, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + normalized, ordinalIsResolved, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), + children.get(0)); } @Override public LogicalAggregate withLogicalProperties(Optional logicalProperties) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, - normalized, sourceRepeat, + normalized, ordinalIsResolved, sourceRepeat, Optional.empty(), logicalProperties, children.get(0)); } public LogicalAggregate withGroupByAndOutput(List groupByExprList, List outputExpressionList) { - return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, sourceRepeat, child()); + return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, + sourceRepeat, Optional.empty(), Optional.empty(), child()); } @Override public LogicalAggregate withAggOutput(List newOutput) { - return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, + return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withNormalized(List normalizedGroupBy, List normalizedOutput, Plan normalizedChild) { - return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, + return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, ordinalIsResolved, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 2e3e39540a..c71616f610 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -42,6 +42,7 @@ import com.google.common.collect.Lists; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -151,11 +152,15 @@ public class JoinUtils { for (Expression expr : join.getHashJoinConjuncts()) { EqualTo equalTo = (EqualTo) expr; - if (!(equalTo.left() instanceof Slot) || !(equalTo.right() instanceof Slot)) { + // TODO: we could meet a = cast(b as xxx) here, need fix normalize join hash equals future + Optional leftSlot = ExpressionUtils.extractSlotOrCastOnSlot(equalTo.left()); + Optional rightSlot = ExpressionUtils.extractSlotOrCastOnSlot(equalTo.right()); + if (!leftSlot.isPresent() || !rightSlot.isPresent()) { continue; } - ExprId leftExprId = ((Slot) equalTo.left()).getExprId(); - ExprId rightExprId = ((Slot) equalTo.right()).getExprId(); + + ExprId leftExprId = leftSlot.get().getExprId(); + ExprId rightExprId = rightSlot.get().getExprId(); if (checker.isCoveredByLeftSlots(leftExprId) && checker.isCoveredByRightSlots(rightExprId)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 11584d74d7..796d852fc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -213,6 +213,10 @@ public class TypeCoercionUtils { } } else if (left instanceof CharacterType && right instanceof CharacterType) { tightestCommonType = CharacterType.widerCharacterType((CharacterType) left, (CharacterType) right); + } else if (left instanceof CharacterType && right instanceof DateLikeType + || left instanceof DateLikeType && right instanceof CharacterType) { + // TODO: need check implicitCastMap to keep the behavior consistent with old optimizer + tightestCommonType = right; } else if (left instanceof CharacterType || right instanceof CharacterType) { tightestCommonType = StringType.INSTANCE; } else if (left instanceof DecimalV2Type && right instanceof IntegralType) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java index 7c365f7e13..d3889bbdcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java @@ -501,6 +501,9 @@ public abstract class SetOperationNode extends PlanNode { for (int j = 0; j < exprList.size(); ++j) { if (resultExprSlots.get(j).isMaterialized()) { newExprList.add(exprList.get(j)); + // TODO: reconsider this, we may change nullable info in previous nereids rules not here. + resultExprSlots.get(j) + .setIsNullable(resultExprSlots.get(j).getIsNullable() || exprList.get(j).isNullable()); } } materializedResultExprLists.add(newExprList); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java index 7231689396..57bd28fd49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java @@ -320,7 +320,13 @@ public class SortNode extends PlanNode { */ public void finalizeForNereids(TupleDescriptor tupleDescriptor, List outputList, List orderingExpr) { - resolvedTupleExprs = Lists.newArrayList(orderingExpr); + resolvedTupleExprs = Lists.newArrayList(); + // TODO: should fix the duplicate order by exprs in nereids code later + for (Expr order : orderingExpr) { + if (!resolvedTupleExprs.contains(order)) { + resolvedTupleExprs.add(order); + } + } for (Expr output : outputList) { if (!resolvedTupleExprs.contains(output)) { resolvedTupleExprs.add(output); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java index 1b262b2e05..948a7c6ccc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java @@ -75,7 +75,9 @@ class OuterJoinLAsscomProjectTest implements PatternMatchSupported { .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .printlnOrigin() .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) + .printlnExploration() .matchesExploration( logicalJoin( logicalProject( @@ -103,20 +105,9 @@ class OuterJoinLAsscomProjectTest implements PatternMatchSupported { PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) .printlnOrigin() - .printlnExploration() - .matchesExploration( - logicalJoin( - logicalProject( - logicalJoin( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1")), - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) - ).when(join -> join.getHashJoinConjuncts().size() == 1) - ).when(project -> project.getProjects().size() == 3), // t1.id Add t3.id, t3.name - logicalProject( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2")) - ).when(project -> project.getProjects().size() == 1) - ).when(join -> join.getHashJoinConjuncts().size() == 2) - ); + .checkMemo(memo -> { + Assertions.assertEquals(1, memo.getRoot().getLogicalExpressions().size()); + }); } @Test @@ -156,22 +147,10 @@ class OuterJoinLAsscomProjectTest implements PatternMatchSupported { .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .printlnOrigin() .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) - .matchesExploration( - logicalJoin( - logicalProject( - logicalJoin( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1")), - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) - ).when(join -> join.getOtherJoinConjuncts().size() == 1 - && join.getHashJoinConjuncts().size() == 1) - ), - logicalProject( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2")) - ) - ).when(join -> join.getOtherJoinConjuncts().size() == 2 - && join.getHashJoinConjuncts().size() == 2) - ) - .printlnExploration(); + .checkMemo(memo -> { + Assertions.assertEquals(1, memo.getRoot().getLogicalExpressions().size()); + }); } } diff --git a/regression-test/data/nereids_syntax_p0/function.out b/regression-test/data/nereids_syntax_p0/function.out index 7f0be9b3f1..f41cd574a8 100644 --- a/regression-test/data/nereids_syntax_p0/function.out +++ b/regression-test/data/nereids_syntax_p0/function.out @@ -48,3 +48,8 @@ 8 9 +-- !floor -- +2 + +-- !ceil -- +3 diff --git a/regression-test/data/nereids_syntax_p0/group_by_constant.out b/regression-test/data/nereids_syntax_p0/group_by_constant.out index a659ddb5f5..719823620e 100644 --- a/regression-test/data/nereids_syntax_p0/group_by_constant.out +++ b/regression-test/data/nereids_syntax_p0/group_by_constant.out @@ -26,3 +26,20 @@ str 24 1310179 10 1 -- !sql -- 2 +-- !sql -- +15 + +-- !sql -- +5 + +-- !sql -- +1310179 +1309892 +1309892 + +-- !sql -- +\N + +-- !sql -- +2 + diff --git a/regression-test/data/nereids_syntax_p0/join_order.out b/regression-test/data/nereids_syntax_p0/join_order.out new file mode 100644 index 0000000000..095c7b2035 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/join_order.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 + diff --git a/regression-test/data/nereids_syntax_p0/set_operation.out b/regression-test/data/nereids_syntax_p0/set_operation.out index 39a0740e6c..91b5c554a8 100644 --- a/regression-test/data/nereids_syntax_p0/set_operation.out +++ b/regression-test/data/nereids_syntax_p0/set_operation.out @@ -581,3 +581,7 @@ hell0 3 d 3 d +-- !union43 -- +2020-05-25 +2020-05-25 + diff --git a/regression-test/data/nereids_syntax_p0/type_cast.out b/regression-test/data/nereids_syntax_p0/type_cast.out new file mode 100644 index 0000000000..0970350d5f --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/type_cast.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 + +-- !sql -- +1 + diff --git a/regression-test/data/query_p0/keyword/order_group.out b/regression-test/data/query_p0/keyword/order_group.out index 5a46327a8b..ff7794c0f2 100644 --- a/regression-test/data/query_p0/keyword/order_group.out +++ b/regression-test/data/query_p0/keyword/order_group.out @@ -1430,7 +1430,6 @@ true 0.1 \N \N -- !orderBy_withNull_3 -- -\N \N \N 15 \N 14 \N 13 @@ -1446,4 +1445,5 @@ true 0.1 \N 3 \N 2 \N 1 +\N \N diff --git a/regression-test/suites/nereids_syntax_p0/function.groovy b/regression-test/suites/nereids_syntax_p0/function.groovy index 761fcd04a1..48d8ab4f2b 100644 --- a/regression-test/suites/nereids_syntax_p0/function.groovy +++ b/regression-test/suites/nereids_syntax_p0/function.groovy @@ -112,5 +112,13 @@ suite("nereids_function") { sql """select "1" == "123", "%%" == "%%" """ result([[false, true]]) } + + qt_floor """ + SELECT floor(2.1); + """ + + qt_ceil """ + SELECT ceil(2.1); + """ } diff --git a/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy b/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy index 6097aa3d41..a8da654096 100644 --- a/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy +++ b/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy @@ -36,4 +36,23 @@ suite("group_by_constant") { qt_sql """select 2 from lineorder group by 1""" + qt_sql """select SUM(lo_tax) FROM lineorder group by null;""" + + qt_sql """select 5 FROM lineorder group by null;""" + + qt_sql """select lo_orderkey from lineorder order by lo_tax desc, lo_tax desc;""" + + test { + sql "select SUM(lo_tax) FROM lineorder group by 1;" + exception "GROUP BY expression must not contain aggregate functions: sum(lo_tax)" + } + + test { + sql "select SUM(lo_tax) FROM lineorder group by SUM(lo_tax);" + exception "GROUP BY expression must not contain aggregate functions: sum(lo_tax)" + } + + qt_sql """select SUM(if(lo_tax=1,lo_tax,0)) FROM lineorder where false;""" + + qt_sql """select 2 FROM lineorder group by 1;""" } diff --git a/regression-test/suites/nereids_syntax_p0/join.groovy b/regression-test/suites/nereids_syntax_p0/join.groovy index e91d6b06d8..52f56a3d96 100644 --- a/regression-test/suites/nereids_syntax_p0/join.groovy +++ b/regression-test/suites/nereids_syntax_p0/join.groovy @@ -22,6 +22,60 @@ suite("join") { sql "SET enable_fallback_to_original_planner=false" + sql """ + drop table if exists test_table_a; + """ + + sql """ + drop table if exists test_table_b; + """ + + sql """ + CREATE TABLE `test_table_a` + ( + `wtid` varchar(30) NOT NULL , + `wfid` varchar(30) NOT NULL + ) ENGINE=OLAP + UNIQUE KEY(`wtid`,`wfid`) + DISTRIBUTED BY HASH(`wfid`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + CREATE TABLE `test_table_b` + ( + `wtid` varchar(100) NOT NULL , + `wfid` varchar(100) NOT NULL + ) ENGINE=OLAP + UNIQUE KEY(`wtid`,`wfid`) + DISTRIBUTED BY HASH(`wfid`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2" + ); + """ + + sql """ + insert into test_table_b values( '1', '1'),('2','2'),('3','3'),('1','2'),('1','3'),('2','3'); + """ + + sql """ + insert into test_table_a values( '1', '1'),('2','2'),('3','3'),('1','2'),('1','3'),('2','3'); + """ + + // must analyze before explain, because empty table may generate different plan + sql """ + analyze table test_table_b; + """ + + sql """ + analyze table test_table_a; + """ + order_qt_inner_join_1 """ SELECT * FROM lineorder JOIN supplier ON lineorder.lo_suppkey = supplier.s_suppkey """ @@ -73,5 +127,106 @@ suite("join") { order_qt_outer_join_with_filter """ select lo_orderkey, lo_partkey, p_partkey, p_size from lineorder inner join part on lo_partkey = p_partkey where lo_orderkey - 1310000 > p_size; """ + + sql """ + drop table if exists outerjoin_A; + """ + + sql """ + drop table if exists outerjoin_B; + """ + + sql """ + drop table if exists outerjoin_C; + """ + + sql """ + drop table if exists outerjoin_D; + """ + + sql """ + create table if not exists outerjoin_A ( a int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + create table if not exists outerjoin_B ( a int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + create table if not exists outerjoin_C ( a int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + create table if not exists outerjoin_D ( a int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + insert into outerjoin_A values( 1 ); + """ + + sql """ + insert into outerjoin_B values( 1 ); + """ + + sql """ + insert into outerjoin_C values( 1 ); + """ + + sql """ + insert into outerjoin_D values( 1 ); + """ + + explain { + sql("select count(*) from outerjoin_A A left join outerjoin_B B on A.a = B.a where B.a in (select a from outerjoin_C);") + contains "INNER JOIN" + } + + explain { + sql("""SELECT count(1) + FROM + (SELECT sub1.wtid, + count(*) + FROM + (SELECT a.wtid , + a.wfid + FROM test_table_b a ) sub1 + INNER JOIN [shuffle] + (SELECT a.wtid, + a.wfid + FROM test_table_a a ) sub2 + ON sub1.wtid = sub2.wtid + AND sub1.wfid = sub2.wfid + GROUP BY sub1.wtid ) qqqq;""") + contains "4:VAGGREGATE (update serialize)" + contains "6:VAGGREGATE (merge finalize)" + } } diff --git a/regression-test/suites/nereids_syntax_p0/join_order.groovy b/regression-test/suites/nereids_syntax_p0/join_order.groovy new file mode 100644 index 0000000000..df838ace74 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/join_order.groovy @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("join_order") { + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + + sql """ drop table if exists outerjoin_A;""" + sql """ + create table outerjoin_A ( a1 bigint not null, a2 bigint not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a1) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + sql """ drop table if exists outerjoin_B;""" + sql """ + create table outerjoin_B ( b int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(b) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + sql """ drop table if exists outerjoin_C;""" + sql """ + create table outerjoin_C ( c int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(c) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + sql """ drop table if exists outerjoin_D;""" + sql """ + create table outerjoin_D ( d1 int not null, d2 int not null, d3 int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(d1) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + sql """ drop table if exists outerjoin_E;""" + sql """ + create table outerjoin_E ( e1 int not null, e2 int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(e1) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """insert into outerjoin_A values( 1,2 );""" + sql """insert into outerjoin_B values( 1 );""" + sql """insert into outerjoin_C values( 1 );""" + sql """insert into outerjoin_D values( 1,2,3 );""" + sql """insert into outerjoin_E values( 1,2 );""" + + qt_sql"""SELECT count(*) + FROM outerjoin_A t1 + LEFT JOIN outerjoin_D dcbc + ON t1.a1 = dcbc.d1 + LEFT JOIN outerjoin_C dcso + ON dcbc.d2 = dcso.c + LEFT JOIN outerjoin_B dcii + ON t1.a2 = dcii.b + LEFT JOIN outerjoin_E dcssm + ON dcii.b = dcssm.e1 + AND dcbc.d3 = dcssm.e2; + """ +} diff --git a/regression-test/suites/nereids_syntax_p0/set_operation.groovy b/regression-test/suites/nereids_syntax_p0/set_operation.groovy index 3f3db650fb..d8820e95b5 100644 --- a/regression-test/suites/nereids_syntax_p0/set_operation.groovy +++ b/regression-test/suites/nereids_syntax_p0/set_operation.groovy @@ -54,6 +54,20 @@ suite("test_nereids_set_operation") { ); """ + sql """ drop table if exists test_table;""" + sql """ + CREATE TABLE `test_table` + ( + `day` date + ) ENGINE = OLAP DUPLICATE KEY(`day`) + DISTRIBUTED BY HASH(`day`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """insert into test_table values('2020-05-25');""" + sql """ INSERT INTO setOperationTable VALUES (1, 1, 1, 3, 'a', 'b'), @@ -262,5 +276,5 @@ suite("test_nereids_set_operation") { (select k1, k5 from setOperationTable) """ - + qt_union43 """select '2020-05-25' day from test_table union all select day from test_table;""" } diff --git a/regression-test/suites/nereids_syntax_p0/type_cast.groovy b/regression-test/suites/nereids_syntax_p0/type_cast.groovy new file mode 100644 index 0000000000..9bbc2724ef --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/type_cast.groovy @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("type_cast") { + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + + sql """ drop table if exists test_table2;""" + sql """ + CREATE TABLE `test_table2` + ( + `day` date + ) ENGINE = OLAP DUPLICATE KEY(`day`) + DISTRIBUTED BY HASH(`day`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """insert into test_table2 values('2020-05-25');""" + + def ret = sql"""explain verbose select * from test_table2 where day > CONVERT_tz('2020-05-25 00:00:00', 'Asia/Shanghai', 'Asia/Shanghai');""" + assertTrue(ret.toString().contains("CAST(day[#0] AS DATETIME)")) + + qt_sql """select count(*) from test_table2 where 'a' = 'a';""" + qt_sql """select count(*) from test_table2 where cast('2020-01-01' as date) = cast('2020-01-01' as date);""" +}