diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index d563e18eb7..d84720ee9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -361,9 +361,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor output = emptyRelation.getOutput(); TupleDescriptor tupleDescriptor = generateTupleDesc(output, null, context); for (int i = 0; i < output.size(); i++) { - SlotDescriptor slotDescriptor = tupleDescriptor.getSlots().get(i); - slotDescriptor.setIsNullable(true); // we should set to nullable, or else BE would core - Slot slot = output.get(i); SlotRef slotRef = context.findSlotRef(slot.getExprId()); slotRef.setLabel(slot.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java index 17f0e72e7a..275901dff7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java @@ -86,6 +86,7 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob { .add(topDownBatch(ImmutableList.of(new ExpressionNormalization(cascadesContext.getConnectContext())))) .add(topDownBatch(ImmutableList.of(new ExpressionOptimization()))) .add(topDownBatch(ImmutableList.of(new ExtractSingleTableExpressionFromDisjunction()))) + .add(topDownBatch(ImmutableList.of(new EliminateGroupByConstant()))) .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) .add(visitorJob(RuleType.INFER_PREDICATES, new InferPredicates())) @@ -107,7 +108,6 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob { .add(topDownBatch(ImmutableList.of(new CountDistinctRewrite()))) // we need to execute this rule at the end of rewrite // to avoid two consecutive same project appear when we do optimization. - .add(topDownBatch(ImmutableList.of(new EliminateGroupByConstant()))) .add(topDownBatch(ImmutableList.of(new EliminateOrderByConstant()))) .add(topDownBatch(ImmutableList.of(new EliminateUnnecessaryProject()))) .add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithAggregate()))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 951fbc7cc8..56260c999b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1108,7 +1108,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { public OrderKey visitSortItem(SortItemContext ctx) { return ParserUtils.withOrigin(ctx, () -> { boolean isAsc = ctx.DESC() == null; - boolean isNullFirst = ctx.LAST() == null; + boolean isNullFirst = ctx.FIRST() != null || (ctx.LAST() == null && isAsc); Expression expression = typedVisit(ctx.expression()); return new OrderKey(expression, isAsc, isNullFirst); }); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java index 269dc94be1..090f523b18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.processor.post; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TRuntimeFilterMode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -57,7 +58,9 @@ public class PlanPostProcessors { public List getProcessors() { // add processor if we need Builder builder = ImmutableList.builder(); - if (cascadesContext.getConnectContext().getSessionVariable().isEnableNereidsRuntimeFilter()) { + if (cascadesContext.getConnectContext().getSessionVariable().isEnableNereidsRuntimeFilter() + && !cascadesContext.getConnectContext().getSessionVariable().getRuntimeFilterMode() + .toUpperCase().equals(TRuntimeFilterMode.OFF.name())) { builder.add(new RuntimeFilterGenerator()); if (ConnectContext.get().getSessionVariable().enableRuntimeFilterPrune) { builder.add(new RuntimeFilterPruner()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java index a4e14bfc80..8d45087b84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java @@ -104,5 +104,12 @@ public class CheckAnalysis implements AnalysisRuleFactory { throw new AnalysisException( "The query contains multi count distinct or sum distinct, each can't have multi columns"); } + Optional expr = aggregate.getGroupByExpressions().stream() + .filter(expression -> expression.containsType(AggregateFunction.class)).findFirst(); + if (expr.isPresent()) { + throw new AnalysisException( + "GROUP BY expression must not contain aggregate functions: " + + expr.get().toSql()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java index b04f085cef..5ee1a8fad6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -63,7 +64,7 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { }) )) .add(RuleType.RESOLVE_ORDINAL_IN_GROUP_BY.build( - logicalAggregate().then(agg -> { + logicalAggregate().whenNot(agg -> agg.isOrdinalIsResolved()).then(agg -> { List aggOutput = agg.getOutputExpressions(); List groupByWithoutOrd = new ArrayList<>(); boolean ordExists = false; @@ -74,6 +75,9 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { int ord = i.getIntValue(); checkOrd(ord, aggOutput.size()); Expression aggExpr = aggOutput.get(ord - 1); + if (aggExpr instanceof Alias) { + aggExpr = ((Alias) aggExpr).child(); + } groupByWithoutOrd.add(aggExpr); ordExists = true; } else { @@ -81,11 +85,11 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { } } if (ordExists) { - return new LogicalAggregate(groupByWithoutOrd, agg.getOutputExpressions(), agg.child()); + return new LogicalAggregate(groupByWithoutOrd, agg.getOutputExpressions(), true, + agg.child()); } else { return agg; } - }))).build(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java index a356a71dea..d9ed3ffe04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; import org.apache.doris.nereids.util.PlanUtils; @@ -75,26 +76,25 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { GroupPlan a = bottomJoin.left(); GroupPlan b = bottomJoin.right(); GroupPlan c = topJoin.right(); - Set bOutputSet = b.getOutputSet(); + Set aOutputSet = a.getOutputSet(); Set cOutputSet = c.getOutputSet(); /* ********** Split projects ********** */ Map> projectExprsMap = projects.stream() .collect(Collectors.partitioningBy(projectExpr -> { Set usedSlots = projectExpr.collect(SlotReference.class::isInstance); - return bOutputSet.containsAll(usedSlots); + return aOutputSet.containsAll(usedSlots); })); - List newLeftProjects = projectExprsMap.get(Boolean.FALSE); - List newRightProjects = projectExprsMap.get(Boolean.TRUE); - - Set bExprIdSet = InnerJoinLAsscomProject.getExprIdSetForB(bottomJoin.right(), - newRightProjects); + List newLeftProjects = projectExprsMap.get(Boolean.TRUE); + List newRightProjects = projectExprsMap.get(Boolean.FALSE); + Set aExprIdSet = getExprIdSetForA(bottomJoin.left(), + newLeftProjects); /* ********** split Conjuncts ********** */ - Map> splitHashJoinConjuncts - = InnerJoinLAsscomProject.splitConjunctsWithAlias( - topJoin.getHashJoinConjuncts(), bottomJoin.getHashJoinConjuncts(), bExprIdSet); - List newTopHashJoinConjuncts = splitHashJoinConjuncts.get(true); + Map> newHashJoinConjuncts + = createNewConjunctsWithAlias( + topJoin.getHashJoinConjuncts(), bottomJoin.getHashJoinConjuncts(), aExprIdSet); + List newTopHashJoinConjuncts = newHashJoinConjuncts.get(true); Preconditions.checkState(!newTopHashJoinConjuncts.isEmpty(), "LAsscom newTopHashJoinConjuncts join can't empty"); // When newTopHashJoinConjuncts.size() != bottomJoin.getHashJoinConjuncts().size() @@ -103,25 +103,21 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { && newTopHashJoinConjuncts.size() != bottomJoin.getHashJoinConjuncts().size()) { return null; } - List newBottomHashJoinConjuncts = splitHashJoinConjuncts.get(false); + List newBottomHashJoinConjuncts = newHashJoinConjuncts.get(false); if (newBottomHashJoinConjuncts.size() == 0) { return null; } - Map> splitOtherJoinConjuncts - = InnerJoinLAsscomProject.splitConjunctsWithAlias( + Map> newOtherJoinConjuncts + = createNewConjunctsWithAlias( topJoin.getOtherJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), - bExprIdSet); - List newTopOtherJoinConjuncts = splitOtherJoinConjuncts.get(true); - // When topJoin type differ from bottomJoin type (LOJ-inner or inner LOJ), - // we just can exchange topJoin and bottomJoin. like: - // Failed in: (A LOJ B on A.id = B.id) join C on c.id = A.id & c.id = B.id (top contain c.id = B.id) - // If type is same like LOJ(LOJ()), we can LAsscom. - if (topJoin.getJoinType() != bottomJoin.getJoinType() - && newTopOtherJoinConjuncts.size() != bottomJoin.getOtherJoinConjuncts().size()) { + aExprIdSet); + List newTopOtherJoinConjuncts = newOtherJoinConjuncts.get(true); + List newBottomOtherJoinConjuncts = newOtherJoinConjuncts.get(false); + if (newBottomOtherJoinConjuncts.size() != topJoin.getOtherJoinConjuncts().size() + || newTopOtherJoinConjuncts.size() != bottomJoin.getOtherJoinConjuncts().size()) { return null; } - List newBottomOtherJoinConjuncts = splitOtherJoinConjuncts.get(false); /* ********** replace Conjuncts by projects ********** */ Map inputToOutput = new HashMap<>(); @@ -158,10 +154,10 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { return usedSlotRefs.stream(); }) .filter(slot -> !cOutputSet.contains(slot)) - .collect(Collectors.partitioningBy(slot -> bExprIdSet.contains(slot.getExprId()), + .collect(Collectors.partitioningBy(slot -> aExprIdSet.contains(slot.getExprId()), Collectors.toSet())); - Set aUsedSlots = abOnUsedSlots.get(Boolean.FALSE); - Set bUsedSlots = abOnUsedSlots.get(Boolean.TRUE); + Set aUsedSlots = abOnUsedSlots.get(Boolean.TRUE); + Set bUsedSlots = abOnUsedSlots.get(Boolean.FALSE); JoinUtils.addSlotsUsedByOn(bUsedSlots, newRightProjects); JoinUtils.addSlotsUsedByOn(aUsedSlots, newLeftProjects); @@ -200,4 +196,35 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { return PlanUtils.project(new ArrayList<>(topJoin.getOutput()), newTopJoin).get(); }).toRule(RuleType.LOGICAL_OUTER_JOIN_LASSCOM_PROJECT); } + + private Map> createNewConjunctsWithAlias(List topConjuncts, + List bottomConjuncts, Set bExprIdSet) { + // if top join's conjuncts are all related to A, we can do reorder + Map> splitOn = new HashMap<>(); + splitOn.put(true, new ArrayList<>()); + if (topConjuncts.stream().allMatch(topHashOn -> { + Set usedSlots = topHashOn.getInputSlots(); + Set usedSlotsId = usedSlots.stream().map(NamedExpression::getExprId) + .collect(Collectors.toSet()); + + return ExpressionUtils.isIntersecting(bExprIdSet, usedSlotsId); + })) { + // do reorder, create new bottom join conjuncts + splitOn.put(false, new ArrayList<>(topConjuncts)); + } else { + // can't reorder, return empty list + splitOn.put(false, new ArrayList<>()); + } + + List newTopHashJoinConjuncts = splitOn.get(true); + newTopHashJoinConjuncts.addAll(bottomConjuncts); + + return splitOn; + } + + private Set getExprIdSetForA(GroupPlan a, List aProject) { + return Stream.concat( + a.getOutput().stream().map(NamedExpression::getExprId), + aProject.stream().map(NamedExpression::getExprId)).collect(Collectors.toSet()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java index 0f4c57a09f..c601b8720d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java @@ -54,9 +54,10 @@ public class SemiJoinLogicalJoinTranspose extends OneExplorationRuleFactory { @Override public Rule build() { return logicalJoin(logicalJoin(), group()) - .when(topJoin -> topJoin.getJoinType() == JoinType.LEFT_SEMI_JOIN - || topJoin.getJoinType() == JoinType.LEFT_ANTI_JOIN - || topJoin.getJoinType() == JoinType.NULL_AWARE_LEFT_ANTI_JOIN) + .when(topJoin -> (topJoin.getJoinType().isLeftSemiOrAntiJoin() + && (topJoin.left().getJoinType().isInnerJoin() + || topJoin.left().getJoinType().isLeftOuterJoin() + || topJoin.left().getJoinType().isRightOuterJoin()))) .whenNot(topJoin -> topJoin.left().getJoinType().isSemiOrAntiJoin()) .when(this::conditionChecker) .whenNot(topJoin -> topJoin.hasJoinHint() || topJoin.left().hasJoinHint()) @@ -75,6 +76,8 @@ public class SemiJoinLogicalJoinTranspose extends OneExplorationRuleFactory { lasscom = ExpressionUtils.isIntersecting(usedSlot, aOutputSet) || lasscom; } + JoinType newTopJoinType = JoinType.INNER_JOIN; + if (lasscom) { /* * topSemiJoin newTopJoin @@ -85,9 +88,10 @@ public class SemiJoinLogicalJoinTranspose extends OneExplorationRuleFactory { */ LogicalJoin newBottomSemiJoin = new LogicalJoin<>( topSemiJoin.getJoinType(), - topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), JoinHint.NONE, + topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), + JoinHint.NONE, a, c); - return new LogicalJoin<>(bottomJoin.getJoinType(), bottomJoin.getHashJoinConjuncts(), + return new LogicalJoin<>(newTopJoinType, bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), JoinHint.NONE, newBottomSemiJoin, b); } else { /* @@ -102,7 +106,7 @@ public class SemiJoinLogicalJoinTranspose extends OneExplorationRuleFactory { topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), JoinHint.NONE, b, c); - return new LogicalJoin<>(bottomJoin.getJoinType(), bottomJoin.getHashJoinConjuncts(), + return new LogicalJoin<>(newTopJoinType, bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), JoinHint.NONE, a, newBottomSemiJoin); } }).toRule(RuleType.LOGICAL_SEMI_JOIN_LOGICAL_JOIN_TRANSPOSE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java index b517c47f3c..20e2176f00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java @@ -56,9 +56,10 @@ public class SemiJoinLogicalJoinTransposeProject extends OneExplorationRuleFacto @Override public Rule build() { return logicalJoin(logicalProject(logicalJoin()), group()) - .when(topJoin -> topJoin.getJoinType() == JoinType.LEFT_SEMI_JOIN - || topJoin.getJoinType() == JoinType.LEFT_ANTI_JOIN - || topJoin.getJoinType() == JoinType.NULL_AWARE_LEFT_ANTI_JOIN) + .when(topJoin -> (topJoin.getJoinType().isLeftSemiOrAntiJoin() + && (topJoin.left().child().getJoinType().isInnerJoin() + || topJoin.left().child().getJoinType().isLeftOuterJoin() + || topJoin.left().child().getJoinType().isRightOuterJoin()))) .whenNot(topJoin -> topJoin.left().child().getJoinType().isSemiOrAntiJoin()) .whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint()) .when(join -> JoinReorderCommon.checkProject(join.left())) @@ -80,6 +81,8 @@ public class SemiJoinLogicalJoinTransposeProject extends OneExplorationRuleFacto lasscom = ExpressionUtils.isIntersecting(usedSlot, aOutputSet) || lasscom; } + JoinType newTopJoinType = JoinType.INNER_JOIN; + if (lasscom) { /*- * topSemiJoin project @@ -94,8 +97,9 @@ public class SemiJoinLogicalJoinTransposeProject extends OneExplorationRuleFacto topSemiJoin.getJoinType(), topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), JoinHint.NONE, a, c); - LogicalJoin newTopJoin = new LogicalJoin<>(bottomJoin.getJoinType(), - bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), JoinHint.NONE, + LogicalJoin newTopJoin = new LogicalJoin<>(newTopJoinType, + bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), + JoinHint.NONE, newBottomSemiJoin, b); return new LogicalProject<>(new ArrayList<>(topSemiJoin.getOutput()), newTopJoin); @@ -113,8 +117,9 @@ public class SemiJoinLogicalJoinTransposeProject extends OneExplorationRuleFacto topSemiJoin.getJoinType(), topSemiJoin.getHashJoinConjuncts(), topSemiJoin.getOtherJoinConjuncts(), JoinHint.NONE, b, c); - LogicalJoin newTopJoin = new LogicalJoin<>(bottomJoin.getJoinType(), - bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), JoinHint.NONE, + LogicalJoin newTopJoin = new LogicalJoin<>(newTopJoinType, + bottomJoin.getHashJoinConjuncts(), bottomJoin.getOtherJoinConjuncts(), + JoinHint.NONE, a, newBottomSemiJoin); return new LogicalProject<>(new ArrayList<>(topSemiJoin.getOutput()), newTopJoin); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java index 4c66fd422c..232d5d3190 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java @@ -76,8 +76,8 @@ public class ApplyPullFilterOnProjectUnderAgg extends OneRewriteRuleFactory { LogicalProject newProject = new LogicalProject<>(newProjects, filter.child()); LogicalFilter newFilter = new LogicalFilter<>(filter.getConjuncts(), newProject); - LogicalAggregate newAgg = new LogicalAggregate<>( - agg.getGroupByExpressions(), agg.getOutputExpressions(), newFilter); + LogicalAggregate newAgg = new LogicalAggregate<>(agg.getGroupByExpressions(), + agg.getOutputExpressions(), agg.isOrdinalIsResolved(), newFilter); return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), apply.getCorrelationFilter(), apply.left(), newAgg); }).toRule(RuleType.APPLY_PULL_FILTER_ON_PROJECT_UNDER_AGG); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java index 35c51485ed..1d52871a77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java @@ -59,7 +59,7 @@ public class EliminateGroupByConstant extends OneRewriteRuleFactory { lit = expression; } } - if (slotGroupByExprs.isEmpty() && lit != null) { + if (slotGroupByExprs.isEmpty() && lit != null && aggregate.getAggregateFunctions().isEmpty()) { slotGroupByExprs.add(lit); } return aggregate.withGroupByAndOutput(ImmutableList.copyOf(slotGroupByExprs), outputExprs); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeToSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeToSlot.java index 34686534e3..d4a53b8ac0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeToSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeToSlot.java @@ -21,6 +21,9 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.types.TinyIntType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -133,7 +136,10 @@ public interface NormalizeToSlot { } Alias alias = new Alias(expression, expression.toSql()); - return new NormalizeToSlotTriplet(expression, alias.toSlot(), alias); + SlotReference slot = (SlotReference) alias.toSlot(); + // BE will create a nullable uint8 column to expand NullLiteral when necessary + return new NormalizeToSlotTriplet(expression, expression instanceof NullLiteral ? slot.withDataType( + TinyIntType.INSTANCE) : slot, alias); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ceil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ceil.java index f2423c6b20..7359aa2ca4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ceil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ceil.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSi import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DoubleType; import com.google.common.base.Preconditions; @@ -39,7 +38,7 @@ public class Ceil extends ScalarFunction implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, ComputePrecisionForRound { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BigIntType.INSTANCE).args(DoubleType.INSTANCE) + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Floor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Floor.java index 6d5a47259c..380bb33431 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Floor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Floor.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSi import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DoubleType; import com.google.common.base.Preconditions; @@ -39,7 +38,7 @@ public class Floor extends ScalarFunction implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, ComputePrecisionForRound { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BigIntType.INSTANCE).args(DoubleType.INSTANCE) + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 7f5fad5af6..812436218d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.trees.expressions.shape.LeafExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.StringType; import org.apache.commons.lang3.StringUtils; @@ -161,16 +160,15 @@ public abstract class Literal extends Expression implements LeafExpression, Comp return Float.compare((float) getValue(), (float) other.getValue()); } else if (type.isDoubleType()) { return Double.compare((double) getValue(), (double) other.getValue()); - } else if (type.isDecimalV2Type()) { - return Long.compare((Long) getValue(), (Long) other.getValue()); } else if (type.isDateLikeType()) { - // todo process date + return Long.compare((Long) getValue(), (Long) other.getValue()); } else if (type.isDecimalV2Type()) { return ((BigDecimal) getValue()).compareTo((BigDecimal) other.getValue()); - } else if (type instanceof StringType) { + } else if (type.isStringLikeType()) { return StringUtils.compare((String) getValue(), (String) other.getValue()); + } else { + throw new RuntimeException(String.format("Literal {} is not supported!", type.toString())); } - return -1; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 1557145392..8aeb5df235 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -60,6 +60,8 @@ public class LogicalAggregate extends LogicalUnary sourceRepeat; + private final boolean ordinalIsResolved; + /** * Desc: Constructor for LogicalAggregate. */ @@ -71,6 +73,12 @@ public class LogicalAggregate extends LogicalUnary groupByExpressions, + List outputExpressions, boolean ordinalIsResolved, CHILD_TYPE child) { + this(groupByExpressions, outputExpressions, false, ordinalIsResolved, Optional.empty(), + Optional.empty(), Optional.empty(), child); + } + /** * Desc: Constructor for LogicalAggregate. * Generated from LogicalRepeat. @@ -89,7 +97,7 @@ public class LogicalAggregate extends LogicalUnary sourceRepeat, CHILD_TYPE child) { - this(groupByExpressions, outputExpressions, normalized, sourceRepeat, + this(groupByExpressions, outputExpressions, normalized, false, sourceRepeat, Optional.empty(), Optional.empty(), child); } @@ -100,6 +108,7 @@ public class LogicalAggregate extends LogicalUnary groupByExpressions, List outputExpressions, boolean normalized, + boolean ordinalIsResolved, Optional sourceRepeat, Optional groupExpression, Optional logicalProperties, @@ -108,6 +117,7 @@ public class LogicalAggregate extends LogicalUnary extends LogicalUnary extends LogicalUnary withChildren(List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, - normalized, sourceRepeat, children.get(0)); + normalized, ordinalIsResolved, sourceRepeat, Optional.empty(), Optional.empty(), children.get(0)); } @Override public LogicalAggregate withGroupExpression(Optional groupExpression) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, - normalized, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + normalized, ordinalIsResolved, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), + children.get(0)); } @Override public LogicalAggregate withLogicalProperties(Optional logicalProperties) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, - normalized, sourceRepeat, + normalized, ordinalIsResolved, sourceRepeat, Optional.empty(), logicalProperties, children.get(0)); } public LogicalAggregate withGroupByAndOutput(List groupByExprList, List outputExpressionList) { - return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, sourceRepeat, child()); + return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, + sourceRepeat, Optional.empty(), Optional.empty(), child()); } @Override public LogicalAggregate withAggOutput(List newOutput) { - return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, + return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withNormalized(List normalizedGroupBy, List normalizedOutput, Plan normalizedChild) { - return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, + return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, ordinalIsResolved, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 2e3e39540a..c71616f610 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -42,6 +42,7 @@ import com.google.common.collect.Lists; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -151,11 +152,15 @@ public class JoinUtils { for (Expression expr : join.getHashJoinConjuncts()) { EqualTo equalTo = (EqualTo) expr; - if (!(equalTo.left() instanceof Slot) || !(equalTo.right() instanceof Slot)) { + // TODO: we could meet a = cast(b as xxx) here, need fix normalize join hash equals future + Optional leftSlot = ExpressionUtils.extractSlotOrCastOnSlot(equalTo.left()); + Optional rightSlot = ExpressionUtils.extractSlotOrCastOnSlot(equalTo.right()); + if (!leftSlot.isPresent() || !rightSlot.isPresent()) { continue; } - ExprId leftExprId = ((Slot) equalTo.left()).getExprId(); - ExprId rightExprId = ((Slot) equalTo.right()).getExprId(); + + ExprId leftExprId = leftSlot.get().getExprId(); + ExprId rightExprId = rightSlot.get().getExprId(); if (checker.isCoveredByLeftSlots(leftExprId) && checker.isCoveredByRightSlots(rightExprId)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 11584d74d7..796d852fc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -213,6 +213,10 @@ public class TypeCoercionUtils { } } else if (left instanceof CharacterType && right instanceof CharacterType) { tightestCommonType = CharacterType.widerCharacterType((CharacterType) left, (CharacterType) right); + } else if (left instanceof CharacterType && right instanceof DateLikeType + || left instanceof DateLikeType && right instanceof CharacterType) { + // TODO: need check implicitCastMap to keep the behavior consistent with old optimizer + tightestCommonType = right; } else if (left instanceof CharacterType || right instanceof CharacterType) { tightestCommonType = StringType.INSTANCE; } else if (left instanceof DecimalV2Type && right instanceof IntegralType) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java index 7c365f7e13..d3889bbdcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java @@ -501,6 +501,9 @@ public abstract class SetOperationNode extends PlanNode { for (int j = 0; j < exprList.size(); ++j) { if (resultExprSlots.get(j).isMaterialized()) { newExprList.add(exprList.get(j)); + // TODO: reconsider this, we may change nullable info in previous nereids rules not here. + resultExprSlots.get(j) + .setIsNullable(resultExprSlots.get(j).getIsNullable() || exprList.get(j).isNullable()); } } materializedResultExprLists.add(newExprList); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java index 7231689396..57bd28fd49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java @@ -320,7 +320,13 @@ public class SortNode extends PlanNode { */ public void finalizeForNereids(TupleDescriptor tupleDescriptor, List outputList, List orderingExpr) { - resolvedTupleExprs = Lists.newArrayList(orderingExpr); + resolvedTupleExprs = Lists.newArrayList(); + // TODO: should fix the duplicate order by exprs in nereids code later + for (Expr order : orderingExpr) { + if (!resolvedTupleExprs.contains(order)) { + resolvedTupleExprs.add(order); + } + } for (Expr output : outputList) { if (!resolvedTupleExprs.contains(output)) { resolvedTupleExprs.add(output); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java index 1b262b2e05..948a7c6ccc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java @@ -75,7 +75,9 @@ class OuterJoinLAsscomProjectTest implements PatternMatchSupported { .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .printlnOrigin() .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) + .printlnExploration() .matchesExploration( logicalJoin( logicalProject( @@ -103,20 +105,9 @@ class OuterJoinLAsscomProjectTest implements PatternMatchSupported { PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) .printlnOrigin() - .printlnExploration() - .matchesExploration( - logicalJoin( - logicalProject( - logicalJoin( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1")), - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) - ).when(join -> join.getHashJoinConjuncts().size() == 1) - ).when(project -> project.getProjects().size() == 3), // t1.id Add t3.id, t3.name - logicalProject( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2")) - ).when(project -> project.getProjects().size() == 1) - ).when(join -> join.getHashJoinConjuncts().size() == 2) - ); + .checkMemo(memo -> { + Assertions.assertEquals(1, memo.getRoot().getLogicalExpressions().size()); + }); } @Test @@ -156,22 +147,10 @@ class OuterJoinLAsscomProjectTest implements PatternMatchSupported { .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .printlnOrigin() .applyExploration(OuterJoinLAsscomProject.INSTANCE.build()) - .matchesExploration( - logicalJoin( - logicalProject( - logicalJoin( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t1")), - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t3")) - ).when(join -> join.getOtherJoinConjuncts().size() == 1 - && join.getHashJoinConjuncts().size() == 1) - ), - logicalProject( - logicalOlapScan().when(scan -> scan.getTable().getName().equals("t2")) - ) - ).when(join -> join.getOtherJoinConjuncts().size() == 2 - && join.getHashJoinConjuncts().size() == 2) - ) - .printlnExploration(); + .checkMemo(memo -> { + Assertions.assertEquals(1, memo.getRoot().getLogicalExpressions().size()); + }); } } diff --git a/regression-test/data/nereids_syntax_p0/function.out b/regression-test/data/nereids_syntax_p0/function.out index 7f0be9b3f1..f41cd574a8 100644 --- a/regression-test/data/nereids_syntax_p0/function.out +++ b/regression-test/data/nereids_syntax_p0/function.out @@ -48,3 +48,8 @@ 8 9 +-- !floor -- +2 + +-- !ceil -- +3 diff --git a/regression-test/data/nereids_syntax_p0/group_by_constant.out b/regression-test/data/nereids_syntax_p0/group_by_constant.out index a659ddb5f5..719823620e 100644 --- a/regression-test/data/nereids_syntax_p0/group_by_constant.out +++ b/regression-test/data/nereids_syntax_p0/group_by_constant.out @@ -26,3 +26,20 @@ str 24 1310179 10 1 -- !sql -- 2 +-- !sql -- +15 + +-- !sql -- +5 + +-- !sql -- +1310179 +1309892 +1309892 + +-- !sql -- +\N + +-- !sql -- +2 + diff --git a/regression-test/data/nereids_syntax_p0/join_order.out b/regression-test/data/nereids_syntax_p0/join_order.out new file mode 100644 index 0000000000..095c7b2035 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/join_order.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 + diff --git a/regression-test/data/nereids_syntax_p0/set_operation.out b/regression-test/data/nereids_syntax_p0/set_operation.out index 39a0740e6c..91b5c554a8 100644 --- a/regression-test/data/nereids_syntax_p0/set_operation.out +++ b/regression-test/data/nereids_syntax_p0/set_operation.out @@ -581,3 +581,7 @@ hell0 3 d 3 d +-- !union43 -- +2020-05-25 +2020-05-25 + diff --git a/regression-test/data/nereids_syntax_p0/type_cast.out b/regression-test/data/nereids_syntax_p0/type_cast.out new file mode 100644 index 0000000000..0970350d5f --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/type_cast.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 + +-- !sql -- +1 + diff --git a/regression-test/data/query_p0/keyword/order_group.out b/regression-test/data/query_p0/keyword/order_group.out index 5a46327a8b..ff7794c0f2 100644 --- a/regression-test/data/query_p0/keyword/order_group.out +++ b/regression-test/data/query_p0/keyword/order_group.out @@ -1430,7 +1430,6 @@ true 0.1 \N \N -- !orderBy_withNull_3 -- -\N \N \N 15 \N 14 \N 13 @@ -1446,4 +1445,5 @@ true 0.1 \N 3 \N 2 \N 1 +\N \N diff --git a/regression-test/suites/nereids_syntax_p0/function.groovy b/regression-test/suites/nereids_syntax_p0/function.groovy index 761fcd04a1..48d8ab4f2b 100644 --- a/regression-test/suites/nereids_syntax_p0/function.groovy +++ b/regression-test/suites/nereids_syntax_p0/function.groovy @@ -112,5 +112,13 @@ suite("nereids_function") { sql """select "1" == "123", "%%" == "%%" """ result([[false, true]]) } + + qt_floor """ + SELECT floor(2.1); + """ + + qt_ceil """ + SELECT ceil(2.1); + """ } diff --git a/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy b/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy index 6097aa3d41..a8da654096 100644 --- a/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy +++ b/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy @@ -36,4 +36,23 @@ suite("group_by_constant") { qt_sql """select 2 from lineorder group by 1""" + qt_sql """select SUM(lo_tax) FROM lineorder group by null;""" + + qt_sql """select 5 FROM lineorder group by null;""" + + qt_sql """select lo_orderkey from lineorder order by lo_tax desc, lo_tax desc;""" + + test { + sql "select SUM(lo_tax) FROM lineorder group by 1;" + exception "GROUP BY expression must not contain aggregate functions: sum(lo_tax)" + } + + test { + sql "select SUM(lo_tax) FROM lineorder group by SUM(lo_tax);" + exception "GROUP BY expression must not contain aggregate functions: sum(lo_tax)" + } + + qt_sql """select SUM(if(lo_tax=1,lo_tax,0)) FROM lineorder where false;""" + + qt_sql """select 2 FROM lineorder group by 1;""" } diff --git a/regression-test/suites/nereids_syntax_p0/join.groovy b/regression-test/suites/nereids_syntax_p0/join.groovy index e91d6b06d8..52f56a3d96 100644 --- a/regression-test/suites/nereids_syntax_p0/join.groovy +++ b/regression-test/suites/nereids_syntax_p0/join.groovy @@ -22,6 +22,60 @@ suite("join") { sql "SET enable_fallback_to_original_planner=false" + sql """ + drop table if exists test_table_a; + """ + + sql """ + drop table if exists test_table_b; + """ + + sql """ + CREATE TABLE `test_table_a` + ( + `wtid` varchar(30) NOT NULL , + `wfid` varchar(30) NOT NULL + ) ENGINE=OLAP + UNIQUE KEY(`wtid`,`wfid`) + DISTRIBUTED BY HASH(`wfid`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + CREATE TABLE `test_table_b` + ( + `wtid` varchar(100) NOT NULL , + `wfid` varchar(100) NOT NULL + ) ENGINE=OLAP + UNIQUE KEY(`wtid`,`wfid`) + DISTRIBUTED BY HASH(`wfid`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2" + ); + """ + + sql """ + insert into test_table_b values( '1', '1'),('2','2'),('3','3'),('1','2'),('1','3'),('2','3'); + """ + + sql """ + insert into test_table_a values( '1', '1'),('2','2'),('3','3'),('1','2'),('1','3'),('2','3'); + """ + + // must analyze before explain, because empty table may generate different plan + sql """ + analyze table test_table_b; + """ + + sql """ + analyze table test_table_a; + """ + order_qt_inner_join_1 """ SELECT * FROM lineorder JOIN supplier ON lineorder.lo_suppkey = supplier.s_suppkey """ @@ -73,5 +127,106 @@ suite("join") { order_qt_outer_join_with_filter """ select lo_orderkey, lo_partkey, p_partkey, p_size from lineorder inner join part on lo_partkey = p_partkey where lo_orderkey - 1310000 > p_size; """ + + sql """ + drop table if exists outerjoin_A; + """ + + sql """ + drop table if exists outerjoin_B; + """ + + sql """ + drop table if exists outerjoin_C; + """ + + sql """ + drop table if exists outerjoin_D; + """ + + sql """ + create table if not exists outerjoin_A ( a int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + create table if not exists outerjoin_B ( a int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + create table if not exists outerjoin_C ( a int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + create table if not exists outerjoin_D ( a int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """ + insert into outerjoin_A values( 1 ); + """ + + sql """ + insert into outerjoin_B values( 1 ); + """ + + sql """ + insert into outerjoin_C values( 1 ); + """ + + sql """ + insert into outerjoin_D values( 1 ); + """ + + explain { + sql("select count(*) from outerjoin_A A left join outerjoin_B B on A.a = B.a where B.a in (select a from outerjoin_C);") + contains "INNER JOIN" + } + + explain { + sql("""SELECT count(1) + FROM + (SELECT sub1.wtid, + count(*) + FROM + (SELECT a.wtid , + a.wfid + FROM test_table_b a ) sub1 + INNER JOIN [shuffle] + (SELECT a.wtid, + a.wfid + FROM test_table_a a ) sub2 + ON sub1.wtid = sub2.wtid + AND sub1.wfid = sub2.wfid + GROUP BY sub1.wtid ) qqqq;""") + contains "4:VAGGREGATE (update serialize)" + contains "6:VAGGREGATE (merge finalize)" + } } diff --git a/regression-test/suites/nereids_syntax_p0/join_order.groovy b/regression-test/suites/nereids_syntax_p0/join_order.groovy new file mode 100644 index 0000000000..df838ace74 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/join_order.groovy @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("join_order") { + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + + sql """ drop table if exists outerjoin_A;""" + sql """ + create table outerjoin_A ( a1 bigint not null, a2 bigint not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(a1) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + sql """ drop table if exists outerjoin_B;""" + sql """ + create table outerjoin_B ( b int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(b) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + sql """ drop table if exists outerjoin_C;""" + sql """ + create table outerjoin_C ( c int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(c) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + sql """ drop table if exists outerjoin_D;""" + sql """ + create table outerjoin_D ( d1 int not null, d2 int not null, d3 int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(d1) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + sql """ drop table if exists outerjoin_E;""" + sql """ + create table outerjoin_E ( e1 int not null, e2 int not null ) + ENGINE=OLAP + DISTRIBUTED BY HASH(e1) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + sql """insert into outerjoin_A values( 1,2 );""" + sql """insert into outerjoin_B values( 1 );""" + sql """insert into outerjoin_C values( 1 );""" + sql """insert into outerjoin_D values( 1,2,3 );""" + sql """insert into outerjoin_E values( 1,2 );""" + + qt_sql"""SELECT count(*) + FROM outerjoin_A t1 + LEFT JOIN outerjoin_D dcbc + ON t1.a1 = dcbc.d1 + LEFT JOIN outerjoin_C dcso + ON dcbc.d2 = dcso.c + LEFT JOIN outerjoin_B dcii + ON t1.a2 = dcii.b + LEFT JOIN outerjoin_E dcssm + ON dcii.b = dcssm.e1 + AND dcbc.d3 = dcssm.e2; + """ +} diff --git a/regression-test/suites/nereids_syntax_p0/set_operation.groovy b/regression-test/suites/nereids_syntax_p0/set_operation.groovy index 3f3db650fb..d8820e95b5 100644 --- a/regression-test/suites/nereids_syntax_p0/set_operation.groovy +++ b/regression-test/suites/nereids_syntax_p0/set_operation.groovy @@ -54,6 +54,20 @@ suite("test_nereids_set_operation") { ); """ + sql """ drop table if exists test_table;""" + sql """ + CREATE TABLE `test_table` + ( + `day` date + ) ENGINE = OLAP DUPLICATE KEY(`day`) + DISTRIBUTED BY HASH(`day`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """insert into test_table values('2020-05-25');""" + sql """ INSERT INTO setOperationTable VALUES (1, 1, 1, 3, 'a', 'b'), @@ -262,5 +276,5 @@ suite("test_nereids_set_operation") { (select k1, k5 from setOperationTable) """ - + qt_union43 """select '2020-05-25' day from test_table union all select day from test_table;""" } diff --git a/regression-test/suites/nereids_syntax_p0/type_cast.groovy b/regression-test/suites/nereids_syntax_p0/type_cast.groovy new file mode 100644 index 0000000000..9bbc2724ef --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/type_cast.groovy @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("type_cast") { + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + + sql """ drop table if exists test_table2;""" + sql """ + CREATE TABLE `test_table2` + ( + `day` date + ) ENGINE = OLAP DUPLICATE KEY(`day`) + DISTRIBUTED BY HASH(`day`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """insert into test_table2 values('2020-05-25');""" + + def ret = sql"""explain verbose select * from test_table2 where day > CONVERT_tz('2020-05-25 00:00:00', 'Asia/Shanghai', 'Asia/Shanghai');""" + assertTrue(ret.toString().contains("CAST(day[#0] AS DATETIME)")) + + qt_sql """select count(*) from test_table2 where 'a' = 'a';""" + qt_sql """select count(*) from test_table2 where cast('2020-01-01' as date) = cast('2020-01-01' as date);""" +}