From dd1b54cf628645214dcd7b96c27d5fa48cfda84b Mon Sep 17 00:00:00 2001 From: minghong Date: Sat, 11 May 2024 09:44:24 +0800 Subject: [PATCH] [pick](nereids)Runtime filter pushdown refactor for branch-2.1 (#34682) * [refactor](Nereids)refactor runtime filter generator (#34275) 1. unify the process of generating rf for hash join and for nested loop join 2. fix some bugs in generating rf 3. remove some duplicated check (cherry picked from commit 07267faac0d9c6ef3bb1fd4ee101b4c761c8a2f2) * [refactor](nereids) do not deny a runtime filter by removing an entry in aliasMap (#34559) in current version, there are 2 approaches to verify whether a join condition can be used to generate a runtime filter, they are 1. remove the output slot from aliasMap 2. pushDownVisitor.visit(...) return false the 1st approach has some drawbacks, we prefer to the 2ed approach. In this pr, all the cases are handled by the 2ed approach, and remove the related code for the 1st approach. (cherry picked from commit a29082bf31e66efa2df193b38347e610f2bf7464) * rebase --- be/src/clucene | 2 +- .../translator/PhysicalPlanTranslator.java | 2 +- .../processor/post/RuntimeFilterContext.java | 25 +- .../post/RuntimeFilterGenerator.java | 107 +-- .../processor/post/RuntimeFilterPruner.java | 16 +- .../post/RuntimeFilterPushDownVisitor.java | 424 +++++++++++ .../plans/physical/AbstractPhysicalPlan.java | 79 -- .../plans/physical/PhysicalCTEConsumer.java | 16 - .../plans/physical/PhysicalDistribute.java | 36 - .../plans/physical/PhysicalHashAggregate.java | 30 - .../plans/physical/PhysicalHashJoin.java | 105 --- .../physical/PhysicalNestedLoopJoin.java | 20 - .../trees/plans/physical/PhysicalProject.java | 75 -- .../plans/physical/PhysicalSetOperation.java | 41 - .../trees/plans/visitor/PlanVisitor.java | 5 - .../trees/plans/visitor/RelationVisitor.java | 5 + .../shape/query35.out | 10 +- .../rf_prune/query35.out | 4 +- .../shape/query35.out | 10 +- .../runtime_filter/test_pushdown_setop.out | 2 +- .../suites/query_p0/join/test_join.groovy | 720 +++++++++--------- 21 files changed, 865 insertions(+), 869 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java diff --git a/be/src/clucene b/be/src/clucene index d3de160871..9f849a47f7 160000 --- a/be/src/clucene +++ b/be/src/clucene @@ -1 +1 @@ -Subproject commit d3de160871dc1e2e293e5702e5b870e220ed42e4 +Subproject commit 9f849a47f70625a57fedbaa1f5a6f89bc8f32967 diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 610981affa..6748e8df24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -1094,7 +1094,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor> relationsUsedByPlan = Maps.newHashMap(); private final List expandedRF = Lists.newArrayList(); /** @@ -160,6 +160,23 @@ public class RuntimeFilterContext { this.limits = new FilterSizeLimits(sessionVariable); } + public void setRelationsUsedByPlan(Plan plan, Set relations) { + relationsUsedByPlan.put(plan, relations); + } + + /** + * return true, if the relation is in the subtree + */ + public boolean isRelationUseByPlan(Plan plan, PhysicalRelation relation) { + Set relations = relationsUsedByPlan.get(plan); + if (relations == null) { + relations = Sets.newHashSet(); + RuntimeFilterGenerator.getAllScanInfo(plan, relations); + relationsUsedByPlan.put(plan, relations); + } + return relations.contains(relation); + } + public SessionVariable getSessionVariable() { return sessionVariable; } @@ -273,10 +290,6 @@ public class RuntimeFilterContext { return aliasTransferMap; } - public Pair aliasTransferMapRemove(NamedExpression slot) { - return aliasTransferMap.remove(slot); - } - public Pair getAliasTransferPair(NamedExpression slot) { return aliasTransferMap.get(slot); } @@ -354,7 +367,7 @@ public class RuntimeFilterContext { } public List getTargetExprIdByFilterJoin(AbstractPhysicalJoin join) { - return joinToTargetExprId.getOrDefault(join, new ArrayList<>()); + return joinToTargetExprId.getOrDefault(join, Lists.newArrayList()); } public SlotReference getCorrespondingOlapSlotReference(SlotReference slot) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index fbb62ece9f..420e25126d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -52,8 +52,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; -import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; -import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; @@ -63,7 +61,6 @@ import org.apache.doris.thrift.TMinMaxRuntimeFilterType; import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -259,8 +256,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { join.right().accept(this, context); join.left().accept(this, context); if (RuntimeFilterGenerator.DENIED_JOIN_TYPES.contains(join.getJoinType()) || join.isMarkJoin()) { - join.right().getOutput().forEach(slot -> - context.getRuntimeFilterContext().aliasTransferMapRemove(slot)); + // do not generate RF on this join + return join; } RuntimeFilterContext ctx = context.getRuntimeFilterContext(); List legalTypes = Arrays.stream(TRuntimeFilterType.values()) @@ -285,8 +282,16 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { continue; } if (equalTo.left().getInputSlots().size() == 1) { - join.pushDownRuntimeFilter(context, generator, join, equalTo.right(), - equalTo.left(), type, buildSideNdv, i); + RuntimeFilterPushDownVisitor.PushDownContext pushDownContext = + RuntimeFilterPushDownVisitor.PushDownContext.createPushDownContextForHashJoin( + equalTo.right(), equalTo.left(), ctx, generator, type, join, + context.getStatementContext().isHasUnknownColStats(), buildSideNdv, i); + // pushDownContext is not valid, if the target is an agg result. + // Currently, we only apply RF on PhysicalScan. So skip this rf. + // example: (select sum(x) as s from A) T join B on T.s=B.s + if (pushDownContext.isValid()) { + join.accept(new RuntimeFilterPushDownVisitor(), pushDownContext); + } } } } @@ -301,7 +306,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { } @Override - public PhysicalCTEProducer visitPhysicalCTEProducer(PhysicalCTEProducer producer, CascadesContext context) { + public PhysicalCTEProducer visitPhysicalCTEProducer(PhysicalCTEProducer producer, + CascadesContext context) { CTEId cteId = producer.getCteId(); context.getRuntimeFilterContext().getCteProduceMap().put(cteId, producer); Set processedCTE = context.getRuntimeFilterContext().getProcessedCTE(); @@ -336,24 +342,12 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { } else { bitmapContains = (BitmapContains) bitmapRuntimeFilterCondition; } - TRuntimeFilterType type = TRuntimeFilterType.BITMAP; - Set targetSlots = bitmapContains.child(1).getInputSlots(); - for (Slot targetSlot : targetSlots) { - if (!checkProbeSlot(ctx, targetSlot)) { - continue; - } - Slot scanSlot = ctx.getAliasTransferPair(targetSlot).second; - PhysicalRelation scan = ctx.getAliasTransferPair(targetSlot).first; - RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), - bitmapContains.child(0), ImmutableList.of(scanSlot), - ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L, - false, scan); - scan.addAppliedRuntimeFilter(filter); - ctx.addJoinToTargetMap(join, scanSlot.getExprId()); - ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter); - ctx.setTargetsOnScanNode(ctx.getAliasTransferPair(targetSlot).first, - scanSlot); - join.addBitmapRuntimeFilterCondition(bitmapRuntimeFilterCondition); + RuntimeFilterPushDownVisitor.PushDownContext pushDownContext = + RuntimeFilterPushDownVisitor.PushDownContext.createPushDownContextForBitMapFilter( + bitmapContains.child(0), bitmapContains.child(1), ctx, generator, join, + -1, i, isNot); + if (pushDownContext.isValid()) { + join.accept(new RuntimeFilterPushDownVisitor(), pushDownContext); } } } @@ -423,15 +417,13 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { Slot olapScanSlot = pair.second; PhysicalRelation scan = pair.first; Preconditions.checkState(olapScanSlot != null && scan != null); - long buildSideNdv = getBuildSideNdv(join, compare); - RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), - compare.child(1), ImmutableList.of(olapScanSlot), ImmutableList.of(olapScanSlot), - TRuntimeFilterType.MIN_MAX, exprOrder, join, true, buildSideNdv, false, - getMinMaxType(compare), scan); - scan.addAppliedRuntimeFilter(filter); - ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); - ctx.setTargetExprIdToFilter(olapScanSlot.getExprId(), filter); - ctx.setTargetsOnScanNode(scan, olapScanSlot); + RuntimeFilterPushDownVisitor.PushDownContext pushDownContext = + RuntimeFilterPushDownVisitor.PushDownContext.createPushDownContextForNljMinMaxFilter( + compare.child(1), compare.child(0), ctx, generator, join, + exprOrder, getMinMaxType(compare)); + if (pushDownContext.isValid()) { + join.accept(new RuntimeFilterPushDownVisitor(), pushDownContext); + } } } } @@ -442,10 +434,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { // TODO: we need to support all type join join.right().accept(this, context); join.left().accept(this, context); - if (RuntimeFilterGenerator.DENIED_JOIN_TYPES.contains(join.getJoinType()) || join.isMarkJoin()) { - join.right().getOutput().forEach(slot -> - context.getRuntimeFilterContext().aliasTransferMapRemove(slot)); + // do not generate RF on this join return join; } RuntimeFilterContext ctx = context.getRuntimeFilterContext(); @@ -555,7 +545,9 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { break; } } - Preconditions.checkState(targetExpr != null); + Preconditions.checkState(targetExpr != null, + "cannot find runtime filter cte.target: " + + cteSlot + "in project " + project.toString()); if (targetExpr instanceof SlotReference && checkCanPushDownIntoBasicTable(project)) { Map pushDownBasicTableInfos = getPushDownBasicTablesInfos(project, (SlotReference) targetExpr, ctx); @@ -594,37 +586,6 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { return false; } - @Override - public PhysicalPlan visitPhysicalTopN(PhysicalTopN topN, CascadesContext context) { - topN.child().accept(this, context); - PhysicalPlan child = (PhysicalPlan) topN.child(); - for (Slot slot : child.getOutput()) { - context.getRuntimeFilterContext().aliasTransferMapRemove(slot); - } - return topN; - } - - @Override - public PhysicalPlan visitPhysicalWindow(PhysicalWindow window, CascadesContext context) { - window.child().accept(this, context); - Set commonPartitionKeys = window.getCommonPartitionKeyFromWindowExpressions(); - window.child().getOutput().stream().filter(slot -> !commonPartitionKeys.contains(slot)).forEach( - slot -> context.getRuntimeFilterContext().aliasTransferMapRemove(slot) - ); - return window; - } - - /** - * Check runtime filter push down project/distribute pre-conditions. - */ - public static boolean checkPushDownPreconditionsForProjectOrDistribute(RuntimeFilterContext ctx, Slot slot) { - if (slot == null || !ctx.aliasTransferMapContains(slot)) { - return false; - } else { - return true; - } - } - /** * check if slot is in ctx.aliasTransferMap */ @@ -703,12 +664,12 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { /** * Get all relation node from current root plan. */ - public static void getAllScanInfo(PhysicalPlan root, Set scans) { + public static void getAllScanInfo(Plan root, Set scans) { if (root instanceof PhysicalRelation) { scans.add((PhysicalRelation) root); } else { - for (Object child : root.children()) { - getAllScanInfo((PhysicalPlan) child, scans); + for (Plan child : root.children()) { + getAllScanInfo(child, scans); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java index f85e5eebd2..92b2960fb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java @@ -94,7 +94,7 @@ public class RuntimeFilterPruner extends PlanPostProcessor { } @Override - public PhysicalNestedLoopJoin visitPhysicalNestedLoopJoin( + public PhysicalNestedLoopJoin visitPhysicalNestedLoopJoin( PhysicalNestedLoopJoin join, CascadesContext context) { join.right().accept(this, context); @@ -108,28 +108,32 @@ public class RuntimeFilterPruner extends PlanPostProcessor { } @Override - public PhysicalCTEAnchor visitPhysicalCTEAnchor(PhysicalCTEAnchor cteAnchor, - CascadesContext context) { + public PhysicalCTEAnchor visitPhysicalCTEAnchor( + PhysicalCTEAnchor cteAnchor, + CascadesContext context) { cteAnchor.child(0).accept(this, context); cteAnchor.child(1).accept(this, context); return cteAnchor; } @Override - public PhysicalTopN visitPhysicalTopN(PhysicalTopN topN, CascadesContext context) { + public PhysicalTopN visitPhysicalTopN(PhysicalTopN topN, CascadesContext context) { topN.child().accept(this, context); context.getRuntimeFilterContext().addEffectiveSrcNode(topN, RuntimeFilterContext.EffectiveSrcType.NATIVE); return topN; } - public PhysicalLimit visitPhysicalLimit(PhysicalLimit limit, CascadesContext context) { + public PhysicalLimit visitPhysicalLimit( + PhysicalLimit limit, + CascadesContext context) { limit.child().accept(this, context); context.getRuntimeFilterContext().addEffectiveSrcNode(limit, RuntimeFilterContext.EffectiveSrcType.NATIVE); return limit; } @Override - public PhysicalHashJoin visitPhysicalHashJoin(PhysicalHashJoin join, + public PhysicalHashJoin visitPhysicalHashJoin( + PhysicalHashJoin join, CascadesContext context) { join.right().accept(this, context); RuntimeFilterContext rfContext = context.getRuntimeFilterContext(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java new file mode 100644 index 0000000000..f268d1ab8b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java @@ -0,0 +1,424 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post; + +import org.apache.doris.common.IdGenerator; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.processor.post.RuntimeFilterPushDownVisitor.PushDownContext; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.EqualPredicate; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; +import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.types.coercion.NumericType; +import org.apache.doris.planner.RuntimeFilterId; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TMinMaxRuntimeFilterType; +import org.apache.doris.thrift.TRuntimeFilterType; + +import cfjd.com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * push down rf + */ +public class RuntimeFilterPushDownVisitor extends PlanVisitor { + // the context to push down rf by join condition: probeExpr = srcExpr + + /** + * PushDownContext + */ + public static class PushDownContext { + final Expression srcExpr; + final Expression probeExpr; + final Slot probeSlot; + final RuntimeFilterContext rfContext; + final IdGenerator rfIdGen; + final TRuntimeFilterType type; + final AbstractPhysicalJoin builderNode; + final boolean hasUnknownColStats; + final long buildSideNdv; + final int exprOrder; + final Pair finalTarget; + //bitmap rf used only + final boolean isNot; + // only used for Min_Max runtime filter + final TMinMaxRuntimeFilterType singleSideMinMax; + + /** + * push down context + */ + // for hash join runtime filter + private PushDownContext(Expression srcExpr, Expression probeExpr, RuntimeFilterContext rfContext, + IdGenerator rfIdGen, TRuntimeFilterType type, + AbstractPhysicalJoin builderNode, + boolean hasUnknownColStats, long buildSideNdv, int exprOrder, boolean isNot, + TMinMaxRuntimeFilterType singleSideMinMax) { + this.probeExpr = probeExpr; + this.rfContext = rfContext; + this.srcExpr = srcExpr; + this.rfIdGen = rfIdGen; + this.type = type; + this.builderNode = builderNode; + this.hasUnknownColStats = hasUnknownColStats; + this.buildSideNdv = buildSideNdv; + this.exprOrder = exprOrder; + this.isNot = isNot; + Expression expr = getSingleNumericSlotOrExpressionCoveredByCast(probeExpr); + if (expr instanceof Slot) { + probeSlot = (Slot) expr; + finalTarget = rfContext.getAliasTransferPair((Slot) probeSlot); + } else { + finalTarget = null; + probeSlot = null; + } + this.singleSideMinMax = singleSideMinMax; + } + + // for BitMap runtime filter + public static PushDownContext createPushDownContextForBitMapFilter(Expression srcExpr, Expression probeExpr, + RuntimeFilterContext rfContext, + IdGenerator rfIdGen, + AbstractPhysicalJoin builderNode, + long buildSideNdv, int exprOrder, boolean isNot) { + return new PushDownContext(srcExpr, probeExpr, rfContext, rfIdGen, TRuntimeFilterType.BITMAP, builderNode, + false, buildSideNdv, + exprOrder, isNot, TMinMaxRuntimeFilterType.MIN_MAX); + } + + // for NLJ min-max runtime filter + public static PushDownContext createPushDownContextForNljMinMaxFilter(Expression srcExpr, Expression probeExpr, + RuntimeFilterContext rfContext, + IdGenerator rfIdGen, + AbstractPhysicalJoin builderNode, + int exprOrder, + TMinMaxRuntimeFilterType singleSideMinMax) { + return new PushDownContext(srcExpr, probeExpr, rfContext, rfIdGen, TRuntimeFilterType.MIN_MAX, builderNode, + false, -1, + exprOrder, false, singleSideMinMax); + } + + public static PushDownContext createPushDownContextForHashJoin(Expression srcExpr, Expression probeExpr, + RuntimeFilterContext rfContext, + IdGenerator rfIdGen, TRuntimeFilterType type, + AbstractPhysicalJoin builderNode, + boolean hasUnknownColStats, long buildSideNdv, int exprOrder) { + return new PushDownContext(srcExpr, probeExpr, rfContext, rfIdGen, type, builderNode, + hasUnknownColStats, buildSideNdv, + exprOrder, false, TMinMaxRuntimeFilterType.MIN_MAX); + } + + public boolean isValid() { + return finalTarget != null && probeSlot != null; + } + + public PushDownContext withNewProbeExpression(Expression newProbe) { + return new PushDownContext(srcExpr, newProbe, this.rfContext, rfIdGen, type, builderNode, + hasUnknownColStats, buildSideNdv, exprOrder, isNot, singleSideMinMax); + } + + private Expression getSingleNumericSlotOrExpressionCoveredByCast(Expression expression) { + if (expression.getInputSlots().size() == 1) { + Slot slot = expression.getInputSlots().iterator().next(); + if (slot.getDataType() instanceof NumericType) { + return expression.getInputSlots().iterator().next(); + } + } + // for other datatype, only support cast. + // example: T1 join T2 on subStr(T1.a, 1,4) = subStr(T2.a, 1,4) + // the cost of subStr is too high, and hence we do not generate RF subStr(T2.a, 1,4)->subStr(T1.a, 1,4) + while (expression instanceof Cast) { + expression = ((Cast) expression).child(); + } + return expression; + } + } + + @Override + public Boolean visit(Plan plan, PushDownContext ctx) { + boolean pushed = false; + for (Plan child : plan.children()) { + pushed |= child.accept(this, ctx); + } + return pushed; + } + + @Override + public Boolean visitPhysicalRelation(PhysicalRelation scan, PushDownContext ctx) { + Preconditions.checkArgument(ctx.isValid(), + "runtime filter pushDownContext is invalid"); + PhysicalRelation relation = ctx.finalTarget.first; + Slot scanSlot = ctx.finalTarget.second; + Slot probeSlot = ctx.probeSlot; + + if (!relation.equals(scan)) { + return Boolean.FALSE; + } + + TRuntimeFilterType type = ctx.type; + if (type == TRuntimeFilterType.IN_OR_BLOOM + && ctx.rfContext.getSessionVariable().getEnablePipelineEngine() + && RuntimeFilterGenerator.hasRemoteTarget(ctx.builderNode, scan) + && !ctx.builderNode.isBroadCastJoin()) { + type = TRuntimeFilterType.BLOOM; + } + + RuntimeFilter filter = ctx.rfContext.getRuntimeFilterBySrcAndType(ctx.srcExpr, type, ctx.builderNode); + if (filter != null) { + if (!filter.hasTargetScan(scan)) { + // A join B on A.a1=B.b and A.a1 = A.a2 + // RF B.b->(A.a1, A.a2) + // however, RF(B.b->A.a2) is implied by RF(B.a->A.a1) and A.a1=A.a2 + // we skip RF(B.b->A.a2) + scan.addAppliedRuntimeFilter(filter); + filter.addTargetSlot(scanSlot, ctx.probeExpr, scan); + ctx.rfContext.addJoinToTargetMap(ctx.builderNode, scanSlot.getExprId()); + ctx.rfContext.setTargetExprIdToFilter(scanSlot.getExprId(), filter); + ctx.rfContext.setTargetsOnScanNode(ctx.rfContext.getAliasTransferPair(probeSlot).first, scanSlot); + } + } else { + filter = new RuntimeFilter(ctx.rfIdGen.getNextId(), + ctx.srcExpr, ImmutableList.of(scanSlot), ImmutableList.of(ctx.probeExpr), + type, ctx.exprOrder, ctx.builderNode, ctx.isNot, ctx.buildSideNdv, + !ctx.hasUnknownColStats, ctx.singleSideMinMax, scan); + scan.addAppliedRuntimeFilter(filter); + ctx.rfContext.addJoinToTargetMap(ctx.builderNode, scanSlot.getExprId()); + ctx.rfContext.setTargetExprIdToFilter(scanSlot.getExprId(), filter); + ctx.rfContext.setTargetsOnScanNode(ctx.rfContext.getAliasTransferPair(probeSlot).first, scanSlot); + ctx.rfContext.setRuntimeFilterIdentityToFilter(ctx.srcExpr, type, ctx.builderNode, filter); + } + return true; + } + + @Override + public Boolean visitPhysicalHashJoin(PhysicalHashJoin join, + PushDownContext ctx) { + boolean pushed = false; + + if (ctx.builderNode instanceof PhysicalHashJoin) { + /* + hashJoin( t1.A <=> t2.A ) + +---->left outer Join(t1.B=T3.B) + +--->t1 + +--->t3 + +---->t2 + RF(t1.A <=> t2.A) cannot be pushed down through left outer join + */ + EqualPredicate equal = (EqualPredicate) ctx.builderNode.getHashJoinConjuncts().get(ctx.exprOrder); + if (equal instanceof NullSafeEqual) { + if (join.getJoinType().isOuterJoin()) { + return false; + } + } + } + AbstractPhysicalPlan leftNode = (AbstractPhysicalPlan) join.child(0); + AbstractPhysicalPlan rightNode = (AbstractPhysicalPlan) join.child(1); + Set probExprList = Sets.newLinkedHashSet(); + probExprList.add(ctx.probeExpr); + Pair srcPair = ctx.rfContext.getAliasTransferMap().get(ctx.srcExpr); + PhysicalRelation srcNode = (srcPair == null) ? null : srcPair.first; + Pair targetPair = ctx.rfContext.getAliasTransferMap().get(ctx.probeExpr); + if (targetPair == null) { + /* cases for "targetPair is null" + when probeExpr is output slot of setOperator, targetPair is null + */ + return false; + } + PhysicalRelation target1 = targetPair.first; + PhysicalRelation target2 = null; + if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().expandRuntimeFilterByInnerJoin) { + if (!join.equals(ctx.builderNode) + && (join.getJoinType() == JoinType.INNER_JOIN || join.getJoinType().isSemiJoin())) { + for (Expression expr : join.getHashJoinConjuncts()) { + EqualPredicate equalTo = (EqualPredicate) expr; + if (ctx.probeExpr.equals(equalTo.left())) { + probExprList.add(equalTo.right()); + targetPair = ctx.rfContext.getAliasTransferMap().get(equalTo.right()); + target2 = (targetPair == null) ? null : targetPair.first; + } else if (ctx.probeExpr.equals(equalTo.right())) { + probExprList.add(equalTo.left()); + targetPair = ctx.rfContext.getAliasTransferMap().get(equalTo.left()); + target2 = (targetPair == null) ? null : targetPair.first; + } + if (target2 != null) { + ctx.rfContext.getExpandedRF().add( + new RuntimeFilterContext.ExpandRF(join, srcNode, target1, target2, equalTo)); + } + } + probExprList.remove(ctx.srcExpr); + + } + } + for (Expression prob : probExprList) { + PushDownContext ctxForChild = prob.equals(ctx.probeExpr) ? ctx : ctx.withNewProbeExpression(prob); + if (ctx.rfContext.isRelationUseByPlan(leftNode, ctxForChild.finalTarget.first)) { + pushed |= leftNode.accept(this, ctxForChild); + } + if (ctx.rfContext.isRelationUseByPlan(rightNode, ctxForChild.finalTarget.first)) { + pushed |= rightNode.accept(this, ctxForChild); + } + } + return pushed; + } + + @Override + public Boolean visitPhysicalNestedLoopJoin(PhysicalNestedLoopJoin join, + PushDownContext ctx) { + if (ctx.builderNode instanceof PhysicalHashJoin) { + /* + hashJoin( t1.A <=> t2.A ) + +---->left outer Join(t1.B=T3.B) + +--->t1 + +--->t3 + +---->t2 + RF(t1.A <=> t2.A) cannot be pushed down through left outer join + */ + EqualPredicate equal = (EqualPredicate) ctx.builderNode.getHashJoinConjuncts().get(ctx.exprOrder); + if (equal instanceof NullSafeEqual) { + if (join.getJoinType().isOuterJoin()) { + return false; + } + } + } + boolean pushed = false; + if (ctx.rfContext.isRelationUseByPlan(join.left(), ctx.finalTarget.first)) { + pushed |= join.left().accept(this, ctx); + } + if (ctx.rfContext.isRelationUseByPlan(join.right(), ctx.finalTarget.first)) { + pushed |= join.right().accept(this, ctx); + } + return pushed; + } + + @Override + public Boolean visitPhysicalProject(PhysicalProject project, PushDownContext ctx) { + // project ( A+1 as x) + // probeExpr: abs(x) => abs(A+1) + PushDownContext ctxProjectProbeExpr = ctx; + if (ctx.probeExpr instanceof SlotReference) { + for (NamedExpression namedExpression : project.getProjects()) { + if (namedExpression instanceof Alias + && namedExpression.getExprId() == ((SlotReference) ctx.probeExpr).getExprId()) { + if (((Alias) namedExpression).child().getInputSlots().size() > 1) { + // only support one-slot probeExpr + return false; + } + ctxProjectProbeExpr = ctx.withNewProbeExpression(((Alias) namedExpression).child()); + break; + } + } + } else { + for (NamedExpression namedExpression : project.getProjects()) { + if (namedExpression instanceof Alias + && ctx.probeExpr.getInputSlots().contains(namedExpression.toSlot())) { + if (((Alias) namedExpression).child().getInputSlots().size() > 1) { + // only support one-slot probeExpr + return false; + } + Map map = Maps.newHashMap(); + // probeExpr only has one input slot + map.put(ctx.probeExpr.getInputSlots().iterator().next(), + ((Alias) namedExpression).child()); + Expression newProbeExpr = ctx.probeExpr.accept(ExpressionVisitors.EXPRESSION_MAP_REPLACER, map); + ctxProjectProbeExpr = ctx.withNewProbeExpression(newProbeExpr); + break; + } + } + } + + return project.child().accept(this, ctxProjectProbeExpr); + } + + @Override + public Boolean visitPhysicalSetOperation(PhysicalSetOperation setOperation, PushDownContext ctx) { + boolean pushedDown = false; + int projIndex = -1; + Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(ctx.probeExpr); + if (probeSlot == null) { + return false; + } + List output = setOperation.getOutputs(); + for (int j = 0; j < output.size(); j++) { + NamedExpression expr = output.get(j); + if (expr.getName().equals(probeSlot.getName())) { + projIndex = j; + break; + } + } + if (projIndex == -1) { + return false; + } + for (int i = 0; i < setOperation.children().size(); i++) { + Map map = Maps.newHashMap(); + // probeExpr only has one input slot + map.put(ctx.probeExpr.getInputSlots().iterator().next(), + setOperation.getRegularChildrenOutputs().get(i).get(projIndex)); + Expression newProbeExpr = ctx.probeExpr.accept(ExpressionVisitors.EXPRESSION_MAP_REPLACER, map); + PushDownContext childPushDownContext = ctx.withNewProbeExpression(newProbeExpr); + if (childPushDownContext.isValid()) { + /* + * childPushDownContext is not valid, for example: + * setop + * +--->scan t1(A, B) + * +--->select 0, 0 + * push down context for "select 0, 0" is invalid + */ + pushedDown |= setOperation.child(i).accept(this, ctx.withNewProbeExpression(newProbeExpr)); + } + } + return pushedDown; + } + + @Override + public Boolean visitPhysicalTopN(PhysicalTopN topN, PushDownContext ctx) { + return false; + } + + @Override + public Boolean visitPhysicalWindow(PhysicalWindow window, PushDownContext ctx) { + Set commonPartitionKeys = window.getCommonPartitionKeyFromWindowExpressions(); + if (commonPartitionKeys.containsAll(ctx.probeExpr.getInputSlots())) { + return window.child().accept(this, ctx); + } + return false; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java index 26d3601bcf..2146b9fe5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java @@ -17,26 +17,17 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.processor.post.RuntimeFilterContext; -import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.util.MutableState; -import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.Statistics; -import org.apache.doris.thrift.TRuntimeFilterType; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -74,76 +65,6 @@ public abstract class AbstractPhysicalPlan extends AbstractPlan implements Physi return physicalProperties; } - /** - * Pushing down runtime filter into different plan node, such as olap scan node, cte sender node, etc. - */ - public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator generator, - AbstractPhysicalJoin builderNode, - Expression src, Expression probeExpr, - TRuntimeFilterType type, long buildSideNdv, int exprOrder) { - RuntimeFilterContext ctx = context.getRuntimeFilterContext(); - // currently, we can ensure children in the two side are corresponding to the equal_to's. - // so right maybe an expression and left is a slot - Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr); - - // aliasTransMap doesn't contain the key, means that the path from the scan to the join - // contains join with denied join type. for example: a left join b on a.id = b.id - if (!RuntimeFilterGenerator.checkProbeSlot(ctx, probeSlot)) { - return false; - } - - boolean pushedDown = false; - for (Object child : children) { - AbstractPhysicalPlan childPlan = (AbstractPhysicalPlan) child; - pushedDown |= childPlan.pushDownRuntimeFilter(context, generator, builderNode, src, probeExpr, - type, buildSideNdv, exprOrder); - } - if (pushedDown) { - return true; - } - - Slot scanSlot = ctx.getAliasTransferPair(probeSlot).second; - PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first; - if (!RuntimeFilterGenerator.checkPushDownPreconditionsForRelation(this, scan)) { - return false; - } - - // in-filter is not friendly to pipeline - if (type == TRuntimeFilterType.IN_OR_BLOOM - && ctx.getSessionVariable().getEnablePipelineEngine() - && RuntimeFilterGenerator.hasRemoteTarget(builderNode, scan) - && !builderNode.isBroadCastJoin()) { - type = TRuntimeFilterType.BLOOM; - } - org.apache.doris.nereids.trees.plans.physical.RuntimeFilter filter = - ctx.getRuntimeFilterBySrcAndType(src, type, builderNode); - Preconditions.checkState(scanSlot != null, "scan slot is null"); - if (filter != null) { - if (!filter.hasTargetScan(scan)) { - // A join B on A.a1=B.b and A.a1 = A.a2 - // RF B.b->(A.a1, A.a2) - // however, RF(B.b->A.a2) is implied by RF(B.a->A.a1) and A.a1=A.a2 - // we skip RF(B.b->A.a2) - this.addAppliedRuntimeFilter(filter); - filter.addTargetSlot(scanSlot, probeExpr, scan); - ctx.addJoinToTargetMap(builderNode, scanSlot.getExprId()); - ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter); - ctx.setTargetsOnScanNode(ctx.getAliasTransferPair(probeSlot).first, scanSlot); - } - } else { - filter = new RuntimeFilter(generator.getNextId(), - src, ImmutableList.of(scanSlot), ImmutableList.of(probeExpr), - type, exprOrder, builderNode, buildSideNdv, - !context.getStatementContext().isHasUnknownColStats(), scan); - this.addAppliedRuntimeFilter(filter); - ctx.addJoinToTargetMap(builderNode, scanSlot.getExprId()); - ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter); - ctx.setTargetsOnScanNode(ctx.getAliasTransferPair(probeSlot).first, scanSlot); - ctx.setRuntimeFilterIdentityToFilter(src, type, builderNode, filter); - } - return true; - } - @Override public Plan getExplainPlan(ConnectContext ctx) { return this; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java index 97714a4a48..a3fab00816 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java @@ -17,22 +17,17 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; -import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.statistics.Statistics; -import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -152,17 +147,6 @@ public class PhysicalCTEConsumer extends PhysicalRelation { return shapeBuilder.toString(); } - @Override - public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator generator, - AbstractPhysicalJoin builderNode, - Expression src, Expression probeExpr, - TRuntimeFilterType type, long buildSideNdv, int exprOrder) { - // push down rf on cte sender - // TODO: refactor pushing down into cte internal here - return super.pushDownRuntimeFilter(context, generator, builderNode, - src, probeExpr, type, buildSideNdv, exprOrder); - } - @Override public boolean canPushDownRuntimeFilter() { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java index 3891da6c1a..d85148af38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java @@ -17,11 +17,7 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.processor.post.RuntimeFilterContext; -import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -31,9 +27,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; -import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.statistics.Statistics; -import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -126,36 +120,6 @@ public class PhysicalDistribute extends PhysicalUnary generator, - AbstractPhysicalJoin builderNode, Expression src, Expression probeExpr, - TRuntimeFilterType type, long buildSideNdv, int exprOrder) { - RuntimeFilterContext ctx = context.getRuntimeFilterContext(); - // currently, we can ensure children in the two side are corresponding to the equal_to's. - // so right maybe an expression and left is a slot - Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr); - if (probeSlot == null) { - return false; - } - if (RuntimeFilterGenerator.checkPushDownPreconditionsForProjectOrDistribute(ctx, probeSlot)) { - PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first; - if (!RuntimeFilterGenerator.checkPushDownPreconditionsForRelation(this, scan)) { - return false; - } - // TODO: global rf need merge stage which is heavy - // add some rule, such as bc only is allowed for - // pushing down through distribute, currently always pushing. - AbstractPhysicalPlan childPlan = (AbstractPhysicalPlan) child(0); - return childPlan.pushDownRuntimeFilter(context, generator, builderNode, src, probeExpr, - type, buildSideNdv, exprOrder); - } else { - // if probe slot doesn't exist in aliasTransferMap, then try to pass it to child - AbstractPhysicalPlan childPlan = (AbstractPhysicalPlan) child(0); - return childPlan.pushDownRuntimeFilter(context, generator, builderNode, src, probeExpr, - type, buildSideNdv, exprOrder); - } - } - @Override public List computeOutput() { return child().getOutput(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java index 2ace7d8586..a79953dc71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java @@ -17,11 +17,7 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.processor.post.RuntimeFilterContext; -import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.properties.RequireProperties; @@ -37,9 +33,7 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; -import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.statistics.Statistics; -import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -291,30 +285,6 @@ public class PhysicalHashAggregate extends PhysicalUnar return builder.toString(); } - @Override - public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator generator, - AbstractPhysicalJoin builderNode, Expression src, Expression probeExpr, - TRuntimeFilterType type, long buildSideNdv, int exprOrder) { - RuntimeFilterContext ctx = context.getRuntimeFilterContext(); - // currently, we can ensure children in the two side are corresponding to the equal_to's. - // so right maybe an expression and left is a slot - Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr); - - // aliasTransMap doesn't contain the key, means that the path from the scan to the join - // contains join with denied join type. for example: a left join b on a.id = b.id - if (!RuntimeFilterGenerator.checkProbeSlot(ctx, probeSlot)) { - return false; - } - PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first; - if (!RuntimeFilterGenerator.checkPushDownPreconditionsForRelation(this, scan)) { - return false; - } - - AbstractPhysicalPlan child = (AbstractPhysicalPlan) child(0); - return child.pushDownRuntimeFilter(context, generator, builderNode, - src, probeExpr, type, buildSideNdv, exprOrder); - } - @Override public List computeOutput() { return outputExpressions.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index e2472db87c..2e6deda5fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -17,34 +17,23 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.hint.DistributeHint; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.processor.post.RuntimeFilterContext; -import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; -import org.apache.doris.nereids.trees.expressions.NullSafeEqual; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.MutableState; -import org.apache.doris.planner.RuntimeFilterId; -import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.Statistics; -import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.List; @@ -201,100 +190,6 @@ public class PhysicalHashJoin< getLogicalProperties(), physicalProperties, statistics, left(), right()); } - @Override - public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator generator, - AbstractPhysicalJoin builderNode, Expression srcExpr, Expression probeExpr, - TRuntimeFilterType type, long buildSideNdv, int exprOrder) { - if (RuntimeFilterGenerator.DENIED_JOIN_TYPES.contains(getJoinType()) || isMarkJoin()) { - if (builderNode instanceof PhysicalHashJoin) { - PhysicalHashJoin builderJoin = (PhysicalHashJoin) builderNode; - if (builderJoin == this) { - return false; - } - EqualPredicate equal = (EqualPredicate) builderNode.getHashJoinConjuncts().get(exprOrder); - if (equal instanceof NullSafeEqual) { - if (this.joinType.isOuterJoin()) { - return false; - } - } - } - } - EqualPredicate equal = (EqualPredicate) builderNode.getHashJoinConjuncts().get(exprOrder); - if (equal instanceof NullSafeEqual) { - if (this.joinType.isOuterJoin()) { - return false; - } - } - RuntimeFilterContext ctx = context.getRuntimeFilterContext(); - - // if rf built between plan nodes containing cte both, for example both src slot and target slot are from cte, - // or two sub-queries both containing cte, disable this rf since this kind of cross-cte rf will make one side - // of cte to wait for a long time until another side cte consumer finished, which will make the rf into - // not ready state. - AbstractPhysicalPlan builderLeftNode = (AbstractPhysicalPlan) builderNode.child(0); - AbstractPhysicalPlan builderRightNode = (AbstractPhysicalPlan) builderNode.child(1); - Preconditions.checkState(builderLeftNode != null && builderRightNode != null, - "builder join node child node is null"); - // if (RuntimeFilterGenerator.hasCTEConsumerDescendant(builderLeftNode) - // && RuntimeFilterGenerator.hasCTEConsumerDescendant(builderRightNode)) { - // return false; - // } - - boolean pushedDown = false; - AbstractPhysicalPlan leftNode = (AbstractPhysicalPlan) child(0); - AbstractPhysicalPlan rightNode = (AbstractPhysicalPlan) child(1); - Preconditions.checkState(leftNode != null && rightNode != null, - "join child node is null"); - - Set probExprList = Sets.newLinkedHashSet(); - probExprList.add(probeExpr); - Pair srcPair = ctx.getAliasTransferMap().get(srcExpr); - PhysicalRelation srcNode = (srcPair == null) ? null : srcPair.first; - Pair targetPair = ctx.getAliasTransferMap().get(probeExpr); - if (targetPair == null) { - /* cases for "targetPair is null" - 1. when probeExpr is output slot of setOperator, targetPair is null - 2. join (topn(table1), table2) - when aliasTransferMap goes through topn, all slots from table1 - are cleared to avoid generate rf(table2->table1) - */ - return false; - } - PhysicalRelation target1 = targetPair.first; - PhysicalRelation target2 = null; - if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().expandRuntimeFilterByInnerJoin) { - if (!this.equals(builderNode) - && (this.getJoinType() == JoinType.INNER_JOIN || this.getJoinType().isSemiJoin())) { - for (Expression expr : this.getHashJoinConjuncts()) { - EqualPredicate equalTo = (EqualPredicate) expr; - if (probeExpr.equals(equalTo.left())) { - probExprList.add(equalTo.right()); - targetPair = ctx.getAliasTransferMap().get(equalTo.right()); - target2 = (targetPair == null) ? null : targetPair.first; - } else if (probeExpr.equals(equalTo.right())) { - probExprList.add(equalTo.left()); - targetPair = ctx.getAliasTransferMap().get(equalTo.left()); - target2 = (targetPair == null) ? null : targetPair.first; - } - if (target2 != null) { - ctx.getExpandedRF().add( - new RuntimeFilterContext.ExpandRF(this, srcNode, target1, target2, equalTo)); - } - } - probExprList.remove(srcExpr); - - } - } - for (Expression prob : probExprList) { - pushedDown |= leftNode.pushDownRuntimeFilter(context, generator, builderNode, - srcExpr, prob, type, buildSideNdv, exprOrder); - pushedDown |= rightNode.pushDownRuntimeFilter(context, generator, builderNode, - srcExpr, prob, type, buildSideNdv, exprOrder); - } - - return pushedDown; - } - @Override public String shapeInfo() { StringBuilder builder = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java index c3c4819ca9..b8e5c17e03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java @@ -17,16 +17,12 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.hint.DistributeHint; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; -import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.DistributeType; import org.apache.doris.nereids.trees.plans.JoinType; @@ -35,9 +31,7 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.MutableState; -import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.statistics.Statistics; -import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; @@ -244,18 +238,4 @@ public class PhysicalNestedLoopJoin< hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference, groupExpression, null, physicalProperties, statistics, left(), right()); } - - @Override - public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator generator, - AbstractPhysicalJoin builderNode, Expression srcExpr, - Expression probeExpr, TRuntimeFilterType type, long buildSideNdv, - int exprOrder) { - EqualPredicate equal = (EqualPredicate) builderNode.getHashJoinConjuncts().get(exprOrder); - if (equal instanceof NullSafeEqual && this.joinType.isOuterJoin()) { - return false; - } - - return super.pushDownRuntimeFilter(context, generator, builderNode, srcExpr, probeExpr, type, buildSideNdv, - exprOrder); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java index 9d51d6f724..02769e4752 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java @@ -17,11 +17,7 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.processor.post.RuntimeFilterContext; -import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Add; @@ -34,11 +30,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Project; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; -import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.statistics.Statistics; -import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -169,74 +162,6 @@ public class PhysicalProject extends PhysicalUnary generator, - AbstractPhysicalJoin builderNode, Expression src, Expression probeExpr, - TRuntimeFilterType type, long buildSideNdv, int exprOrder) { - RuntimeFilterContext ctx = context.getRuntimeFilterContext(); - // currently, we can ensure children in the two side are corresponding to the equal_to's. - // so right maybe an expression and left is a slot - Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr); - if (probeSlot == null) { - return false; - } - - if (RuntimeFilterGenerator.checkPushDownPreconditionsForProjectOrDistribute(ctx, probeSlot)) { - PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first; - Preconditions.checkState(scan != null, "scan is null"); - if (scan instanceof PhysicalCTEConsumer) { - // update the probeExpr - int projIndex = -1; - for (int i = 0; i < getProjects().size(); i++) { - NamedExpression expr = getProjects().get(i); - if (expr.getName().equals(probeSlot.getName())) { - projIndex = i; - break; - } - } - if (projIndex < 0 || projIndex >= getProjects().size()) { - // the pushed down path can't contain the probe expr - return false; - } - NamedExpression newProbeExpr = this.getProjects().get(projIndex); - if (newProbeExpr instanceof Alias) { - Expression child = ExpressionUtils.getExpressionCoveredByCast(newProbeExpr.child(0)); - if (child instanceof NamedExpression) { - newProbeExpr = (NamedExpression) child; - } else { - return false; - } - } - Slot newProbeSlot = RuntimeFilterGenerator.checkTargetChild(newProbeExpr); - if (!RuntimeFilterGenerator.checkProbeSlot(ctx, newProbeSlot)) { - return false; - } - scan = ctx.getAliasTransferPair(newProbeSlot).first; - probeExpr = newProbeExpr; - } - if (!RuntimeFilterGenerator.checkPushDownPreconditionsForRelation(this, scan)) { - return false; - } - if (probeExpr instanceof SlotReference) { - for (NamedExpression namedExpression : projects) { - if (namedExpression instanceof Alias - && namedExpression.getExprId() == ((SlotReference) probeExpr).getExprId()) { - probeExpr = ((Alias) namedExpression).child(); - break; - } - } - } - AbstractPhysicalPlan child = (AbstractPhysicalPlan) child(0); - return child.pushDownRuntimeFilter(context, generator, builderNode, - src, probeExpr, type, buildSideNdv, exprOrder); - } else { - // if probe slot doesn't exist in aliasTransferMap, then try to pass it to child - AbstractPhysicalPlan child = (AbstractPhysicalPlan) child(0); - return child.pushDownRuntimeFilter(context, generator, builderNode, - src, probeExpr, type, buildSideNdv, exprOrder); - } - } - @Override public List computeOutput() { List output = projects; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java index 1018bc3d93..07326a8afb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java @@ -17,30 +17,22 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.planner.RuntimeFilterId; import org.apache.doris.statistics.Statistics; -import org.apache.doris.thrift.TRuntimeFilterType; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Maps; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -145,39 +137,6 @@ public abstract class PhysicalSetOperation extends AbstractPhysicalPlan implemen return children.size(); } - @Override - public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator generator, - AbstractPhysicalJoin builderNode, Expression src, Expression probeExpr, - TRuntimeFilterType type, long buildSideNdv, int exprOrder) { - boolean pushedDown = false; - int projIndex = -1; - Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr); - if (probeSlot == null) { - return false; - } - List output = getOutputs(); - for (int j = 0; j < output.size(); j++) { - NamedExpression expr = output.get(j); - if (expr.getName().equals(probeSlot.getName())) { - projIndex = j; - break; - } - } - if (projIndex == -1) { - return false; - } - for (int i = 0; i < this.children().size(); i++) { - Map map = Maps.newHashMap(); - // probeExpr only has one input slot - map.put(probeExpr.getInputSlots().iterator().next(), regularChildrenOutputs.get(i).get(projIndex)); - Expression newProbeExpr = probeExpr.accept(ExpressionVisitors.EXPRESSION_MAP_REPLACER, map); - AbstractPhysicalPlan child = (AbstractPhysicalPlan) this.child(i); - pushedDown |= child.pushDownRuntimeFilter(context, generator, builderNode, src, - newProbeExpr, type, buildSideNdv, exprOrder); - } - return pushedDown; - } - @Override public List computeOutput() { return outputs.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java index 9a21e5b14e..8db1407220 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java @@ -55,7 +55,6 @@ import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; -import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; @@ -274,10 +273,6 @@ public abstract class PlanVisitor implements CommandVisitor, Relatio return visit(cteAnchor, context); } - public R visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer, C context) { - return visit(cteConsumer, context); - } - public R visitPhysicalCTEProducer(PhysicalCTEProducer cteProducer, C context) { return visit(cteProducer, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java index 5eaf0b7213..95f9c6c96d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; @@ -176,4 +177,8 @@ public interface RelationVisitor { default R visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, C context) { return visitPhysicalRelation(tvfRelation, context); } + + default R visitPhysicalCTEConsumer(PhysicalCTEConsumer consumer, C context) { + return visitPhysicalRelation(consumer, context); + } } diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out index 1fd8953df6..666bab9f30 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out @@ -13,19 +13,19 @@ PhysicalResultSink --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=() ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk] +--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 +------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 ----------------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------------PhysicalProject --------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 1999)) ----------------------------------PhysicalOlapScan[date_dim] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() +--------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk] ----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() +--------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk] ----------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ss_customer_sk] --------------------------------------PhysicalDistribute[DistributionSpecHash] @@ -49,7 +49,7 @@ PhysicalResultSink ----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------PhysicalProject ---------------------------------------------PhysicalOlapScan[customer] +--------------------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF4 ----------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out index 5c8462fecc..8cdd21c2e1 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out @@ -13,9 +13,9 @@ PhysicalResultSink --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=() ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk] +--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 +------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 ----------------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------------PhysicalProject --------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out index 5c8462fecc..f5a56cf3ec 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out @@ -13,19 +13,19 @@ PhysicalResultSink --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=() ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk] +--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 +------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 ----------------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------------PhysicalProject --------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001)) ----------------------------------PhysicalOlapScan[date_dim] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() +--------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk] ----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() +--------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk] ----------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ss_customer_sk] --------------------------------------PhysicalDistribute[DistributionSpecHash] @@ -49,7 +49,7 @@ PhysicalResultSink ----------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------PhysicalProject ---------------------------------------------PhysicalOlapScan[customer] +--------------------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF4 ----------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out index 4eb9e9d7f0..aeca755c29 100644 --- a/regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out +++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out @@ -23,7 +23,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((expr_abs(l_linenumber) = expr_cast(r_regionkey as LARGEINT))) otherCondition=() build RFs:RF0 expr_cast(r_regionkey as LARGEINT)->[abs(l_linenumber),abs(o_orderkey)] +----------hashJoin[INNER_JOIN] hashCondition=((expr_abs(l_linenumber) = expr_cast(r_regionkey as LARGEINT))) otherCondition=() build RFs:RF0 expr_cast(r_regionkey as LARGEINT)->[abs(cast(l_linenumber as BIGINT)),abs(o_orderkey)] ------------PhysicalProject --------------PhysicalExcept ----------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/suites/query_p0/join/test_join.groovy b/regression-test/suites/query_p0/join/test_join.groovy index 35a9f78376..137f30ebce 100644 --- a/regression-test/suites/query_p0/join/test_join.groovy +++ b/regression-test/suites/query_p0/join/test_join.groovy @@ -18,93 +18,89 @@ suite("test_join", "query,p0") { sql"use test_query_db" - def tbName1 = "test" - def tbName2 = "baseall" - def tbName3 = "bigtable" - def empty_name = "empty" - sql"drop view if exists empty" - sql"create view empty as select * from baseall where k1 = 0" + sql"drop view if exists test_join_empty_view" + sql"create view test_join_empty_view as select * from baseall where k1 = 0" - order_sql """select j.*, d.* from ${tbName2} j full outer join ${tbName1} d on (j.k1=d.k1) order by j.k1, j.k2, j.k3, j.k4, d.k1, d.k2 + order_sql """select j.*, d.* from baseall j full outer join test d on (j.k1=d.k1) order by j.k1, j.k2, j.k3, j.k4, d.k1, d.k2 limit 100""" order_sql """select * from (select j.k1 j1, j.k2 j2, j.k3 j3, j.k4 j4, j.k5 j5, j.k6 j6, j.k10 j10, j.k11 j11, j.k7 j7, j.k8 j8, j.k9 j9, d.k1 d1, d.k2 d2, d.k3 d3, d.k4 d4, d.k5 d5, d.k6 d6, d.k10 d10, - d.k11 d11, d.k7 d7, d.k8 d8, d.k9 d9 from ${tbName2} j left join ${tbName1} d on (j.k1=d.k1) + d.k11 d11, d.k7 d7, d.k8 d8, d.k9 d9 from baseall j left join test d on (j.k1=d.k1) union select j.k1 j1, j.k2 j2, j.k3 j3, j.k4 j4, j.k5 j5, j.k6 j6, j.k10 j10, j.k11 j11, j.k7 j7, j.k8 j8, j.k9 j9, d.k1 d1, d.k2 d2, d.k3 d3, d.k4 d4, d.k5 d5, d.k6 d6, d.k10 d10, d.k11 d11, - d.k7 d7, d.k8 d8, d.k9 d9 from ${tbName2} j right join ${tbName1} d on (j.k1=d.k1) ) a order by j1, j2, j3, j4, d1, d2 + d.k7 d7, d.k8 d8, d.k9 d9 from baseall j right join test d on (j.k1=d.k1) ) a order by j1, j2, j3, j4, d1, d2 limit 100""" - qt_join1 """select sum(t1.k1), sum(t1.k3), max(t1.k5), max(t2.k4) from ${tbName1} t1 inner join ${tbName2} t2 on t1.k1 = t2.k1 and + qt_join1 """select sum(t1.k1), sum(t1.k3), max(t1.k5), max(t2.k4) from test t1 inner join baseall t2 on t1.k1 = t2.k1 and t1.k6 is not null and t2.k6 is not null""" - qt_join2 """select k1, k2, k3 from ${tbName1} where k7 is not null order by 1 desc, 2 desc, 3 desc limit 10""" - qt_join3 """select c.k1, c.k8 from ${tbName2} d join (select a.k1 as k1, a.k8 from ${tbName1} a join ${tbName2} b on (a.k1=b.k1)) c + qt_join2 """select k1, k2, k3 from test where k7 is not null order by 1 desc, 2 desc, 3 desc limit 10""" + qt_join3 """select c.k1, c.k8 from baseall d join (select a.k1 as k1, a.k8 from test a join baseall b on (a.k1=b.k1)) c on c.k1 = d.k1 order by 1, 2""" - qt_join4 """select a.k1, b.k1 from ${tbName2} a join (select k1, k2 from ${tbName1} order by k1 limit 10) b + qt_join4 """select a.k1, b.k1 from baseall a join (select k1, k2 from test order by k1 limit 10) b on a.k2=b.k2 order by 1, 2""" - qt_join5 """select a.k1, b.k2 from ${tbName2} as a join (select k1, k2 from ${tbName1}) as b + qt_join5 """select a.k1, b.k2 from baseall as a join (select k1, k2 from test) as b where a.k1 = b.k1 order by a.k1, b.k2""" - qt_join6 """select A.k1,B.k1 from ${tbName2} as A join ${tbName1} as B where A.k1=B.k1+1 + qt_join6 """select A.k1,B.k1 from baseall as A join test as B where A.k1=B.k1+1 order by A.k1, A.k2, A.k3, A.k4""" - qt_join7 """select A.k1, B.k2 from ${tbName2} as A join ${tbName1} as B + qt_join7 """select A.k1, B.k2 from baseall as A join test as B order by A.k1, B.k2 limit 10""" - qt_join8 """select A.k1 from ${tbName2} as A join ${tbName1} as B order by A.k1 limit 10""" - qt_join9 """select a.k4 from ${tbName1} a inner join ${tbName2} b on (a.k1=b.k1) + qt_join8 """select A.k1 from baseall as A join test as B order by A.k1 limit 10""" + qt_join9 """select a.k4 from test a inner join baseall b on (a.k1=b.k1) where a.k2>0 and b.k1=1 and a.k1=1 order by 1""" - qt_join10 """select j.*, d.* from ${tbName1} j inner join ${tbName2} d on (j.k1=d.k1) + qt_join10 """select j.*, d.* from test j inner join baseall d on (j.k1=d.k1) order by j.k1, j.k2, j.k3, j.k4""" qt_join11 """select a.k1, b.k2, c.k3 - from ${tbName1} a join ${tbName2} b on (a.k1=b.k1) join ${tbName1} c on (a.k1 = c.k1) + from test a join baseall b on (a.k1=b.k1) join test c on (a.k1 = c.k1) where a.k2>0 and a.k1+50<0""" - qt_join12 """select t1.k1, t2.k1 from ${tbName1} t1 join ${tbName2} t2 where (t1.k1<3 and t2.k1<3) + qt_join12 """select t1.k1, t2.k1 from test t1 join baseall t2 where (t1.k1<3 and t2.k1<3) order by t1.k1, t2.k1 limit 100""" qt_join13 """select a.k1, b.k1, a.k2, b.k2 from - (select k1, k2 from ${tbName1} where k9>0 and k6="false" union all - select k1, k2 from ${tbName2} where k6="true" union all + (select k1, k2 from test where k9>0 and k6="false" union all + select k1, k2 from baseall where k6="true" union all select 0, 0) a inner join - ${tbName1} b on a.k1=b.k1 and b.k1<5 order by 1, 2, 3, 4""" + test b on a.k1=b.k1 and b.k1<5 order by 1, 2, 3, 4""" qt_join14 """select a.k1, b.k1, a.k2, b.k2 from - ${tbName1} b left outer join - (select k1, k2 from ${tbName2} where k9>0 and k6="false" union all - select k1, k2 from ${tbName1} where k6="true" union all + test b left outer join + (select k1, k2 from baseall where k9>0 and k6="false" union all + select k1, k2 from test where k6="true" union all select 0, 0) a on a.k1=b.k1 where b.k1<5 and a.k1 is not NULL order by 1, 2, 3, 4""" qt_join15 """select a.k1, b.k1, a.k2, b.k2 from - (select k1, k2 from ${tbName1} where k1=1 and lower(k6) like "%w%" union all - select k1, k2 from ${tbName2} where k1=2 union all + (select k1, k2 from test where k1=1 and lower(k6) like "%w%" union all + select k1, k2 from baseall where k1=2 union all select 0, 1) a join - (select k1, k2 from ${tbName2} where k1=1 and lower(k6) like "%w%" union all - select k1, k2 from ${tbName1} where k1>0 union all + (select k1, k2 from baseall where k1=1 and lower(k6) like "%w%" union all + select k1, k2 from test where k1>0 union all select 1, 2) b on a.k1 = b.k1 where b.k1<5 order by 1, 2, 3, 4""" qt_join16 """select count(*) from - (select k1 from ${tbName1} union distinct - select k1 from ${tbName2}) a inner join - (select k2 from ${tbName2} union distinct - select k2 from ${tbName1}) b on a.k1+1000=b.k2 inner join - (select distinct k1 from ${tbName1}) c on a.k1=c.k1""" - qt_join17 """select count(t1.k1) as wj from ${tbName2} t1 left join - ${tbName2} t2 on t1.k10=t2.k10 left join - ${tbName2} t3 on t2.k1 = t3.k1""" - qt_join18 """select j.*, d.* from ${tbName1} j left join ${tbName2} d on (lower(j.k6) = lower(d.k6)) + (select k1 from test union distinct + select k1 from baseall) a inner join + (select k2 from baseall union distinct + select k2 from test) b on a.k1+1000=b.k2 inner join + (select distinct k1 from test) c on a.k1=c.k1""" + qt_join17 """select count(t1.k1) as wj from baseall t1 left join + baseall t2 on t1.k10=t2.k10 left join + baseall t3 on t2.k1 = t3.k1""" + qt_join18 """select j.*, d.* from test j left join baseall d on (lower(j.k6) = lower(d.k6)) order by j.k1, j.k2, j.k3, j.k4, d.k2, d.k3, d.k4""" - qt_join19 """select j.*, d.* from ${tbName1} j right join ${tbName2} d on (lower(j.k6) = lower(d.k6)) + qt_join19 """select j.*, d.* from test j right join baseall d on (lower(j.k6) = lower(d.k6)) order by j.k1, j.k2, j.k3, j.k4, d.k2, d.k3, d.k4""" - qt_join20 """select k1, v.k2 from ${tbName2} c, (select k2 from ${tbName1} order by k2 limit 2) v + qt_join20 """select k1, v.k2 from baseall c, (select k2 from test order by k2 limit 2) v where k1 in (1, 2, 3) order by 1, 2""" - qt_join21 """select k1, v.k2 from ${tbName2} c, (select k2 from ${tbName1} order by k2 limit 2) v + qt_join21 """select k1, v.k2 from baseall c, (select k2 from test order by k2 limit 2) v where k1 in (1, 2, 3) and v.k2%2=0 order by 1, 2""" - qt_join22 """select k1, k2, cnt, avp from ${tbName1} c, (select count(k1) cnt, avg(k2) avp from ${tbName2}) v where k1 <3 + qt_join22 """select k1, k2, cnt, avp from test c, (select count(k1) cnt, avg(k2) avp from baseall) v where k1 <3 order by 1, 2, 3, 4""" - qt_join23 """select k1, avg(maxp) from ${tbName1} c, (select max(k8) maxp from ${tbName1} group by k1) v where k1<3 group by k1 + qt_join23 """select k1, avg(maxp) from test c, (select max(k8) maxp from test group by k1) v where k1<3 group by k1 order by 1, 2""" - qt_join24 """select k1, v.k3, cnt, avp from ${tbName2} c, (select count(k1) cnt, avg(k9) avp, k3 from ${tbName2} group by k3) v + qt_join24 """select k1, v.k3, cnt, avp from baseall c, (select count(k1) cnt, avg(k9) avp, k3 from baseall group by k3) v where k1<0 order by 1, 2, 3, 4""" - qt_join25 """select count(k5), k2 from ${tbName1} c, (select ca.k1 okey, cb.k2 opr from ${tbName2} ca, ${tbName1} cb where + qt_join25 """select count(k5), k2 from test c, (select ca.k1 okey, cb.k2 opr from baseall ca, test cb where ca.k1=cb.k1 and ca.k2+cb.k2>2) v group by k2 order by 1, 2""" - qt_join26 """select count(k6), k1 from ${tbName1} c, - (select ca.k1 wj, ca.k2 opr from ${tbName2} ca left outer join ${tbName1} cb on ca.k1 = cb.k1) v + qt_join26 """select count(k6), k1 from test c, + (select ca.k1 wj, ca.k2 opr from baseall ca left outer join test cb on ca.k1 = cb.k1) v group by k1 order by 1, 2""" - qt_join27 """select count(k6), k1 from ${tbName1} c, - (select ca.k1 wj, ca.k2 opr from ${tbName2} ca right outer join ${tbName1} cb + qt_join27 """select count(k6), k1 from test c, + (select ca.k1 wj, ca.k2 opr from baseall ca right outer join test cb on ca.k1 = cb.k1 and ca.k2+cb.k2>2) v group by k1 order by 1, 2""" // Ocurrs time out with specified time 299969 MILLISECONDS @@ -112,236 +108,236 @@ suite("test_join", "query,p0") { List selected = ["a.k1, b.k1, a.k2, b.k2, a.k3, b.k3", "count(a.k1), count(b.k1), count(a.k2), count(b.k2), count(*)"] for( i in selected) { - qt_join28"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join28"""select ${i} from test a join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join29"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join29"""select ${i} from test a join baseall b on a.k1 > b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join30"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join30"""select ${i} from test a join baseall b on a.k1 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join31"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join31"""select ${i} from test a join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join32"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join32"""select ${i} from test a join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join33"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join33"""select ${i} from test a join baseall b order by 1, 2, 3, 4, 5 limit 65535""" - qt_join34"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join34"""select ${i} from test a join baseall b on a.k1 = b.k1 or a.k2 = b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join35"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join35"""select ${i} from test a join baseall b on a.k1 < b.k1 or a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join36"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join36"""select ${i} from test a join baseall b on a.k1 = b.k1 or a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join37"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join37"""select ${i} from test a join baseall b on a.k1 = b.k1 or a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join38"""select ${i} from ${tbName1} a join ${tbName2} b + qt_join38"""select ${i} from test a join baseall b on a.k1 < b.k1 or a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join39"""select ${i} from ${tbName1} a join ${tbName2} b on a.k1 = b.k1 - join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join40"""select ${i} from ${tbName1} a join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 + qt_join39"""select ${i} from test a join baseall b on a.k1 = b.k1 + join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + qt_join40"""select ${i} from test a join baseall b on a.k2 = b.k2 and a.k1 > 0 + join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by 1, 2, 3, 4, 5 limit 65535""" // test_inner_join - qt_inner_join1"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join1"""select ${i} from test a inner join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join2"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join2"""select ${i} from test a inner join baseall b on a.k1 > b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join3"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join3"""select ${i} from test a inner join baseall b on a.k1 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join4"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join4"""select ${i} from test a inner join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join5"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join5"""select ${i} from test a inner join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join6"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join6"""select ${i} from test a inner join baseall b order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join7"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join7"""select ${i} from test a inner join baseall b on a.k1 = b.k1 or a.k2 = b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join8"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join8"""select ${i} from test a inner join baseall b on a.k1 < b.k1 or a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join9"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join9"""select ${i} from test a inner join baseall b on a.k1 = b.k1 or a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join10"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join10"""select ${i} from test a inner join baseall b on a.k1 = b.k1 or a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join11"""select ${i} from ${tbName1} a inner join ${tbName2} b + qt_inner_join11"""select ${i} from test a inner join baseall b on a.k1 < b.k1 or a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join12"""select ${i} from ${tbName1} a inner join ${tbName2} b on a.k1 = b.k1 - inner join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_inner_join13"""select ${i} from ${tbName1} a inner join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - inner join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 + qt_inner_join12"""select ${i} from test a inner join baseall b on a.k1 = b.k1 + inner join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + qt_inner_join13"""select ${i} from test a inner join baseall b on a.k2 = b.k2 and a.k1 > 0 + inner join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by 1, 2, 3, 4, 5 limit 65535""" } // test_left_join String i = "a.k1, b.k1, a.k2, b.k2, a.k3, b.k3" - qt_left_join1"""select ${i} from ${tbName1} a left join ${tbName2} b + qt_left_join1"""select ${i} from test a left join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, isnull(b.k1), 2, 3, 4, 5 limit 65535""" - qt_left_join2"""select ${i} from ${tbName1} a left join ${tbName2} b + qt_left_join2"""select ${i} from test a left join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by 1, isnull(b.k1), 2, 3, 4, 5 limit 65535""" - qt_left_join3"""select ${i} from ${tbName1} a left join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - left join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 + qt_left_join3"""select ${i} from test a left join baseall b on a.k2 = b.k2 and a.k1 > 0 + left join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by 1, isnull(b.k1), 2, 3, 4, 5 limit 65535""" for (s in selected) { - qt_left_join4"""select ${s} from ${tbName1} a left join ${tbName2} b + qt_left_join4"""select ${s} from test a left join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left join ${tbName2} b + sql"""select ${s} from test a left join baseall b on a.k1 > b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left join ${tbName2} b + sql"""select ${s} from test a left join baseall b on a.k1 > 0 order by 1, 2, 3, 4, 5 limit 65535""" test { - sql"""select ${s} from ${tbName1} a left join ${tbName2} b + sql"""select ${s} from test a left join baseall b order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } - sql"""select ${s} from ${tbName1} a left join ${tbName2} b + sql"""select ${s} from test a left join baseall b on a.k1 = b.k1 or a.k2 = b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left join ${tbName2} b + sql"""select ${s} from test a left join baseall b on a.k1 < b.k1 or a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left join ${tbName2} b + sql"""select ${s} from test a left join baseall b on a.k1 = b.k1 or a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left join ${tbName2} b + sql"""select ${s} from test a left join baseall b on a.k1 = b.k1 or a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left join ${tbName2} b + sql"""select ${s} from test a left join baseall b on a.k1 < b.k1 or a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - qt_left_join5"""select ${s} from ${tbName1} a left join ${tbName2} b on a.k1 = b.k1 - left join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + qt_left_join5"""select ${s} from test a left join baseall b on a.k1 = b.k1 + left join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" } // left_outer_join - qt_left_outer_join1"""select ${i} from ${tbName1} a left outer join ${tbName2} b + qt_left_outer_join1"""select ${i} from test a left outer join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, isnull(b.k1), 2, 3, 4, 5 limit 65535""" - qt_left_outer_join2"""select ${i} from ${tbName1} a left outer join ${tbName2} b + qt_left_outer_join2"""select ${i} from test a left outer join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by 1, isnull(b.k1), 2, 3, 4, 5 limit 65535""" - qt_left_outer_join3"""select ${i} from ${tbName1} a left outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - left join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 + qt_left_outer_join3"""select ${i} from test a left outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + left join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by 1, isnull(b.k1), 2, 3, 4, 5 limit 65535""" for (s in selected) { - qt_left_outer_join4"""select ${s} from ${tbName1} a left outer join ${tbName2} b + qt_left_outer_join4"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + sql"""select ${s} from test a left outer join baseall b on a.k1 > b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + sql"""select ${s} from test a left outer join baseall b on a.k1 > 0 order by 1, 2, 3, 4, 5 limit 65535""" test { - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + sql"""select ${s} from test a left outer join baseall b where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + sql"""select ${s} from test a left outer join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + sql"""select ${s} from test a left outer join baseall b on a.k1 < b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_left_outer_join5"""select ${s} from ${tbName1} a left outer join ${tbName2} b on a.k1 = b.k1 - left outer join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + qt_left_outer_join5"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 + left outer join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" } // right join - qt_right_join1"""select ${i} from ${tbName1} a right join ${tbName2} b + qt_right_join1"""select ${i} from test a right join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by isnull(a.k1), 1, 2, 3, 4, 5 limit 65535""" for (s in selected) { - qt_right_join2"""select ${s} from ${tbName1} a right join ${tbName2} b + qt_right_join2"""select ${s} from test a right join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right join ${tbName2} b + sql"""select ${s} from test a right join baseall b on a.k1 > b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right join ${tbName2} b + sql"""select ${s} from test a right join baseall b on a.k1 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_right_join3"""select ${s} from ${tbName1} a right join ${tbName2} b + qt_right_join3"""select ${s} from test a right join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" test { - sql"""select ${s} from ${tbName1} a right join ${tbName2} b + sql"""select ${s} from test a right join baseall b where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> logger.info(exception.message) assertTrue(exception != null) } } - sql"""select ${s} from ${tbName1} a right join ${tbName2} b + sql"""select ${s} from test a right join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right join ${tbName2} b + sql"""select ${s} from test a right join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right join ${tbName2} b + sql"""select ${s} from test a right join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right join ${tbName2} b + sql"""select ${s} from test a right join baseall b on a.k1 = b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right join ${tbName2} b + sql"""select ${s} from test a right join baseall b on a.k1 < b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_right_join4"""select ${s} from ${tbName1} a right join ${tbName2} b on a.k1 = b.k1 - right join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + qt_right_join4"""select ${s} from test a right join baseall b on a.k1 = b.k1 + right join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" } - qt_right_join5"""select ${i} from ${tbName1} a right join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - right join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 + qt_right_join5"""select ${i} from test a right join baseall b on a.k2 = b.k2 and a.k1 > 0 + right join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by isnull(a.k1), 1, 2, 3, 4, 5 limit 65535""" // right outer join - qt_right_outer_join1"""select ${i} from ${tbName1} a right outer join ${tbName2} b + qt_right_outer_join1"""select ${i} from test a right outer join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by isnull(a.k1), 1, 2, 3, 4, 5 limit 65535""" for (s in selected) { - qt_right_outer_join2"""select ${s} from ${tbName1} a right outer join ${tbName2} b + qt_right_outer_join2"""select ${s} from test a right outer join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right outer join ${tbName2} b + sql"""select ${s} from test a right outer join baseall b on a.k1 > b.k1 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right outer join ${tbName2} b + sql"""select ${s} from test a right outer join baseall b on a.k1 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_right_outer_join3"""select ${s} from ${tbName1} a right outer join ${tbName2} b + qt_right_outer_join3"""select ${s} from test a right outer join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" test { - sql"""select ${s} from ${tbName1} a right outer join ${tbName2} b + sql"""select ${s} from test a right outer join baseall b where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } - sql"""select ${s} from ${tbName1} a right outer join ${tbName2} b + sql"""select ${s} from test a right outer join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right outer join ${tbName2} b + sql"""select ${s} from test a right outer join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right outer join ${tbName2} b + sql"""select ${s} from test a right outer join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right outer join ${tbName2} b + sql"""select ${s} from test a right outer join baseall b on a.k1 = b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right outer join ${tbName2} b + sql"""select ${s} from test a right outer join baseall b on a.k1 < b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_right_outer_join4"""select ${s} from ${tbName1} a right outer join ${tbName2} b on a.k1 = b.k1 - right outer join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + qt_right_outer_join4"""select ${s} from test a right outer join baseall b on a.k1 = b.k1 + right outer join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" } - qt_right_outer_join5"""select ${i} from ${tbName1} a right outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - right outer join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 + qt_right_outer_join5"""select ${i} from test a right outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + right outer join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by isnull(a.k1), 1, 2, 3, 4, 5 limit 65535""" // right outer join with other join predicates @@ -351,71 +347,71 @@ suite("test_join", "query,p0") { // full outer join for (s in selected) { - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b on a.k1 = b.k1 + sql"""select ${s} from test a full outer join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b on a.k1 = b.k1 + sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b on a.k1 > b.k1 + sql"""select ${s} from test a full outer join baseall b on a.k1 > b.k1 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b on a.k1 > 0 + sql"""select ${s} from test a full outer join baseall b on a.k1 > 0 order by 1, 2, 3, 4, 5 limit 65535""" test { - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b + sql"""select ${s} from test a full outer join baseall b where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b + sql"""select ${s} from test a full outer join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b + sql"""select ${s} from test a full outer join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b + sql"""select ${s} from test a full outer join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b + sql"""select ${s} from test a full outer join baseall b on a.k1 = b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b + sql"""select ${s} from test a full outer join baseall b on a.k1 < b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a full outer join ${tbName2} b on a.k1 = b.k1 - full outer join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b on a.k1 = b.k1 - left outer join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + sql"""select ${s} from test a full outer join baseall b on a.k1 = b.k1 + full outer join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 + left outer join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" } - sql"""select a.k1 k1, a.k2, a.k3, b.k1, b.k2, b.k3 from ${tbName1} a full outer join ${tbName2} b + sql"""select a.k1 k1, a.k2, a.k3, b.k1, b.k2, b.k3 from test a full outer join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by isnull(k1), 1, 2, 3, 4, 5 limit 65535""" - sql"""select a.k1 ak1, a.k2 ak2, a.k3 ak3, b.k1 bk1, b.k2 bk2, b.k3 bk3 from ${tbName1} a left outer join ${tbName2} b + sql"""select a.k1 ak1, a.k2 ak2, a.k3 ak3, b.k1 bk1, b.k2 bk2, b.k3 bk3 from test a left outer join baseall b on a.k1 = b.k1 and a.k2 > b.k2 union (select a.k1 ak1, a.k2 ak2, a.k3 ak3, b.k1 bk1, b.k2 bk2, b.k3 bk3 - from ${tbName1} a right outer join ${tbName2} b on a.k1 = b.k1 and a.k2 > b.k2) + from test a right outer join baseall b on a.k1 = b.k1 and a.k2 > b.k2) order by isnull(ak1), 1, 2, 3, 4, 5 limit 65535""" - sql"""select count(*) from ${tbName1} a full outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - full outer join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0""" - sql"""select count(*) from ((select a.k1 as k1, b.k1 as k2, a.k2 as k3, b.k2 as k4, a.k3 as k5, b.k3 as k6, c.k1 as k7, c.k2 as k8, c.k3 as k9 from ${tbName1} a - left outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - left outer join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0) union - (select a.k1, b.k1, a.k2, b.k2, a.k3, b.k3, c.k1, c.k2, c.k3 from ${tbName1} a - left outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - right outer join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0) union - (select a.k1, b.k1, a.k2, b.k2, a.k3, b.k3, c.k1, c.k2, c.k3 from ${tbName1} a - right outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - left outer join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0) - union (select a.k1, b.k1, a.k2, b.k2, a.k3, b.k3, c.k1, c.k2, c.k3 from ${tbName1} a - right outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - right outer join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0))a""" - sql"""select ${i} from ${tbName1} a full outer join ${tbName2} b on a.k1 = b.k1 + sql"""select count(*) from test a full outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + full outer join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0""" + sql"""select count(*) from ((select a.k1 as k1, b.k1 as k2, a.k2 as k3, b.k2 as k4, a.k3 as k5, b.k3 as k6, c.k1 as k7, c.k2 as k8, c.k3 as k9 from test a + left outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + left outer join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0) union + (select a.k1, b.k1, a.k2, b.k2, a.k3, b.k3, c.k1, c.k2, c.k3 from test a + left outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + right outer join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0) union + (select a.k1, b.k1, a.k2, b.k2, a.k3, b.k3, c.k1, c.k2, c.k3 from test a + right outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + left outer join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0) + union (select a.k1, b.k1, a.k2, b.k2, a.k3, b.k3, c.k1, c.k2, c.k3 from test a + right outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + right outer join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 and c.k3 > 0))a""" + sql"""select ${i} from test a full outer join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, isnull(b.k1), 2, 3, 4, 5 limit 65535""" - sql"""select ${i} from ${tbName1} a left outer join ${tbName2} b on a.k1 = b.k1 and a.k2 > 0 + sql"""select ${i} from test a left outer join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, isnull(b.k1), 2, 3, 4, 5 limit 65535""" // cross join for (s in selected){ test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 = b.k1 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -424,7 +420,7 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 > b.k1 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -433,7 +429,7 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -442,7 +438,7 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 = b.k1 and a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -451,7 +447,7 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 = b.k1 and a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -459,10 +455,10 @@ suite("test_join", "query,p0") { logger.info(exception.message) } } - qt_cross_join1"""select ${s} from ${tbName1} a cross join ${tbName2} b + qt_cross_join1"""select ${s} from test a cross join baseall b where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -471,7 +467,7 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -480,7 +476,7 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -489,7 +485,7 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 = b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -498,7 +494,7 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b + sql"""select ${s} from test a cross join baseall b on a.k1 < b.k1 or a.k2 > 0 where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> @@ -507,16 +503,16 @@ suite("test_join", "query,p0") { } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b on a.k1 = b.k1 - cross join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + sql"""select ${s} from test a cross join baseall b on a.k1 = b.k1 + cross join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } test { - sql"""select ${s} from ${tbName1} a cross join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - cross join ${tbName3} c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 + sql"""select ${s} from test a cross join baseall b on a.k2 = b.k2 and a.k1 > 0 + cross join bigtable c on a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) @@ -529,129 +525,129 @@ suite("test_join", "query,p0") { // left_semi_join List left_selected = ["a.k1, a.k2, a.k3, a.k4, a.k5", "count(a.k1), count(a.k2), count(a.k4), count(a.k3), count(*)"] for (s in left_selected){ - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b on a.k1 = b.k1 + sql"""select ${s} from test a left semi join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b on a.k1 = b.k1 + sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 where b.k3 is not null order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b on a.k1 > b.k1 + sql"""select ${s} from test a left semi join baseall b on a.k1 > b.k1 where a.k2 > 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b on a.k1 > 0 + sql"""select ${s} from test a left semi join baseall b on a.k1 > 0 where a.k2 > 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - def res15 = sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b + def res15 = sql"""select ${s} from test a left semi join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - def res16 = sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + def res16 = sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 and a.k2 > 0 where b.k3 is not null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res15, res16) - def res17 = sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b + def res17 = sql"""select ${s} from test a left semi join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - def res18 = sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + def res18 = sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 and a.k2 > b.k2 where b.k3 is not null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res17, res18) test { - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b + sql"""select ${s} from test a left semi join baseall b where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b + sql"""select ${s} from test a left semi join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where a.k2 > 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b + sql"""select ${s} from test a left semi join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where a.k2 > 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b + sql"""select ${s} from test a left semi join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where a.k2 > 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b + sql"""select ${s} from test a left semi join baseall b on a.k1 = b.k1 or a.k2 > 0 where a.k2 > 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b + sql"""select ${s} from test a left semi join baseall b on a.k1 < b.k1 or a.k2 > 0 where a.k2 > 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - def res19 = sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b on a.k1 = b.k1 - left semi join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" - def res20 = sql"""select ${s} from (select distinct a.* from ${tbName1} a left outer join ${tbName2} b on a.k1 = b.k1 - left outer join ${tbName3} c on a.k2 = c.k2 where a.k1 is not null + def res19 = sql"""select ${s} from test a left semi join baseall b on a.k1 = b.k1 + left semi join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + def res20 = sql"""select ${s} from (select distinct a.* from test a left outer join baseall b on a.k1 = b.k1 + left outer join bigtable c on a.k2 = c.k2 where a.k1 is not null and b.k1 is not null and c.k1 is not null) a order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res19, res20) - def res21 = sql"""select ${s} from ${tbName1} a left semi join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - left semi join ${tbName3} c on a.k3 = c.k3 and a.k1 = c.k1 + 1 and c.k3 > 0 + def res21 = sql"""select ${s} from test a left semi join baseall b on a.k2 = b.k2 and a.k1 > 0 + left semi join bigtable c on a.k3 = c.k3 and a.k1 = c.k1 + 1 and c.k3 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - def res22 = sql"""select ${s} from (select distinct a.* from ${tbName1} a left outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - left outer join ${tbName3} c on a.k3 = c.k3 and a.k1 = c.k1 + 1 and c.k3 > 0 + def res22 = sql"""select ${s} from (select distinct a.* from test a left outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + left outer join bigtable c on a.k3 = c.k3 and a.k1 = c.k1 + 1 and c.k3 > 0 where a.k1 is not null and b.k1 is not null and c.k1 is not null and a.k1 > 0 and c.k3 > 0) a order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res21, res22) } qt_cross_join2 """ - select t1.k1 from ${tbName1} t1 cross join ${tbName1} t2 where t1.k1 = t2.k1 + 1 group by t1.k1 order by t1.k1; + select t1.k1 from test t1 cross join test t2 where t1.k1 = t2.k1 + 1 group by t1.k1 order by t1.k1; """ // right semi join List right_selected = ["b.k1, b.k2, b.k3, b.k4, b.k5", "count(b.k1), count(b.k2), count(b.k4), count(b.k3), count(*)"] for (s in right_selected){ - def res23 = sql"""select ${s} from ${tbName1} a right semi join ${tbName1} b + def res23 = sql"""select ${s} from test a right semi join test b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - def res24 = sql"""select ${s} from ${tbName1} a right outer join ${tbName1} b + def res24 = sql"""select ${s} from test a right outer join test b on a.k1 = b.k1 where a.k2 is not null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res23, res24) - sql"""select ${s} from ${tbName1} a right semi join ${tbName2} b + sql"""select ${s} from test a right semi join baseall b on a.k1 > b.k1 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right semi join ${tbName2} b + sql"""select ${s} from test a right semi join baseall b on a.k1 > 0 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - def res25 = sql"""select ${s} from ${tbName2} a right semi join ${tbName1} b + def res25 = sql"""select ${s} from baseall a right semi join test b on a.k1 = b.k1 and a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - def res26 = sql"""select ${s} from ${tbName2} a right outer join ${tbName1} b on a.k1 = b.k1 and + def res26 = sql"""select ${s} from baseall a right outer join test b on a.k1 = b.k1 and a.k2 > 0 where a.k2 is not null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res25, res26) - def res27 = sql"""select ${s} from ${tbName2} a right semi join ${tbName1} b + def res27 = sql"""select ${s} from baseall a right semi join test b on a.k1 = b.k1 and a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - def res28 = sql"""select ${s} from ${tbName2} a right outer join ${tbName1} b + def res28 = sql"""select ${s} from baseall a right outer join test b on a.k1 = b.k1 and a.k2 > b.k2 where a.k2 is not null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res27, res28) test { - sql"""select ${s} from ${tbName1} a right semi join ${tbName2} b + sql"""select ${s} from test a right semi join baseall b where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } - sql"""select ${s} from ${tbName1} a right semi join ${tbName2} b + sql"""select ${s} from test a right semi join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right semi join ${tbName2} b + sql"""select ${s} from test a right semi join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right semi join ${tbName2} b + sql"""select ${s} from test a right semi join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right semi join ${tbName2} b + sql"""select ${s} from test a right semi join baseall b on a.k1 = b.k1 or a.k2 > 0 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right semi join ${tbName2} b + sql"""select ${s} from test a right semi join baseall b on a.k1 < b.k1 or a.k2 > 0 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - def res29 = sql"""select ${s} from ${tbName3} a right semi join ${tbName1} c on a.k1 = c.k1 - right semi join ${tbName2} b on b.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + def res29 = sql"""select ${s} from bigtable a right semi join test c on a.k1 = c.k1 + right semi join baseall b on b.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" - def res30 = sql"""select ${s} from (select distinct b.* from ${tbName3} a right outer join ${tbName1} c on a.k1 = c.k1 - right outer join ${tbName2} b on b.k2 = c.k2 where a.k1 is not null + def res30 = sql"""select ${s} from (select distinct b.* from bigtable a right outer join test c on a.k1 = c.k1 + right outer join baseall b on b.k2 = c.k2 where a.k1 is not null and b.k1 is not null and c.k1 is not null) b order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res29, res30) - def res31 = sql"""select ${s} from ${tbName2} c right semi join ${tbName1} a on c.k2 = a.k2 and c.k1 > 0 - right semi join ${tbName3} b on a.k3 = b.k3 and b.k1 = a.k1 + 1 and a.k3 > 0 + def res31 = sql"""select ${s} from baseall c right semi join test a on c.k2 = a.k2 and c.k1 > 0 + right semi join bigtable b on a.k3 = b.k3 and b.k1 = a.k1 + 1 and a.k3 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - def res32 = sql"""select ${s} from (select distinct a.* from ${tbName2} c right outer join ${tbName1} b1 on c.k2 = b1.k2 and c.k1 > 0 - right outer join ${tbName3} a on c.k3 = a.k3 and a.k1 = c.k1 + 1 and a.k3 > 0 + def res32 = sql"""select ${s} from (select distinct a.* from baseall c right outer join test b1 on c.k2 = b1.k2 and c.k1 > 0 + right outer join bigtable a on c.k3 = a.k3 and a.k1 = c.k1 + 1 and a.k3 > 0 where a.k1 is not null and b1.k1 is not null and a.k1 is not null and a.k1 > 0 and c.k3 > 0) b order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res31, res32) @@ -659,212 +655,212 @@ suite("test_join", "query,p0") { // left anti join for (s in left_selected){ - def res33 = sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + def res33 = sql"""select ${s} from test a left anti join baseall b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - def res34 = sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + def res34 = sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 where b.k3 is null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res33, res34) - sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + sql"""select ${s} from test a left anti join baseall b on a.k1 > b.k1 where a.k2 > 0 and a.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + sql"""select ${s} from test a left anti join baseall b on a.k1 > 0 where a.k2 > 0 and a.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - def res35 = sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + def res35 = sql"""select ${s} from test a left anti join baseall b on a.k1 = b.k1 and a.k2 > 0 order by 1, 2, 3, 4, 5 limit 50000""" - def res36 = sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + def res36 = sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 and a.k2 > 0 where b.k3 is null order by 1, 2, 3, 4, 5 limit 50000""" check2_doris(res35, res36) - def res37 = sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + def res37 = sql"""select ${s} from test a left anti join baseall b on a.k1 = b.k1 and a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - def res38 = sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b + def res38 = sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 and a.k2 > b.k2 where b.k3 is null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res37, res38) test { - sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + sql"""select ${s} from test a left anti join baseall b where a.k2 > 0 and a.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } - sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + sql"""select ${s} from test a left anti join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where a.k2 > 0 and a.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + sql"""select ${s} from test a left anti join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where a.k2 > 0 and a.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + sql"""select ${s} from test a left anti join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where a.k2 > 0 and a.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + sql"""select ${s} from test a left anti join baseall b on a.k1 = b.k1 or a.k2 > 0 where a.k2 > 0 and a.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b + sql"""select ${s} from test a left anti join baseall b on a.k1 < b.k1 or a.k2 > 0 where a.k2 > 0 and a.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - def res39 = sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b on a.k1 = b.k1 - left anti join ${tbName3} c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" - def res40 = sql"""select ${s} from ${tbName1} a left outer join ${tbName2} b on a.k1 = b.k1 - left outer join ${tbName3} c on a.k2 = c.k2 where + def res39 = sql"""select ${s} from test a left anti join baseall b on a.k1 = b.k1 + left anti join bigtable c on a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" + def res40 = sql"""select ${s} from test a left outer join baseall b on a.k1 = b.k1 + left outer join bigtable c on a.k2 = c.k2 where b.k1 is null and c.k1 is null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res39, res40) - def res41 = sql"""select ${s} from ${tbName1} a left anti join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - left anti join ${tbName3} c on a.k3 = c.k3 and a.k1 = c.k1 + 1 and c.k3 > 0 + def res41 = sql"""select ${s} from test a left anti join baseall b on a.k2 = b.k2 and a.k1 > 0 + left anti join bigtable c on a.k3 = c.k3 and a.k1 = c.k1 + 1 and c.k3 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - def res42 = sql"""select ${s} from (select distinct a.* from ${tbName1} a left outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - left outer join ${tbName3} c on a.k3 = c.k3 and a.k1 = c.k1 + 1 and c.k3 > 0 + def res42 = sql"""select ${s} from (select distinct a.* from test a left outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + left outer join bigtable c on a.k3 = c.k3 and a.k1 = c.k1 + 1 and c.k3 > 0 where b.k1 is null and c.k1 is null) a order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res41, res42) } - qt_left_anti_join_with_other_pred "select b.k1 from ${tbName2} b left anti join ${tbName1} t on b.k1 = t.k1 and 1 = 2 order by b.k1" + qt_left_anti_join_with_other_pred "select b.k1 from baseall b left anti join test t on b.k1 = t.k1 and 1 = 2 order by b.k1" - qt_left_anti_join_null_1 "select b.k1 from ${tbName2} b left anti join ${tbName1} t on b.k1 = t.k1 order by b.k1" + qt_left_anti_join_null_1 "select b.k1 from baseall b left anti join test t on b.k1 = t.k1 order by b.k1" - qt_left_anti_join_null_2 "select b.k1 from ${tbName2} b left anti join ${empty_name} t on b.k1 = t.k1 order by b.k1" + qt_left_anti_join_null_2 "select b.k1 from baseall b left anti join test_join_empty_view t on b.k1 = t.k1 order by b.k1" - qt_left_anti_join_null_3 "select b.k1 from ${tbName2} b left anti join ${tbName1} t on b.k1 > t.k2 order by b.k1" + qt_left_anti_join_null_3 "select b.k1 from baseall b left anti join test t on b.k1 > t.k2 order by b.k1" - qt_left_anti_join_null_4 "select b.k1 from ${tbName2} b left anti join ${empty_name} t on b.k1 > t.k2 order by b.k1" + qt_left_anti_join_null_4 "select b.k1 from baseall b left anti join test_join_empty_view t on b.k1 > t.k2 order by b.k1" // right anti join for (s in right_selected){ - def res43 = sql"""select ${s} from ${tbName2} a right anti join ${tbName1} b + def res43 = sql"""select ${s} from baseall a right anti join test b on a.k1 = b.k1 order by 1, 2, 3, 4, 5 limit 65535""" - def res44 = sql"""select ${s} from ${tbName2} a right outer join ${tbName1} b + def res44 = sql"""select ${s} from baseall a right outer join test b on a.k1 = b.k1 where a.k2 is null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res43, res44) - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} b + sql"""select ${s} from test a right anti join baseall b on a.k1 > b.k1 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} b + sql"""select ${s} from test a right anti join baseall b on a.k1 > 0 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - def res45 = sql"""select ${s} from ${tbName2} a right anti join ${tbName1} b + def res45 = sql"""select ${s} from baseall a right anti join test b on a.k1 = b.k1 and a.k2 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - def res46 = sql"""select ${s} from ${tbName2} a right outer join ${tbName1} b + def res46 = sql"""select ${s} from baseall a right outer join test b on a.k1 = b.k1 and a.k2 > 0 where a.k2 is null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res45 , res46) - def res47 = sql"""select ${s} from ${tbName2} a right anti join ${tbName1} b + def res47 = sql"""select ${s} from baseall a right anti join test b on a.k1 = b.k1 and a.k2 > b.k2 order by 1, 2, 3, 4, 5 limit 65535""" - def res48 = sql"""select ${s} from ${tbName2} a right outer join ${tbName1} b + def res48 = sql"""select ${s} from baseall a right outer join test b on a.k1 = b.k1 and a.k2 > b.k2 where a.k2 is null order by 1, 2, 3, 4, 5 limit 65535""" check2_doris(res47, res48) test { - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} b + sql"""select ${s} from test a right anti join baseall b where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" check{result, exception, startTime, endTime -> assertTrue(exception != null) logger.info(exception.message) } } - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} b + sql"""select ${s} from test a right anti join baseall b on a.k1 = b.k1 or a.k2 = b.k2 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} b + sql"""select ${s} from test a right anti join baseall b on a.k1 < b.k1 or a.k2 > b.k2 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} b + sql"""select ${s} from test a right anti join baseall b on a.k1 = b.k1 or a.k2 > b.k2 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} b + sql"""select ${s} from test a right anti join baseall b on a.k1 = b.k1 or a.k2 > 0 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} b + sql"""select ${s} from test a right anti join baseall b on a.k1 < b.k1 or a.k2 > 0 where b.k2 > 0 and b.k3 != 0 and b.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} c on a.k1 = c.k1 - right anti join ${tbName3} b on c.k2 = b.k2 order by 1, 2, 3, 4, 5 limit 65535""" + sql"""select ${s} from test a right anti join baseall c on a.k1 = c.k1 + right anti join bigtable b on c.k2 = b.k2 order by 1, 2, 3, 4, 5 limit 65535""" sql"""select ${s} from (select distinct b.k1, b.k2, b.k3, b.k4, b.k5 from - ${tbName1} a right outer join ${tbName2} c on a.k1 = c.k1 right outer join - ${tbName3} b on c.k2=b.k2) b order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from ${tbName1} a right anti join ${tbName2} c on a.k2 = c.k2 and a.k1 > 0 - right anti join ${tbName3} b on c.k3 = b.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 + test a right outer join baseall c on a.k1 = c.k1 right outer join + bigtable b on c.k2=b.k2) b order by 1, 2, 3, 4, 5 limit 65535""" + sql"""select ${s} from test a right anti join baseall c on a.k2 = c.k2 and a.k1 > 0 + right anti join bigtable b on c.k3 = b.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by 1, 2, 3, 4, 5 limit 65535""" - sql"""select ${s} from (select distinct c.* from ${tbName1} a right outer join ${tbName2} b on a.k2 = b.k2 and a.k1 > 0 - right outer join ${tbName3} c on b.k3 = c.k3 and c.k1 = b.k1 + 1 and c.k3 > 0 + sql"""select ${s} from (select distinct c.* from test a right outer join baseall b on a.k2 = b.k2 and a.k1 > 0 + right outer join bigtable c on b.k3 = c.k3 and c.k1 = b.k1 + 1 and c.k3 > 0 where b.k1 is null and a.k1 is null and a.k1 > 0) b order by 1, 2, 3, 4, 5 limit 65535""" sql"""select ${s} from (select distinct c.k1 k1, c.k2 k2, c.k3 k3, c.k4 k4, c.k5 k5 from - (select b2.* from ${tbName1} a right outer join ${tbName2} b2 on a.k2 = b2.k2 and a.k1 > 0 - where a.k1 is null and a.k1 > 0) b1 right outer join ${tbName3} c + (select b2.* from test a right outer join baseall b2 on a.k2 = b2.k2 and a.k1 > 0 + where a.k1 is null and a.k1 > 0) b1 right outer join bigtable c on b1.k3 = c.k3 and c.k1 = b1.k1 + 1 and c.k3 > 0 where b1.k1 is null) b order by 1, 2, 3, 4, 5 limit 65535""" } - qt_right_anti_join_with_other_pred "select t.k1 from ${tbName2} b right anti join ${tbName1} t on b.k1 = t.k1 and 1 = 2 order by t.k1" + qt_right_anti_join_with_other_pred "select t.k1 from baseall b right anti join test t on b.k1 = t.k1 and 1 = 2 order by t.k1" - qt_right_anti_join_null_1 "select b.k1 from ${tbName1} t right anti join ${tbName2} b on b.k1 > t.k1 order by b.k1" + qt_right_anti_join_null_1 "select b.k1 from test t right anti join baseall b on b.k1 > t.k1 order by b.k1" - qt_right_anti_join_null_2 "select /*+SET_VAR(batch_size=3) */ b.k1 from ${empty_name} t right anti join ${tbName2} b on b.k1 > t.k1 order by b.k1" + qt_right_anti_join_null_2 "select /*+SET_VAR(batch_size=3) */ b.k1 from test_join_empty_view t right anti join baseall b on b.k1 > t.k1 order by b.k1" // join with no join keyword for (s in selected){ - qt_join_without_keyword1"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword1"""select ${s} from test a , baseall b where a.k1 = b.k1 and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword2"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword2"""select ${s} from test a , baseall b where a.k1 > b.k1 and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword3"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword3"""select ${s} from test a , baseall b where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword4"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword4"""select ${s} from test a , baseall b where a.k1 = b.k1 and a.k2 > 0 and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword5"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword5"""select ${s} from test a , baseall b where a.k1 = b.k1 and a.k2 > b.k2 and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword6"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword6"""select ${s} from test a , baseall b where a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword7"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword7"""select ${s} from test a , baseall b where (a.k1 = b.k1 or a.k2 = b.k2) and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword8"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword8"""select ${s} from test a , baseall b where (a.k1 < b.k1 or a.k2 > b.k2) and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword9"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword9"""select ${s} from test a , baseall b where (a.k1 = b.k1 or a.k2 > b.k2) and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword10"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword10"""select ${s} from test a , baseall b where (a.k1 = b.k1 or a.k2 > 0) and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword11"""select ${s} from ${tbName1} a , ${tbName2} b + qt_join_without_keyword11"""select ${s} from test a , baseall b where (a.k1 < b.k1 or a.k2 > 0) and a.k2 > 0 and b.k3 != 0 and a.k6 > "000" order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword12"""select ${s} from ${tbName1} a, ${tbName2} b, ${tbName3} c where a.k1 = b.k1 + qt_join_without_keyword12"""select ${s} from test a, baseall b, bigtable c where a.k1 = b.k1 and a.k2 = c.k2 order by 1, 2, 3, 4, 5 limit 65535""" - qt_join_without_keyword13"""select ${s} from ${tbName1} a, ${tbName2} b, ${tbName3} c where a.k2 = b.k2 and a.k1 > 0 + qt_join_without_keyword13"""select ${s} from test a, baseall b, bigtable c where a.k2 = b.k2 and a.k1 > 0 and a.k3 = c.k3 and b.k1 = c.k1 + 1 and c.k3 > 0 order by 1, 2, 3, 4, 5 limit 65535""" } // join with empty table - qt_join_with_emptyTable1"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from ${tbName2} a join ${empty_name} b on a.k1 = b.k1 + qt_join_with_emptyTable1"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from baseall a join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3, 4, 5""" - qt_join_with_emptyTable2"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from ${tbName2} a inner join ${empty_name} b on a.k1 = b.k1 + qt_join_with_emptyTable2"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from baseall a inner join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3, 4, 5""" - qt_join_with_emptyTable3"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from ${tbName2} a left join ${empty_name} b on a.k1 = b.k1 + qt_join_with_emptyTable3"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from baseall a left join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3, 4, 5""" - qt_join_with_emptyTable4"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from ${tbName2} a right join ${empty_name} b on a.k1 = b.k1 + qt_join_with_emptyTable4"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from baseall a right join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3, 4, 5""" - def res53 = sql"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from ${tbName2} a full outer join ${empty_name} b on a.k1 = b.k1 + def res53 = sql"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from baseall a full outer join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3, 4, 5""" - def res54 = sql"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from ${tbName2} a left join ${empty_name} b on a.k1 = b.k1 + def res54 = sql"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from baseall a left join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3, 4, 5""" check2_doris(res53, res54) - // qt_join_with_emptyTable5"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from ${tbName2} a cross join ${empty_name} b on a.k1 = b.k1 + // qt_join_with_emptyTable5"""select a.k1, a.k2, a.k3, b.k1, b.k2, b.k3 from baseall a cross join test_join_empty_view b on a.k1 = b.k1 // order by 1, 2, 3, 4, 5""" test { - sql"""select a.k1, a.k2, a.k3 from ${tbName2} a left semi join ${empty_name} b on a.k1 = b.k1 + sql"""select a.k1, a.k2, a.k3 from baseall a left semi join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3""" check{result, exception, startTime, endTime -> logger.info(result.toString()) @@ -872,18 +868,18 @@ suite("test_join", "query,p0") { } } test { - sql"""select b.k1, b.k2, b.k3 from ${tbName2} a right semi join ${empty_name} b on a.k1 = b.k1 + sql"""select b.k1, b.k2, b.k3 from baseall a right semi join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3""" check{result, exception, startTime, endTime -> assertTrue(result.isEmpty()) } } - def res55 = sql"""select a.k1, a.k2, a.k3 from ${tbName2} a left anti join ${empty_name} b on a.k1 = b.k1 + def res55 = sql"""select a.k1, a.k2, a.k3 from baseall a left anti join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3""" - def res56 = sql"""select k1, k2, k3 from ${tbName2} order by 1, 2, 3""" + def res56 = sql"""select k1, k2, k3 from baseall order by 1, 2, 3""" check2_doris(res55, res56) test { - sql"""select b.k1, b.k2, b.k3 from ${tbName2} a right anti join ${empty_name} b on a.k1 = b.k1 + sql"""select b.k1, b.k2, b.k3 from baseall a right anti join test_join_empty_view b on a.k1 = b.k1 order by 1, 2, 3""" check{result, exception, startTime, endTime -> assertTrue(result.isEmpty()) @@ -941,46 +937,46 @@ suite("test_join", "query,p0") { List join_types = ["inner", "left outer", "right outer", ""] for (type in join_types) { for (c in columns) { - qt_join_basic1"""select * from ${tbName2} a ${type} join ${tbName1} b on (a.${c} = b.${c}) + qt_join_basic1"""select * from baseall a ${type} join test b on (a.${c} = b.${c}) order by isnull(a.k1), a.k1, a.k2, a.k3, isnull(b.k1), b.k1, b.k2, b.k3 limit 60015""" } } for (c in columns){ - sql"""select * from ${tbName2} a full outer join ${tbName1} b on (a.${c} = b.${c}) + sql"""select * from baseall a full outer join test b on (a.${c} = b.${c}) order by isnull(a.k1), a.k1, a.k2, a.k3, a.k4, isnull(b.k1), b.k1, b.k2, b.k3, b.k4 limit 65535""" sql"""select a.k1 ak1, a.k2 ak2, a.k3 ak3, a.k4 ak4, a.k5 ak5, a.k6 ak7, a.k10 ak10, a.k11 ak11, a.k7 ak7, a.k8 ak8, a.k9 ak9, b.k1 bk1, b.k2 bk2, b.k3 bk3, b.k4 bk4, b.k5 bk5, b.k6 bk6, b.k10 bk10, b.k11 bk11, b.k7 bk7, b.k8 bk8, b.k9 bk9 - from ${tbName2} a left outer join ${tbName1} b on (a.${c} = b.${c}) + from baseall a left outer join test b on (a.${c} = b.${c}) union select a.k1 ak1, a.k2 ak2, a.k3 ak3, a.k4 ak4, a.k5 ak5, a.k6 ak6, a.k10 ak10, a.k11 ak11, a.k7 ak7, a.k8 ak8, a.k9 ak9, b.k1 bk1, b.k2 bk2, b.k3 bk3, b.k4 bk4, b.k5 bk5, b.k6 bk6, b.k10 bk10, b.k11 bk11, b.k7 bk7, b.k8 bk8, b.k9 bk9 from - ${tbName2} a right outer join ${tbName1} b on (a.${c} = b.${c}) order by + baseall a right outer join test b on (a.${c} = b.${c}) order by isnull(ak1), 1, 2, 3, 4, isnull(bk1), 12, 13, 14, 15 limit 65535""" - def res67 = sql"""select * from ${tbName2} a left semi join ${tbName1} b on (a.${c} = b.${c}) + def res67 = sql"""select * from baseall a left semi join test b on (a.${c} = b.${c}) order by a.k1, a.k2, a.k3""" - def res68 = sql"""select distinct a.* from ${tbName2} a left outer join ${tbName1} b on (a.${c} = b.${c}) + def res68 = sql"""select distinct a.* from baseall a left outer join test b on (a.${c} = b.${c}) where b.k1 is not null order by a.k1, a.k2, a.k3""" check2_doris(res67, res68) - def res69 = sql"""select * from ${tbName2} a right semi join ${tbName1} b on (a.${c} = b.${c}) + def res69 = sql"""select * from baseall a right semi join test b on (a.${c} = b.${c}) order by b.k1, b.k2, b.k3""" - def res70 = sql"""select distinct b.* from ${tbName2} a right outer join ${tbName1} b on (a.${c} = b.${c}) + def res70 = sql"""select distinct b.* from baseall a right outer join test b on (a.${c} = b.${c}) where a.k1 is not null order by b.k1, b.k2, b.k3""" check2_doris(res69, res70) - def res71 = sql"""select * from ${tbName2} a left anti join ${tbName1} b on (a.${c} = b.${c}) + def res71 = sql"""select * from baseall a left anti join test b on (a.${c} = b.${c}) order by a.k1, a.k2, a.k3""" - def res72 = sql"""select distinct a.* from ${tbName2} a left outer join ${tbName1} b on (a.${c} = b.${c}) + def res72 = sql"""select distinct a.* from baseall a left outer join test b on (a.${c} = b.${c}) where b.k1 is null order by a.k1, a.k2, a.k3""" check2_doris(res71, res72) - def res73 = sql"""select * from ${tbName2} a right anti join ${tbName1} b on (a.${c} = b.${c}) + def res73 = sql"""select * from baseall a right anti join test b on (a.${c} = b.${c}) order by b.k1, b.k2, b.k3""" - def res74 = sql"""select distinct b.* from ${tbName2} a right outer join ${tbName1} b on (a.${c} = b.${c}) + def res74 = sql"""select distinct b.* from baseall a right outer join test b on (a.${c} = b.${c}) where a.k1 is null order by b.k1, b.k2, b.k3""" check2_doris(res73, res74) } @@ -990,53 +986,53 @@ suite("test_join", "query,p0") { // complex join String col = "k1" for (t in join_types){ - qt_complex_join1"""select count(a.k1), count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a ${t} join (select k1, k2, k6 from ${tbName2} where k1 < 5 + qt_complex_join1"""select count(a.k1), count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a ${t} join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.${col} = b.${col})""" } - def res75 = sql"""select count(a.k1), count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a full outer join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res75 = sql"""select count(a.k1), count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a full outer join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1)""" def res76 = sql"""select count(c.k1), count(c.m1), count(*) from (select distinct a.*, b.* from (select k1 + 2 as m1, k2 + 1000 as m2, k6 as m6 - from ${tbName2} where k1 < 5 order by k1) a left outer join - (select k1, k2, k6 from ${tbName2} where k1 < 5 order by k1) b on (a.m1 = b.k1) + from baseall where k1 < 5 order by k1) a left outer join + (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.m1 = b.k1) union (select distinct a.*, b.* from - (select k1 + 2 as m1, k2 + 1000 as m2, k6 as m6 from ${tbName2} where k1 < 5 - order by k1) a right outer join (select k1, k2, k6 from ${tbName2} where k1 < 5 + (select k1 + 2 as m1, k2 + 1000 as m2, k6 as m6 from baseall where k1 < 5 + order by k1) a right outer join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.m1 = b.k1))) c""" check2_doris(res75, res76) - def res77 = sql"""select count(a.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a left semi join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res77 = sql"""select count(a.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a left semi join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1)""" - def res78 = sql"""select count(a.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a left outer join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res78 = sql"""select count(a.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a left outer join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1) where b.k1 is not null """ check2_doris(res77, res78) - def res79 = sql"""select count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a right semi join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res79 = sql"""select count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a right semi join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1) """ - def res80 = sql"""select count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a right outer join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res80 = sql"""select count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a right outer join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1) where a.k1 is not null""" check2_doris(res79, res80) - def res81 = sql"""select count(a.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a left anti join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res81 = sql"""select count(a.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a left anti join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1)""" - def res82 = sql"""select count(a.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a left outer join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res82 = sql"""select count(a.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a left outer join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1) where b.k1 is null""" check2_doris(res81, res82) - def res83 = sql"""select count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a right anti join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res83 = sql"""select count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a right anti join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1) """ - def res84 = sql"""select count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from ${tbName2} where k1 < 5 - order by k1) a right outer join (select k1, k2, k6 from ${tbName2} where k1 < 5 + def res84 = sql"""select count(b.k1), count(*) from (select k1 + 2 as k1, k2 + 1000 as k2, k6 from baseall where k1 < 5 + order by k1) a right outer join (select k1, k2, k6 from baseall where k1 < 5 order by k1) b on (a.k1 = b.k1) where a.k1 is null""" check2_doris(res83, res84) @@ -1047,58 +1043,58 @@ suite("test_join", "query,p0") { String null_name = "nullable" for (t in join_types){ - qt_join_multi_table1"""select * from ${tbName2} a ${t} join ${null_name} b on a.k1 = b.n1 order by + qt_join_multi_table1"""select * from baseall a ${t} join ${null_name} b on a.k1 = b.n1 order by a.k1, b.n1""" - qt_join_multi_table2"""select * from ${tbName2} a ${t} join ${null_name} b on a.k1 = b.n2 order by + qt_join_multi_table2"""select * from baseall a ${t} join ${null_name} b on a.k1 = b.n2 order by a.k1, b.n1""" } test { - sql"""select a.k1, a.k2 from ${tbName2} a left semi join ${null_name} b on a.k1 = b.n2 + sql"""select a.k1, a.k2 from baseall a left semi join ${null_name} b on a.k1 = b.n2 order by a.k1""" check{result, exception, startTime, endTime -> assertTrue(result.isEmpty()) } } test { - sql"""select b.n1, b.n2 from ${tbName2} a right semi join ${null_name} b on a.k1 = b.n2 + sql"""select b.n1, b.n2 from baseall a right semi join ${null_name} b on a.k1 = b.n2 order by b.n1""" check{result, exception, startTime, endTime -> assertTrue(result.isEmpty()) } } test { - sql"""select b.k1, b.k2 from ${null_name} a right semi join ${tbName2} b on b.k1 = a.n2 + sql"""select b.k1, b.k2 from ${null_name} a right semi join baseall b on b.k1 = a.n2 order by b.k1""" check{result, exception, startTime, endTime -> assertTrue(result.isEmpty()) } } test { - sql"""select a.n1, a.n2 from ${null_name} a left semi join ${tbName2} b on b.k1 = a.n2 + sql"""select a.n1, a.n2 from ${null_name} a left semi join baseall b on b.k1 = a.n2 order by 1, 2""" check{result, exception, startTime, endTime -> assertTrue(result.isEmpty()) } } - def res85 = sql"""select a.k1, a.k2 from ${tbName2} a left anti join ${null_name} b on a.k1 = b.n2 + def res85 = sql"""select a.k1, a.k2 from baseall a left anti join ${null_name} b on a.k1 = b.n2 order by 1, 2""" - def res86 = sql"""select k1, k2 from ${tbName2} order by k1, k2""" + def res86 = sql"""select k1, k2 from baseall order by k1, k2""" check2_doris(res85, res86) - def res87 = sql"""select b.n1, b.n2 from ${tbName2} a right anti join ${null_name} b on a.k1 = b.n2 + def res87 = sql"""select b.n1, b.n2 from baseall a right anti join ${null_name} b on a.k1 = b.n2 order by 1, 2""" def res88 = sql"""select n1, n2 from ${null_name} order by n1, n2""" check2_doris(res87, res88) - def res89 = sql"""select b.k1, b.k2 from ${null_name} a right anti join ${tbName2} b on b.k1 = a.n2 + def res89 = sql"""select b.k1, b.k2 from ${null_name} a right anti join baseall b on b.k1 = a.n2 order by 1, 2""" - def res90 = sql"""select k1, k2 from ${tbName2} order by k1, k2""" + def res90 = sql"""select k1, k2 from baseall order by k1, k2""" check2_doris(res89, res90) // join on predicate - qt_join_on_predicate1"""select c.k1 from ${tbName2} a join ${tbName1} b on a.k2 between 0 and 1000 - join ${tbName3} c on a.k10 = c.k10 order by k1 limit 65535""" + qt_join_on_predicate1"""select c.k1 from baseall a join test b on a.k2 between 0 and 1000 + join bigtable c on a.k10 = c.k10 order by k1 limit 65535""" qt_join_on_predicate2"""select a.k1 from baseall a join test b on b.k2 between 0 and 1000 and a.k1 = b.k1 order by k1;""" qt_join_on_predicate3"""select a.k1 from baseall a join test b on b.k2 between 0 and 1000 order by k1;""" qt_join_on_predicate4"""select a.k1 from baseall a join test b on b.k2 in (49, 60, 85) order by k1;"""