diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 9672e651dd..fc226c46bc 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -22,12 +22,6 @@ parser grammar DorisParser; options { tokenVocab = DorisLexer; } @members { - /** - * When false, INTERSECT is given the greater precedence over the other set - * operations (UNION, EXCEPT and MINUS) as per the SQL standard. - */ - public boolean legacy_setops_precedence_enabled = false; - /** * When false, a literal with an exponent would be converted into * double type rather than decimal type. @@ -37,7 +31,7 @@ options { tokenVocab = DorisLexer; } /** * When true, the behavior of keywords follows ANSI SQL standard. */ - public boolean SQL_standard_keyword_behavior = false; + public boolean SQL_standard_keyword_behavior = true; } multiStatements @@ -77,6 +71,13 @@ query queryTerm : queryPrimary #queryTermDefault + | left=queryTerm operator=(UNION | EXCEPT | INTERSECT) + setQuantifier? right=queryTerm #setOperation + ; + +setQuantifier + : DISTINCT + | ALL ; queryPrimary @@ -462,6 +463,7 @@ ansiNonReserved | FUNCTIONS | GLOBAL | GROUPING + | GRAPH | HOUR | IF | IGNORE @@ -520,6 +522,7 @@ ansiNonReserved | PIVOT | PLACING | PLAN + | POLICY | POSITION | PRECEDING | PRINCIPALS @@ -599,6 +602,7 @@ ansiNonReserved | UPDATE | USE | VALUES + | VERBOSE | VERSION | VIEW | VIEWS diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java index e8149474b7..0f85172a7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -43,17 +44,21 @@ import java.util.Optional; * e.g. select 100, 'value' */ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRowRelation { + private final RelationId id; private final List projects; - public UnboundOneRowRelation(List projects) { - this(projects, Optional.empty(), Optional.empty()); + public UnboundOneRowRelation(RelationId id, List projects) { + this(id, projects, Optional.empty(), Optional.empty()); } - private UnboundOneRowRelation(List projects, Optional groupExpression, + private UnboundOneRowRelation(RelationId id, + List projects, + Optional groupExpression, Optional logicalProperties) { super(PlanType.LOGICAL_UNBOUND_ONE_ROW_RELATION, groupExpression, logicalProperties); Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(Slot.class)), "OneRowRelation can not contains any slot"); + this.id = id; this.projects = ImmutableList.copyOf(projects); } @@ -74,12 +79,12 @@ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRo @Override public Plan withGroupExpression(Optional groupExpression) { - return new UnboundOneRowRelation(projects, groupExpression, Optional.of(logicalPropertiesSupplier.get())); + return new UnboundOneRowRelation(id, projects, groupExpression, Optional.of(logicalPropertiesSupplier.get())); } @Override public Plan withLogicalProperties(Optional logicalProperties) { - return new UnboundOneRowRelation(projects, Optional.empty(), logicalProperties); + return new UnboundOneRowRelation(id, projects, Optional.empty(), logicalProperties); } @Override @@ -95,6 +100,7 @@ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRo @Override public String toString() { return Utils.toSqlString("UnboundOneRowRelation", + "relationId", id, "projects", projects ); } @@ -111,11 +117,11 @@ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRo return false; } UnboundOneRowRelation that = (UnboundOneRowRelation) o; - return Objects.equals(projects, that.projects); + return Objects.equals(id, that.id) && Objects.equals(projects, that.projects); } @Override public int hashCode() { - return Objects.hash(projects); + return Objects.hash(id, projects); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 2fcbcff964..225fb883c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -45,6 +45,7 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.AggregateExpression; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -63,9 +64,11 @@ import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; @@ -74,20 +77,26 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.planner.AggregationNode; import org.apache.doris.planner.AssertNumRowsNode; import org.apache.doris.planner.DataPartition; import org.apache.doris.planner.EmptySetNode; +import org.apache.doris.planner.ExceptNode; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.HashJoinNode; import org.apache.doris.planner.HashJoinNode.DistributionMode; +import org.apache.doris.planner.IntersectNode; import org.apache.doris.planner.JoinNodeBase; import org.apache.doris.planner.NestedLoopJoinNode; import org.apache.doris.planner.OlapScanNode; @@ -96,6 +105,7 @@ import org.apache.doris.planner.PlanNode; import org.apache.doris.planner.RepeatNode; import org.apache.doris.planner.ScanNode; import org.apache.doris.planner.SelectNode; +import org.apache.doris.planner.SetOperationNode; import org.apache.doris.planner.SortNode; import org.apache.doris.planner.UnionNode; import org.apache.doris.tablefunction.TableValuedFunctionIf; @@ -104,6 +114,7 @@ import org.apache.doris.thrift.TPushAggOp; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -115,6 +126,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -364,6 +376,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slots = oneRowRelation.getLogicalProperties().getOutput(); TupleDescriptor oneRowTuple = generateTupleDesc(slots, null, context); @@ -382,7 +398,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor()); PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), unionNode, DataPartition.UNPARTITIONED); context.addPlanFragment(planFragment); @@ -1061,6 +1077,12 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor inputFragment = null + if (inputFragment == null) { + return inputFragment; + } + PlanNode planNode = inputFragment.getPlanRoot(); if (planNode instanceof ExchangeNode || planNode instanceof SortNode || planNode instanceof UnionNode) { // the three nodes don't support conjuncts, need create a SelectNode to filter data @@ -1086,6 +1108,12 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor physicalLimit, PlanTranslatorContext context) { PlanFragment inputFragment = physicalLimit.child(0).accept(this, context); + + // Union contains oneRowRelation + if (inputFragment == null) { + return inputFragment; + } + PlanNode child = inputFragment.getPlanRoot(); // physical plan: limit --> sort @@ -1154,6 +1182,163 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor childrenFragments = new ArrayList<>(); + Map childNodeToFragment = new HashMap<>(); + for (Plan plan : setOperation.children()) { + PlanFragment planFragment = plan.accept(this, context); + if (planFragment != null) { + childrenFragments.add(planFragment); + } + childNodeToFragment.put(plan, planFragment); + } + + PlanFragment setOperationFragment; + SetOperationNode setOperationNode; + + List allSlots = new Builder() + .addAll(setOperation.getOutput()) + .build(); + TupleDescriptor setTuple = generateTupleDesc(allSlots, null, context); + List outputSLotDescs = new ArrayList<>(setTuple.getSlots()); + + // create setOperationNode + if (setOperation instanceof PhysicalUnion) { + setOperationNode = new UnionNode( + context.nextPlanNodeId(), setTuple.getId()); + } else if (setOperation instanceof PhysicalExcept) { + setOperationNode = new ExceptNode( + context.nextPlanNodeId(), setTuple.getId()); + } else if (setOperation instanceof PhysicalIntersect) { + setOperationNode = new IntersectNode( + context.nextPlanNodeId(), setTuple.getId()); + } else { + throw new RuntimeException("not support"); + } + + SetOperationResult setOperationResult = collectSetOperationResult(setOperation, childNodeToFragment); + for (List expressions : setOperationResult.getResultExpressions()) { + List resultExprs = expressions + .stream() + .map(expr -> ExpressionTranslator.translate(expr, context)) + .collect(ImmutableList.toImmutableList()); + setOperationNode.addResultExprLists(resultExprs); + } + + for (List expressions : setOperationResult.getConstExpressions()) { + List constExprs = expressions + .stream() + .map(expr -> ExpressionTranslator.translate(expr, context)) + .collect(ImmutableList.toImmutableList()); + setOperationNode.addConstExprList(constExprs); + } + + for (PlanFragment childFragment : childrenFragments) { + if (childFragment != null) { + setOperationNode.addChild(childFragment.getPlanRoot()); + } + } + setOperationNode.finalizeForNereids(outputSLotDescs, outputSLotDescs); + + // create setOperationFragment + // If all child fragments are unpartitioned, return a single unpartitioned fragment + // with a UnionNode that merges all child fragments. + if (allChildFragmentsUnPartitioned(childrenFragments)) { + setOperationFragment = new PlanFragment( + context.nextFragmentId(), setOperationNode, DataPartition.UNPARTITIONED); + // Absorb the plan trees of all childFragments into unionNode + // and fix up the fragment tree in the process. + for (int i = 0; i < childrenFragments.size(); ++i) { + setOperationFragment.setFragmentInPlanTree(setOperationNode.getChild(i)); + setOperationFragment.addChildren(childrenFragments.get(i).getChildren()); + } + } else { + setOperationFragment = new PlanFragment(context.nextFragmentId(), setOperationNode, + new DataPartition(TPartitionType.HASH_PARTITIONED, + setOperationNode.getMaterializedResultExprLists().get(0))); + for (int i = 0; i < childrenFragments.size(); ++i) { + PlanFragment childFragment = childrenFragments.get(i); + // Connect the unpartitioned child fragments to SetOperationNode via a random exchange. + connectChildFragmentNotCheckExchangeNode( + setOperationNode, i, setOperationFragment, childFragment, context); + childFragment.setOutputPartition( + DataPartition.hashPartitioned(setOperationNode.getMaterializedResultExprLists().get(i))); + } + } + context.addPlanFragment(setOperationFragment); + return setOperationFragment; + } + + private List castCommonDataTypeOutputs(List outputs, List childOutputs) { + List newChildOutputs = new ArrayList<>(); + for (int i = 0; i < outputs.size(); ++i) { + Slot right = childOutputs.get(i); + DataType tightestCommonType = outputs.get(i).getDataType(); + Expression newRight = TypeCoercionUtils.castIfNotSameType(right, tightestCommonType); + newChildOutputs.add(newRight); + } + return ImmutableList.copyOf(newChildOutputs); + } + + private SetOperationResult collectSetOperationResult( + PhysicalSetOperation setOperation, Map childPlanToFragment) { + List> resultExprs = new ArrayList<>(); + List> constExprs = new ArrayList<>(); + List outputs = setOperation.getOutput(); + for (Plan child : setOperation.children()) { + List castCommonDataTypeOutputs = castCommonDataTypeOutputs(outputs, child.getOutput()); + if (child.anyMatch(PhysicalOneRowRelation.class::isInstance) && childPlanToFragment.get(child) == null) { + constExprs.add(collectConstExpressions(castCommonDataTypeOutputs, child)); + } else { + resultExprs.add(castCommonDataTypeOutputs); + } + } + return new SetOperationResult(resultExprs, constExprs); + } + + private List collectConstExpressions( + List castExpressions, Plan child) { + List newCastExpressions = new ArrayList<>(); + for (int i = 0; i < castExpressions.size(); ++i) { + Expression expression = castExpressions.get(i); + if (expression instanceof Cast) { + newCastExpressions.add(expression.withChildren( + (collectPhysicalOneRowRelation(child).getProjects().get(i).children()))); + } else { + newCastExpressions.add( + (collectPhysicalOneRowRelation(child).getProjects().get(i))); + } + } + return newCastExpressions; + } + + private PhysicalOneRowRelation collectPhysicalOneRowRelation(Plan child) { + return (PhysicalOneRowRelation) + ((ImmutableSet) child.collect(PhysicalOneRowRelation.class::isInstance)).asList().get(0); + } + + private boolean allChildFragmentsUnPartitioned(List childrenFragments) { + boolean allChildFragmentsUnPartitioned = true; + for (PlanFragment child : childrenFragments) { + allChildFragmentsUnPartitioned = allChildFragmentsUnPartitioned && !child.isPartitioned(); + } + return allChildFragmentsUnPartitioned; + } + private void extractExecSlot(Expr root, Set slotRefList) { if (root instanceof SlotRef) { slotRefList.add(((SlotRef) root).getDesc().getId().asInt()); @@ -1229,6 +1414,18 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor> resultExpressions; + private final List> constExpressions; + + public SetOperationResult(List> resultExpressions, List> constExpressions) { + this.resultExpressions = ImmutableList.copyOf(resultExpressions); + this.constExpressions = ImmutableList.copyOf(constExpressions); + } + + public List> getConstExpressions() { + return constExpressions; + } + + public List> getResultExpressions() { + return resultExpressions; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java index 6f61e7bc0f..d72837aee8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy; import org.apache.doris.nereids.rules.analysis.Scope; import org.apache.doris.nereids.rules.analysis.UserAuthentication; +import org.apache.doris.nereids.rules.rewrite.logical.HideOneRowRelationUnderUnion; import com.google.common.collect.ImmutableList; @@ -59,7 +60,8 @@ public class AnalyzeRulesJob extends BatchRulesJob { new BindFunction(), new ProjectToGlobalAggregate(), new ResolveOrdinalInOrderByAndGroupBy(), - new ReplaceExpressionByChildOutput() + new ReplaceExpressionByChildOutput(), + new HideOneRowRelationUnderUnion() )), topDownBatch(ImmutableList.of( new FillUpMissingSlots(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java index 9936a44d74..587ae5b6ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization import org.apache.doris.nereids.rules.expression.rewrite.ExpressionOptimization; import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate; import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate; +import org.apache.doris.nereids.rules.rewrite.logical.BuildAggForUnion; import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning; import org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate; import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter; @@ -39,6 +40,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin; import org.apache.doris.nereids.rules.rewrite.logical.InferPredicates; import org.apache.doris.nereids.rules.rewrite.logical.LimitPushDown; import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; +import org.apache.doris.nereids.rules.rewrite.logical.MergeSetOperations; import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate; import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition; import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet; @@ -102,6 +104,8 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob { .add(topDownBatch(ImmutableList.of(new EliminateOrderByConstant()))) .add(topDownBatch(ImmutableList.of(new EliminateUnnecessaryProject()))) .add(topDownBatch(ImmutableList.of(new EliminateAggregate()))) + .add(bottomUpBatch(ImmutableList.of(new MergeSetOperations()))) + .add(topDownBatch(ImmutableList.of(new BuildAggForUnion()))) // this rule batch must keep at the end of rewrite to do some plan check .add(bottomUpBatch(ImmutableList.of(new CheckAfterRewrite()))) .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 65e1773fe4..5510cb493c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -71,12 +71,14 @@ import org.apache.doris.nereids.DorisParser.RelationContext; import org.apache.doris.nereids.DorisParser.SelectClauseContext; import org.apache.doris.nereids.DorisParser.SelectColumnClauseContext; import org.apache.doris.nereids.DorisParser.SelectHintContext; +import org.apache.doris.nereids.DorisParser.SetOperationContext; import org.apache.doris.nereids.DorisParser.SingleStatementContext; import org.apache.doris.nereids.DorisParser.SortClauseContext; import org.apache.doris.nereids.DorisParser.SortItemContext; import org.apache.doris.nereids.DorisParser.StarContext; import org.apache.doris.nereids.DorisParser.StatementDefaultContext; import org.apache.doris.nereids.DorisParser.StringLiteralContext; +import org.apache.doris.nereids.DorisParser.SubqueryContext; import org.apache.doris.nereids.DorisParser.SubqueryExpressionContext; import org.apache.doris.nereids.DorisParser.TableAliasContext; import org.apache.doris.nereids.DorisParser.TableNameContext; @@ -169,6 +171,7 @@ import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; @@ -176,8 +179,10 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -186,6 +191,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.IntegerType; @@ -317,6 +323,39 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { }); } + @Override + public LogicalPlan visitSetOperation(SetOperationContext ctx) { + return ParserUtils.withOrigin(ctx, () -> { + LogicalPlan leftQuery = plan(ctx.left); + LogicalPlan rightQuery = plan(ctx.right); + Qualifier qualifier; + if (ctx.setQuantifier() == null || ctx.setQuantifier().DISTINCT() != null) { + qualifier = Qualifier.DISTINCT; + } else { + qualifier = Qualifier.ALL; + } + + List newChildren = new ImmutableList.Builder() + .add(leftQuery) + .add(rightQuery) + .build(); + + if (ctx.UNION() != null) { + return new LogicalUnion(qualifier, newChildren); + } else if (ctx.EXCEPT() != null) { + return new LogicalExcept(qualifier, newChildren); + } else if (ctx.INTERSECT() != null) { + return new LogicalIntersect(qualifier, newChildren); + } + throw new ParseException("not support", ctx); + }); + } + + @Override + public LogicalPlan visitSubquery(SubqueryContext ctx) { + return ParserUtils.withOrigin(ctx, () -> visitQuery(ctx.query())); + } + @Override public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationContext ctx) { return ParserUtils.withOrigin(ctx, () -> { @@ -1076,7 +1115,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { return ParserUtils.withOrigin(selectCtx, () -> { // fromClause does not exists. List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); - return new UnboundOneRowRelation(projects); + return new UnboundOneRowRelation(RelationUtil.newRelationId(), projects); }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java index 70249cdd57..c00f1711c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java @@ -73,4 +73,8 @@ public class PatternDescriptor { MatchedMultiAction matchedAction) { return new PatternMatcher<>(pattern, defaultPromise, matchedAction); } + + public Pattern getPattern() { + return pattern; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java index 190d0d075c..765c087f19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java @@ -25,15 +25,21 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.UnaryPlan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalBinary; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.logical.LogicalUnary; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.physical.PhysicalBinary; import org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalUnary; +import java.util.Arrays; + /** * An interface provided some PatternDescriptor. * Child Interface(RuleFactory) can use to declare a pattern shape, then convert to a rule. @@ -174,6 +180,98 @@ public interface Patterns { return new PatternDescriptor(new TypePattern(LogicalRelation.class), defaultPromise()); } + /** + * create a logicalSetOperation pattern. + */ + default PatternDescriptor + logicalSetOperation( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalSetOperation group. + */ + default PatternDescriptor logicalSetOperation() { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalUnion( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion group. + */ + default PatternDescriptor logicalUnion() { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalExcept pattern. + */ + default PatternDescriptor + logicalExcept( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalExcept group. + */ + default PatternDescriptor logicalExcept() { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalIntersect( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion group. + */ + default PatternDescriptor logicalIntersect() { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, multiGroup().pattern), + defaultPromise()); + } + /* abstract physical plan patterns */ /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 3668bf6db5..e40fa322e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -28,7 +28,9 @@ import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject; import org.apache.doris.nereids.rules.implementation.LogicalAssertNumRowsToPhysicalAssertNumRows; import org.apache.doris.nereids.rules.implementation.LogicalEmptyRelationToPhysicalEmptyRelation; +import org.apache.doris.nereids.rules.implementation.LogicalExceptToPhysicalExcept; import org.apache.doris.nereids.rules.implementation.LogicalFilterToPhysicalFilter; +import org.apache.doris.nereids.rules.implementation.LogicalIntersectToPhysicalIntersect; import org.apache.doris.nereids.rules.implementation.LogicalJoinToHashJoin; import org.apache.doris.nereids.rules.implementation.LogicalJoinToNestedLoopJoin; import org.apache.doris.nereids.rules.implementation.LogicalLimitToPhysicalLimit; @@ -39,6 +41,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalRepeatToPhysicalRepe import org.apache.doris.nereids.rules.implementation.LogicalSortToPhysicalQuickSort; import org.apache.doris.nereids.rules.implementation.LogicalTVFRelationToPhysicalTVFRelation; import org.apache.doris.nereids.rules.implementation.LogicalTopNToPhysicalTopN; +import org.apache.doris.nereids.rules.implementation.LogicalUnionToPhysicalUnion; import org.apache.doris.nereids.rules.rewrite.AggregateStrategies; import org.apache.doris.nereids.rules.rewrite.logical.EliminateOuterJoin; import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters; @@ -49,6 +52,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughAggre import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughJoin; import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject; import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughRepeat; +import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughSetOperation; import org.apache.doris.nereids.rules.rewrite.logical.PushdownJoinOtherCondition; import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughLimit; @@ -83,6 +87,7 @@ public class RuleSet { new PushdownExpressionsInHashCondition(), new PushdownFilterThroughAggregation(), new PushdownFilterThroughRepeat(), + new PushdownFilterThroughSetOperation(), new PushdownProjectThroughLimit(), new EliminateOuterJoin(), new MergeProjects(), @@ -104,6 +109,9 @@ public class RuleSet { .add(new LogicalEmptyRelationToPhysicalEmptyRelation()) .add(new LogicalTVFRelationToPhysicalTVFRelation()) .add(new AggregateStrategies()) + .add(new LogicalUnionToPhysicalUnion()) + .add(new LogicalExceptToPhysicalExcept()) + .add(new LogicalIntersectToPhysicalIntersect()) .build(); public static final List LEFT_DEEP_TREE_JOIN_REORDER = planRuleFactories() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 598a3ba64c..a4b4afed4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -40,6 +40,7 @@ public enum RuleType { BINDING_REPEAT_SLOT(RuleTypeClass.REWRITE), BINDING_HAVING_SLOT(RuleTypeClass.REWRITE), BINDING_SORT_SLOT(RuleTypeClass.REWRITE), + BINDING_SORT_SET_OPERATION_SLOT(RuleTypeClass.REWRITE), BINDING_LIMIT_SLOT(RuleTypeClass.REWRITE), BINDING_ONE_ROW_RELATION_FUNCTION(RuleTypeClass.REWRITE), BINDING_PROJECT_FUNCTION(RuleTypeClass.REWRITE), @@ -51,6 +52,7 @@ public enum RuleType { BINDING_SORT_FUNCTION(RuleTypeClass.REWRITE), BINDING_JOIN_FUNCTION(RuleTypeClass.REWRITE), BINDING_UNBOUND_TVF_RELATION_FUNCTION(RuleTypeClass.REWRITE), + BINDING_SET_OPERATION_SLOT(RuleTypeClass.REWRITE), REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT(RuleTypeClass.REWRITE), @@ -89,6 +91,7 @@ public enum RuleType { DISTINCT_AGGREGATE_DISASSEMBLE(RuleTypeClass.REWRITE), ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE), MERGE_PROJECTS(RuleTypeClass.REWRITE), + MARK_NECESSARY_PROJECT(RuleTypeClass.REWRITE), LOGICAL_SUB_QUERY_ALIAS_TO_LOGICAL_PROJECT(RuleTypeClass.REWRITE), ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE), ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE), @@ -115,6 +118,7 @@ public enum RuleType { PUSH_FILTER_INSIDE_JOIN(RuleTypeClass.REWRITE), PUSHDOWN_FILTER_THROUGH_PROJECT(RuleTypeClass.REWRITE), PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE), + PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE), // column prune rules, COLUMN_PRUNE_AGGREGATION_CHILD(RuleTypeClass.REWRITE), COLUMN_PRUNE_FILTER_CHILD(RuleTypeClass.REWRITE), @@ -151,6 +155,10 @@ public enum RuleType { OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE), PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE), EXTRACT_SINGLE_TABLE_EXPRESSION_FROM_DISJUNCTION(RuleTypeClass.REWRITE), + HIDE_ONE_ROW_RELATION_UNDER_UNION(RuleTypeClass.REWRITE), + MERGE_SET_OPERATION(RuleTypeClass.REWRITE), + BUILD_AGG_FOR_UNION(RuleTypeClass.REWRITE), + REWRITE_SENTINEL(RuleTypeClass.REWRITE), // limit push down PUSH_LIMIT_THROUGH_JOIN(RuleTypeClass.REWRITE), @@ -203,6 +211,9 @@ public enum RuleType { TWO_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI(RuleTypeClass.IMPLEMENTATION), TWO_PHASE_AGGREGATE_WITH_MULTI_DISTINCT(RuleTypeClass.IMPLEMENTATION), THREE_PHASE_AGGREGATE_WITH_DISTINCT(RuleTypeClass.IMPLEMENTATION), + LOGICAL_UNION_TO_PHYSICAL_UNION(RuleTypeClass.IMPLEMENTATION), + LOGICAL_EXCEPT_TO_PHYSICAL_EXCEPT(RuleTypeClass.IMPLEMENTATION), + LOGICAL_INTERSECT_TO_PHYSICAL_INTERSECT(RuleTypeClass.IMPLEMENTATION), IMPLEMENTATION_SENTINEL(RuleTypeClass.IMPLEMENTATION), LOGICAL_SEMI_JOIN_SEMI_JOIN_TRANPOSE_PROJECT(RuleTypeClass.EXPLORATION), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java index df196ea063..b434737224 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java @@ -49,14 +49,18 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.planner.PlannerContext; @@ -246,6 +250,22 @@ public class BindSlotReference implements AnalysisRuleFactory { return new LogicalSort<>(sortItemList, sort.child()); }) ), + RuleType.BINDING_SORT_SET_OPERATION_SLOT.build( + logicalSort(logicalSetOperation()).when(Plan::canBind).thenApply(ctx -> { + LogicalSort sort = ctx.root; + List sortItemList = sort.getOrderKeys() + .stream() + .map(orderKey -> { + Expression item = bind(orderKey.getExpr(), sort.children(), sort, ctx.cascadesContext); + if (item.containsType(UnboundSlot.class)) { + item = bind(item, sort.child().children(), sort, ctx.cascadesContext); + } + return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); + }).collect(Collectors.toList()); + + return new LogicalSort<>(sortItemList, sort.child()); + }) + ), RuleType.BINDING_HAVING_SLOT.build( logicalHaving(any()).when(Plan::canBind).thenApply(ctx -> { LogicalHaving having = ctx.root; @@ -272,6 +292,29 @@ public class BindSlotReference implements AnalysisRuleFactory { return new LogicalOneRowRelation(projects); }) ), + RuleType.BINDING_SET_OPERATION_SLOT.build( + logicalSetOperation().when(Plan::canBind).then(setOperation -> { + // check whether the left and right child output columns are the same + if (setOperation.child(0).getOutput().size() != setOperation.child(1).getOutput().size()) { + throw new AnalysisException("Operands have unequal number of columns:\n" + + "'" + setOperation.child(0).getOutput() + "' has " + + setOperation.child(0).getOutput().size() + " column(s)\n" + + "'" + setOperation.child(1).getOutput() + "' has " + + setOperation.child(1).getOutput().size() + " column(s)"); + } + + // INTERSECT and EXCEPT does not support ALL qualifie + if (setOperation.getQualifier() == Qualifier.ALL + && (setOperation instanceof LogicalExcept || setOperation instanceof LogicalIntersect)) { + throw new AnalysisException("INTERSECT and EXCEPT does not support ALL qualifie"); + } + + List> castExpressions = setOperation.collectCastExpressions(); + List newOutputs = setOperation.buildNewOutputs(castExpressions.get(0)); + + return setOperation.withNewOutputs(newOutputs); + }) + ), RuleType.BINDING_NON_LEAF_LOGICAL_PLAN.build( logicalPlan() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExceptToPhysicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExceptToPhysicalExcept.java new file mode 100644 index 0000000000..be8a03a17c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExceptToPhysicalExcept.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.implementation; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; + +/** + * Implementation rule that convert logical Except to Physical Except. + */ +public class LogicalExceptToPhysicalExcept extends OneImplementationRuleFactory { + @Override + public Rule build() { + return logicalExcept().then(except -> + new PhysicalExcept(except.getQualifier(), + except.getLogicalProperties(), + except.children()) + ).toRule(RuleType.LOGICAL_EXCEPT_TO_PHYSICAL_EXCEPT); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIntersectToPhysicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIntersectToPhysicalIntersect.java new file mode 100644 index 0000000000..e05ae9b310 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIntersectToPhysicalIntersect.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.implementation; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; + +/** + * Implementation rule that convert logical Intersect to Physical Intersect. + */ +public class LogicalIntersectToPhysicalIntersect extends OneImplementationRuleFactory { + @Override + public Rule build() { + return logicalIntersect().then(intersect -> + new PhysicalIntersect(intersect.getQualifier(), + intersect.getLogicalProperties(), + intersect.children()) + ).toRule(RuleType.LOGICAL_INTERSECT_TO_PHYSICAL_INTERSECT); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java index d81a8797dd..95fa819bae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java @@ -28,7 +28,8 @@ public class LogicalOneRowRelationToPhysicalOneRowRelation extends OneImplementa @Override public Rule build() { return logicalOneRowRelation() - .then(relation -> new PhysicalOneRowRelation(relation.getProjects(), relation.getLogicalProperties())) + .then(relation -> new PhysicalOneRowRelation( + relation.getProjects(), relation.buildUnionNode(), relation.getLogicalProperties())) .toRule(RuleType.LOGICAL_ONE_ROW_RELATION_TO_PHYSICAL_ONE_ROW_RELATION); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalUnionToPhysicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalUnionToPhysicalUnion.java new file mode 100644 index 0000000000..3ae88e33d5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalUnionToPhysicalUnion.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.implementation; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; + +/** + * Implementation rule that convert logical Union to Physical Union. + */ +public class LogicalUnionToPhysicalUnion extends OneImplementationRuleFactory { + @Override + public Rule build() { + return logicalUnion().then(union -> + new PhysicalUnion(union.getQualifier(), + union.getLogicalProperties(), + union.children()) + ).toRule(RuleType.LOGICAL_UNION_TO_PHYSICAL_UNION); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/BuildAggForUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/BuildAggForUnion.java new file mode 100644 index 0000000000..628944e4b8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/BuildAggForUnion.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; + +import java.util.Optional; + +/** + * For distinct union, add agg node. + */ +public class BuildAggForUnion extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalUnion().whenNot(LogicalUnion::hasBuildAgg).then(union -> { + if (union.getQualifier() == Qualifier.DISTINCT) { + return new LogicalAggregate(union.getOutputs(), union.getOutputs(), + true, Optional.empty(), union.withHasBuildAgg()); + } + return union; + }).toRule(RuleType.BUILD_AGG_FOR_UNION); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java index 675560fd7e..61e1f3c99a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java @@ -19,33 +19,54 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import com.google.common.collect.ImmutableList; + +import java.util.List; + /** * remove the project that output same with its child to avoid we get two consecutive projects in best plan. * for more information, please see this PR */ -public class EliminateUnnecessaryProject extends OneRewriteRuleFactory { - +public class EliminateUnnecessaryProject implements RewriteRuleFactory { @Override - public Rule build() { - return logicalProject(any()) - .when(project -> project.getOutputSet().equals(project.child().getOutputSet())) - .thenApply(ctx -> { - int rootGroupId = ctx.cascadesContext.getMemo().getRoot().getGroupId().asInt(); - LogicalProject project = ctx.root; - // if project is root, we need to ensure the output order is same. - if (project.getGroupExpression().get().getOwnerGroup().getGroupId().asInt() == rootGroupId) { - if (project.getOutput().equals(project.child().getOutput())) { - return project.child(); + public List buildRules() { + return ImmutableList.of( + RuleType.MARK_NECESSARY_PROJECT.build( + logicalSetOperation(logicalProject(), group()) + .thenApply(ctx -> { + LogicalProject project = (LogicalProject) ctx.root.child(0); + return ctx.root.withChildren(project.withEliminate(false), ctx.root.child(1)); + }) + ), + RuleType.MARK_NECESSARY_PROJECT.build( + logicalSetOperation(group(), logicalProject()) + .thenApply(ctx -> { + LogicalProject project = (LogicalProject) ctx.root.child(1); + return ctx.root.withChildren(ctx.root.child(0), project.withEliminate(false)); + }) + ), + RuleType.ELIMINATE_UNNECESSARY_PROJECT.build( + logicalProject(any()) + .when(LogicalProject::canEliminate) + .when(project -> project.getOutputSet().equals(project.child().getOutputSet())) + .thenApply(ctx -> { + int rootGroupId = ctx.cascadesContext.getMemo().getRoot().getGroupId().asInt(); + LogicalProject project = ctx.root; + // if project is root, we need to ensure the output order is same. + if (project.getGroupExpression().get().getOwnerGroup().getGroupId().asInt() + == rootGroupId) { + if (project.getOutput().equals(project.child().getOutput())) { + return project.child(); + } else { + return null; + } } else { - return null; + return project.child(); } - } else { - return project.child(); - } - }).toRule(RuleType.ELIMINATE_UNNECESSARY_PROJECT); + }))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java new file mode 100644 index 0000000000..79e7305c1d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.AnalysisRuleFactory; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * 1. Include oneRowRelation in the union, hide the oneRowRelation, and reduce the generation of union nodes. + * eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2; + * before: + * logicalUnion() + * / \ + * logicalUnion() logicalProject + * / \ + * logicalProject logicalOneRowRelation(BuildUnionNode:true) + * eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2; + * + * after: + * logicalUnion() + * / \ + * logicalUnion() logicalProject + * / \ + * logicalProject logicalOneRowRelation(BuildUnionNode:false) + */ +public class HideOneRowRelationUnderUnion implements AnalysisRuleFactory { + @Override + public List buildRules() { + return ImmutableList.of( + RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build( + logicalUnion(logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode), group()) + .then(union -> { + List newChildren = new ImmutableList.Builder() + .add(((LogicalOneRowRelation) union.child(0)).withBuildUnionNode(false)) + .add(union.child(1)) + .build(); + return union.withChildren(newChildren); + }) + ), + RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build( + logicalUnion(group(), logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode)) + .then(union -> { + List children = new ImmutableList.Builder() + .add(union.child(0)) + .add(((LogicalOneRowRelation) union.child(1)).withBuildUnionNode(false)) + .build(); + return union.withChildren(children); + }) + ) + ); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java new file mode 100644 index 0000000000..d3a7cf1526 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * optimization. + * Merge nodes of the same type and same qualifier. + * + * eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2; + * before: + * logicalUnion() + * / \ + * logicalUnion() logicalProject + * / \ + * logicalProject logicalOneRowRelation + * + * after: + * 2. MERGE_SET_OPERATION + * logicalUnion() + * / \ \ + * logicalProject logicalOneRowRelation logicalProject + */ +public class MergeSetOperations implements RewriteRuleFactory { + @Override + public List buildRules() { + return ImmutableList.of( + RuleType.MERGE_SET_OPERATION.build( + logicalSetOperation(logicalSetOperation(), group()).thenApply(ctx -> { + LogicalSetOperation parentSetOperation = ctx.root; + LogicalSetOperation childSetOperation = (LogicalSetOperation) parentSetOperation.child(0); + + if (isSameClass(parentSetOperation, childSetOperation) + && isSameQualifierOrChildQualifierIsAll(parentSetOperation, childSetOperation)) { + List newChildren = new ImmutableList.Builder() + .addAll(childSetOperation.children()) + .add(parentSetOperation.child(1)) + .build(); + return parentSetOperation.withChildren(newChildren); + } + return parentSetOperation; + }) + ), + RuleType.MERGE_SET_OPERATION.build( + logicalSetOperation(group(), logicalSetOperation()).thenApply(ctx -> { + LogicalSetOperation parentSetOperation = ctx.root; + LogicalSetOperation childSetOperation = (LogicalSetOperation) parentSetOperation.child(1); + + if (isSameClass(parentSetOperation, childSetOperation) + && isSameQualifierOrChildQualifierIsAll(parentSetOperation, childSetOperation)) { + List newChildren = new ImmutableList.Builder() + .add(parentSetOperation.child(0)) + .addAll(childSetOperation.children()) + .build(); + return parentSetOperation.withNewChildren(newChildren); + } + return parentSetOperation; + }) + ) + ); + } + + private boolean isSameQualifierOrChildQualifierIsAll(LogicalSetOperation parentSetOperation, + LogicalSetOperation childSetOperation) { + return parentSetOperation.getQualifier() == childSetOperation.getQualifier() + || childSetOperation.getQualifier() == Qualifier.ALL; + } + + private boolean isSameClass(LogicalSetOperation parentSetOperation, + LogicalSetOperation childSetOperation) { + return parentSetOperation.getClass().isAssignableFrom(childSetOperation.getClass()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java new file mode 100644 index 0000000000..f32a8eb53b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Convert the expression in the filter into the output column corresponding to the child node and push it down. + */ +public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalFilter(logicalSetOperation()).then(filter -> { + LogicalSetOperation setOperation = filter.child(); + + if (setOperation instanceof LogicalUnion && ((LogicalUnion) setOperation).hasPushedFilter()) { + return filter; + } + + List newChildren = new ArrayList<>(); + for (Plan child : setOperation.children()) { + Map replaceMap = new HashMap<>(); + for (int i = 0; i < setOperation.getOutputs().size(); ++i) { + NamedExpression output = setOperation.getOutputs().get(i); + replaceMap.put(output, child.getOutput().get(i)); + } + + List newFilterPredicates = Lists.newArrayList(); + ExpressionUtils.extractConjunction(filter.getPredicates()).forEach(conjunct -> { + newFilterPredicates.add(ExpressionUtils.replace(conjunct, replaceMap)); + }); + newChildren.add(new LogicalFilter<>(ExpressionUtils.and(newFilterPredicates), child)); + } + if (setOperation instanceof LogicalUnion && setOperation.getQualifier() == Qualifier.DISTINCT) { + return new LogicalFilter<>(filter.getPredicates(), + ((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren)); + } + return setOperation.withNewChildren(newChildren); + }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_SET_OPERATION); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 31975280e1..ec73d77592 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -35,11 +35,14 @@ import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; import org.apache.doris.nereids.trees.plans.algebra.Project; import org.apache.doris.nereids.trees.plans.algebra.Repeat; import org.apache.doris.nereids.trees.plans.algebra.Scan; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.algebra.TopN; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -49,12 +52,15 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalLocalQuickSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; @@ -66,6 +72,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.ColumnStatisticBuilder; @@ -186,6 +193,24 @@ public class StatsCalculator extends DefaultPlanVisitor return computeAssertNumRows(assertNumRows.getAssertNumRowsElement().getDesiredNumOfRows()); } + @Override + public StatsDeriveResult visitLogicalUnion( + LogicalUnion union, Void context) { + return computeUnion(union); + } + + @Override + public StatsDeriveResult visitLogicalExcept( + LogicalExcept except, Void context) { + return computeExcept(); + } + + @Override + public StatsDeriveResult visitLogicalIntersect( + LogicalIntersect intersect, Void context) { + return computeIntersect(intersect); + } + @Override public StatsDeriveResult visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelation, Void context) { return computeEmptyRelation(emptyRelation); @@ -274,6 +299,21 @@ public class StatsCalculator extends DefaultPlanVisitor return computeAssertNumRows(assertNumRows.getAssertNumRowsElement().getDesiredNumOfRows()); } + @Override + public StatsDeriveResult visitPhysicalUnion(PhysicalUnion union, Void context) { + return computeUnion(union); + } + + @Override + public StatsDeriveResult visitPhysicalExcept(PhysicalExcept except, Void context) { + return computeExcept(); + } + + @Override + public StatsDeriveResult visitPhysicalIntersect(PhysicalIntersect intersect, Void context) { + return computeIntersect(intersect); + } + private StatsDeriveResult computeAssertNumRows(long desiredNumOfRows) { StatsDeriveResult statsDeriveResult = groupExpression.childStatistics(0); statsDeriveResult.updateByLimit(1); @@ -434,4 +474,68 @@ public class StatsCalculator extends DefaultPlanVisitor int rowCount = 0; return new StatsDeriveResult(rowCount, columnStatsMap); } + + private StatsDeriveResult computeUnion(SetOperation setOperation) { + + StatsDeriveResult leftStatsResult = groupExpression.childStatistics(0); + Map leftStatsSlotIdToColumnStats = leftStatsResult.getSlotIdToColumnStats(); + Map newColumnStatsMap = new HashMap<>(); + double rowCount = leftStatsResult.getRowCount(); + + for (int j = 0; j < setOperation.getArity() - 1; ++j) { + StatsDeriveResult rightStatsResult = groupExpression.childStatistics(j + 1); + Map rightStatsSlotIdToColumnStats = rightStatsResult.getSlotIdToColumnStats(); + + for (int i = 0; i < setOperation.getOutputs().size(); ++i) { + Slot leftSlot = getLeftSlot(j, i, setOperation); + Slot rightSlot = setOperation.getChildOutput(j + 1).get(i); + + ColumnStatistic leftStats = getLeftStats(j, leftSlot, leftStatsSlotIdToColumnStats, newColumnStatsMap); + ColumnStatistic rightStats = rightStatsSlotIdToColumnStats.get(rightSlot.getExprId()); + newColumnStatsMap.put(setOperation.getOutputs().get(i).getExprId(), new ColumnStatistic( + leftStats.count + rightStats.count, + leftStats.ndv + rightStats.ndv, + leftStats.avgSizeByte, + leftStats.numNulls + rightStats.numNulls, + leftStats.dataSize + rightStats.dataSize, + Math.min(leftStats.minValue, rightStats.minValue), + Math.max(leftStats.maxValue, rightStats.maxValue), + 1.0 / (leftStats.ndv + rightStats.ndv), + leftStats.minExpr, + leftStats.maxExpr, + leftStats.isUnKnown)); + } + rowCount = Math.min(rowCount, rightStatsResult.getRowCount()); + } + return new StatsDeriveResult(rowCount, newColumnStatsMap); + } + + private Slot getLeftSlot(int fistSetOperation, int outputSlotIdx, SetOperation setOperation) { + return fistSetOperation == 0 + ? setOperation.getFirstOutput().get(outputSlotIdx) + : setOperation.getOutputs().get(outputSlotIdx).toSlot(); + } + + private ColumnStatistic getLeftStats(int fistSetOperation, + Slot leftSlot, + Map leftStatsSlotIdToColumnStats, + Map newColumnStatsMap) { + return fistSetOperation == 0 + ? leftStatsSlotIdToColumnStats.get(leftSlot.getExprId()) + : newColumnStatsMap.get(leftSlot.getExprId()); + } + + private StatsDeriveResult computeExcept() { + return groupExpression.childStatistics(0); + } + + private StatsDeriveResult computeIntersect(SetOperation setOperation) { + StatsDeriveResult leftStatsResult = groupExpression.childStatistics(0); + double rowCount = leftStatsResult.getRowCount(); + for (int i = 1; i < setOperation.getArity(); ++i) { + rowCount = Math.min(rowCount, groupExpression.childStatistics(i).getRowCount()); + } + return new StatsDeriveResult( + rowCount, leftStatsResult.getSlotIdToColumnStats()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index db28a2a15a..368a561d3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -48,6 +48,9 @@ public enum PlanType { LOGICAL_HAVING, LOGICAL_MULTI_JOIN, LOGICAL_CHECK_POLICY, + LOGICAL_UNION, + LOGICAL_EXCEPT, + LOGICAL_INTERSECT, GROUP_PLAN, // physical plan @@ -68,5 +71,8 @@ public enum PlanType { PHYSICAL_NESTED_LOOP_JOIN, PHYSICAL_EXCHANGE, PHYSICAL_DISTRIBUTION, - PHYSICAL_ASSERT_NUM_ROWS; + PHYSICAL_ASSERT_NUM_ROWS, + PHYSICAL_UNION, + PHYSICAL_EXCEPT, + PHYSICAL_INTERSECT } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/SetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/SetOperation.java new file mode 100644 index 0000000000..f9cb16df03 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/SetOperation.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.algebra; + +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; + +import java.util.List; + +/** + * Common interface for logical/physical SetOperation. + */ +public interface SetOperation { + /** + * SetOperation qualifier type. + */ + enum Qualifier { + ALL, + DISTINCT + } + + Qualifier getQualifier(); + + List getFirstOutput(); + + List getChildOutput(int i); + + List getOutputs(); + + int getArity(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java new file mode 100644 index 0000000000..de8b4f6f51 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import java.util.List; +import java.util.Optional; + +/** + * Logical Except. + */ +public class LogicalExcept extends LogicalSetOperation { + + public LogicalExcept(Qualifier qualifier, List inputs) { + super(PlanType.LOGICAL_EXCEPT, qualifier, inputs); + } + + public LogicalExcept(Qualifier qualifier, List outputs, List inputs) { + super(PlanType.LOGICAL_EXCEPT, qualifier, outputs, inputs); + } + + public LogicalExcept(Qualifier qualifier, List outputs, + Optional groupExpression, Optional logicalProperties, + List inputs) { + super(PlanType.LOGICAL_EXCEPT, qualifier, outputs, groupExpression, logicalProperties, inputs); + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalExcept", + "qualifier", qualifier, + "outputs", outputs); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalExcept(this, context); + } + + @Override + public LogicalExcept withChildren(List children) { + return new LogicalExcept(qualifier, outputs, children); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new LogicalExcept(qualifier, outputs, groupExpression, + Optional.of(getLogicalProperties()), children); + } + + @Override + public Plan withLogicalProperties(Optional logicalProperties) { + return new LogicalExcept(qualifier, outputs, + Optional.empty(), logicalProperties, children); + } + + @Override + public Plan withNewOutputs(List newOutputs) { + return new LogicalExcept(qualifier, newOutputs, Optional.empty(), Optional.empty(), children); + } + + @Override + public Plan withNewChildren(List children) { + return withChildren(children); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java new file mode 100644 index 0000000000..e1dd407546 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import java.util.List; +import java.util.Optional; + +/** + * Logical Intersect. + */ +public class LogicalIntersect extends LogicalSetOperation { + + public LogicalIntersect(Qualifier qualifier, List inputs) { + super(PlanType.LOGICAL_INTERSECT, qualifier, inputs); + } + + public LogicalIntersect(Qualifier qualifier, List outputs, + List inputs) { + super(PlanType.LOGICAL_INTERSECT, qualifier, outputs, inputs); + } + + public LogicalIntersect(Qualifier qualifier, List outputs, + Optional groupExpression, Optional logicalProperties, + List inputs) { + super(PlanType.LOGICAL_INTERSECT, qualifier, outputs, groupExpression, logicalProperties, inputs); + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalIntersect", + "qualifier", qualifier, + "outputs", outputs); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalIntersect(this, context); + } + + @Override + public LogicalIntersect withChildren(List children) { + return new LogicalIntersect(qualifier, outputs, children); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new LogicalIntersect(qualifier, outputs, groupExpression, + Optional.of(getLogicalProperties()), children); + } + + @Override + public Plan withLogicalProperties(Optional logicalProperties) { + return new LogicalIntersect(qualifier, outputs, + Optional.empty(), logicalProperties, children); + } + + @Override + public Plan withNewOutputs(List newOutputs) { + return new LogicalIntersect(qualifier, newOutputs, + Optional.empty(), Optional.empty(), children); + } + + @Override + public Plan withNewChildren(List children) { + return withChildren(children); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 5b26472e9a..8f01428d8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -41,17 +41,21 @@ import java.util.Optional; */ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation { private final ImmutableList projects; + private final boolean buildUnionNode; public LogicalOneRowRelation(List projects) { - this(projects, Optional.empty(), Optional.empty()); + this(projects, true, Optional.empty(), Optional.empty()); } - private LogicalOneRowRelation(List projects, Optional groupExpression, - Optional logicalProperties) { + private LogicalOneRowRelation(List projects, + boolean buildUnionNode, + Optional groupExpression, + Optional logicalProperties) { super(PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties); Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(Slot.class)), "OneRowRelation can not contains any slot"); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); + this.buildUnionNode = buildUnionNode; } @Override @@ -71,12 +75,13 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalOneRowRelation(projects, groupExpression, Optional.of(logicalPropertiesSupplier.get())); + return new LogicalOneRowRelation(projects, buildUnionNode, + groupExpression, Optional.of(logicalPropertiesSupplier.get())); } @Override public Plan withLogicalProperties(Optional logicalProperties) { - return new LogicalOneRowRelation(projects, Optional.empty(), logicalProperties); + return new LogicalOneRowRelation(projects, buildUnionNode, Optional.empty(), logicalProperties); } @Override @@ -89,7 +94,8 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation @Override public String toString() { return Utils.toSqlString("LogicalOneRowRelation", - "projects", projects + "projects", projects, + "buildUnionNode", buildUnionNode ); } @@ -105,11 +111,20 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation return false; } LogicalOneRowRelation that = (LogicalOneRowRelation) o; - return Objects.equals(projects, that.projects); + return Objects.equals(projects, that.projects) + && Objects.equals(buildUnionNode, that.buildUnionNode); } @Override public int hashCode() { - return Objects.hash(projects); + return Objects.hash(projects, buildUnionNode); + } + + public boolean buildUnionNode() { + return buildUnionNode; + } + + public Plan withBuildUnionNode(boolean buildUnionNode) { + return new LogicalOneRowRelation(projects, buildUnionNode, Optional.empty(), Optional.empty()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 7280312f39..0f03bc0883 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -44,12 +44,20 @@ public class LogicalProject extends LogicalUnary projects; private final ImmutableList excepts; - public LogicalProject(List projects, List excepts, CHILD_TYPE child) { - this(projects, excepts, Optional.empty(), Optional.empty(), child); - } + // For project nodes under union, erasure cannot be configured, so add this flag. + private final boolean canEliminate; public LogicalProject(List projects, CHILD_TYPE child) { - this(projects, Collections.emptyList(), child); + this(projects, Collections.emptyList(), true, child); + } + + public LogicalProject(List projects, List excepts, CHILD_TYPE child) { + this(projects, excepts, true, child); + } + + public LogicalProject(List projects, List excepts, + boolean canEliminate, CHILD_TYPE child) { + this(projects, excepts, canEliminate, Optional.empty(), Optional.empty(), child); } /** @@ -57,12 +65,13 @@ public class LogicalProject extends LogicalUnary projects, List excepts, + public LogicalProject(List projects, List excepts, boolean canEliminate, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties, child); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); this.excepts = ImmutableList.copyOf(excepts); + this.canEliminate = canEliminate; } /** @@ -90,7 +99,8 @@ public class LogicalProject extends LogicalUnary extends LogicalUnary withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalProject<>(projects, excepts, children.get(0)); + return new LogicalProject<>(projects, excepts, canEliminate, children.get(0)); } @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalProject<>(projects, excepts, groupExpression, Optional.of(getLogicalProperties()), child()); + return new LogicalProject<>(projects, excepts, canEliminate, + groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withLogicalProperties(Optional logicalProperties) { - return new LogicalProject<>(projects, excepts, Optional.empty(), logicalProperties, child()); + return new LogicalProject<>(projects, excepts, canEliminate, Optional.empty(), logicalProperties, child()); + } + + public boolean canEliminate() { + return canEliminate; + } + + public Plan withEliminate(boolean isEliminate) { + return new LogicalProject<>(projects, excepts, isEliminate, child()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java new file mode 100644 index 0000000000..10c5eb275c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Logical SetOperation. + * The type can have any number of children. + * After parse, there will only be two children. + * But after rewriting rules such as merging of the same nodes and elimination of oneRowRelation, + * there will be multiple or no children. + * + * eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2; + */ +public abstract class LogicalSetOperation extends AbstractLogicalPlan implements SetOperation { + + // eg value: qualifier:DISTINCT + protected final Qualifier qualifier; + + // The newly created output column, used to display the output. + // eg value: outputs:[k1, k2] + protected final List outputs; + + public LogicalSetOperation(PlanType planType, Qualifier qualifier, List inputs) { + super(planType, inputs.toArray(new Plan[0])); + this.qualifier = qualifier; + this.outputs = ImmutableList.of(); + } + + public LogicalSetOperation(PlanType planType, Qualifier qualifier, + List outputs, + List inputs) { + super(planType, inputs.toArray(new Plan[0])); + this.qualifier = qualifier; + this.outputs = ImmutableList.copyOf(outputs); + } + + public LogicalSetOperation(PlanType planType, Qualifier qualifier, List outputs, + Optional groupExpression, Optional logicalProperties, + List inputs) { + super(planType, groupExpression, logicalProperties, inputs.toArray(new Plan[0])); + this.qualifier = qualifier; + this.outputs = ImmutableList.copyOf(outputs); + } + + @Override + public List computeOutput() { + return outputs.stream() + .map(NamedExpression::toSlot) + .collect(ImmutableList.toImmutableList()); + } + + public List> collectCastExpressions() { + return castCommonDataTypeOutputs(resetNullableForLeftOutputs()); + } + + /** + * Generate new output for SetOperation. + */ + public List buildNewOutputs(List leftCastExpressions) { + ImmutableList.Builder newOutputs = new Builder<>(); + for (Expression expression : leftCastExpressions) { + if (expression instanceof Cast) { + newOutputs.add(new SlotReference( + ((Cast) expression).child().toSql(), expression.getDataType(), + ((Cast) expression).child().nullable())); + } else if (expression instanceof Slot) { + newOutputs.add(new SlotReference( + expression.toSql(), expression.getDataType(), expression.nullable())); + } + } + return newOutputs.build(); + } + + // If the right child is nullable, need to ensure that the left child is also nullable + private List resetNullableForLeftOutputs() { + Preconditions.checkState(children.size() == 2); + List resetNullableForLeftOutputs = new ArrayList<>(); + for (int i = 0; i < child(1).getOutput().size(); ++i) { + if (child(1).getOutput().get(i).nullable() && !child(0).getOutput().get(i).nullable()) { + resetNullableForLeftOutputs.add(child(0).getOutput().get(i).withNullable(true)); + } else { + resetNullableForLeftOutputs.add(child(0).getOutput().get(i)); + } + } + return ImmutableList.copyOf(resetNullableForLeftOutputs); + } + + private List> castCommonDataTypeOutputs(List resetNullableForLeftOutputs) { + List newLeftOutputs = new ArrayList<>(); + List newRightOutpus = new ArrayList<>(); + // Ensure that the output types of the left and right children are consistent and expand upward. + for (int i = 0; i < resetNullableForLeftOutputs.size(); ++i) { + boolean hasPushed = false; + Slot left = resetNullableForLeftOutputs.get(i); + Slot right = child(1).getOutput().get(i); + if (TypeCoercionUtils.canHandleTypeCoercion(left.getDataType(), right.getDataType())) { + Optional tightestCommonType = + TypeCoercionUtils.findTightestCommonType(left.getDataType(), right.getDataType()); + if (tightestCommonType.isPresent()) { + Expression newLeft = TypeCoercionUtils.castIfNotSameType(left, tightestCommonType.get()); + Expression newRight = TypeCoercionUtils.castIfNotSameType(right, tightestCommonType.get()); + newLeftOutputs.add(newLeft); + newRightOutpus.add(newRight); + hasPushed = true; + } + } + + if (!hasPushed) { + newLeftOutputs.add(left); + newRightOutpus.add(right); + } + } + + List> resultExpressions = new ArrayList<>(); + resultExpressions.add(newLeftOutputs); + resultExpressions.add(newRightOutpus); + return ImmutableList.copyOf(resultExpressions); + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalSetOperation", + "qualifier", qualifier, + "outputs", outputs); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LogicalSetOperation that = (LogicalSetOperation) o; + return Objects.equals(qualifier, that.qualifier) + && Objects.equals(outputs, that.outputs); + } + + @Override + public int hashCode() { + return Objects.hash(qualifier, outputs); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalSetOperation(this, context); + } + + @Override + public List getExpressions() { + return ImmutableList.of(); + } + + @Override + public Qualifier getQualifier() { + return qualifier; + } + + @Override + public List getFirstOutput() { + return child(0).getOutput(); + } + + @Override + public List getChildOutput(int i) { + return child(i).getOutput(); + } + + @Override + public List getOutputs() { + return outputs; + } + + public abstract Plan withNewOutputs(List newOutputs); + + @Override + public int getArity() { + return children.size(); + } + + public abstract Plan withNewChildren(List children); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java new file mode 100644 index 0000000000..4f15ef2991 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Logical Union. + */ +public class LogicalUnion extends LogicalSetOperation { + // When the union is DISTINCT, an additional LogicalAggregation needs to be created, + // so add this flag to judge whether agg has been created to avoid repeated creation + private final boolean hasBuildAgg; + + // When there is an agg on the union and there is a filter on the agg, + // it is necessary to keep the filter on the agg and push the filter down to each child of the union. + private final boolean hasPushedFilter; + + public LogicalUnion(Qualifier qualifier, List inputs) { + super(PlanType.LOGICAL_UNION, qualifier, inputs); + this.hasBuildAgg = false; + this.hasPushedFilter = false; + } + + public LogicalUnion(Qualifier qualifier, List outputs, + boolean hasBuildAgg, boolean hasPushedFilter, + List inputs) { + super(PlanType.LOGICAL_UNION, qualifier, outputs, inputs); + this.hasBuildAgg = hasBuildAgg; + this.hasPushedFilter = hasPushedFilter; + } + + public LogicalUnion(Qualifier qualifier, List outputs, + boolean hasBuildAgg, boolean hasPushedFilter, + Optional groupExpression, Optional logicalProperties, + List inputs) { + super(PlanType.LOGICAL_UNION, qualifier, outputs, groupExpression, logicalProperties, inputs); + this.hasBuildAgg = hasBuildAgg; + this.hasPushedFilter = hasPushedFilter; + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalUnion", + "qualifier", qualifier, + "outputs", outputs, + "hasBuildAgg", hasBuildAgg, + "hasPushedFilter", hasPushedFilter); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LogicalUnion that = (LogicalUnion) o; + return super.equals(that) + && hasBuildAgg == that.hasBuildAgg + && hasPushedFilter == that.hasPushedFilter; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), hasBuildAgg, hasPushedFilter); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalUnion(this, context); + } + + @Override + public LogicalUnion withChildren(List children) { + return new LogicalUnion(qualifier, outputs, hasBuildAgg, hasPushedFilter, children); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new LogicalUnion(qualifier, outputs, hasBuildAgg, hasPushedFilter, groupExpression, + Optional.of(getLogicalProperties()), children); + } + + @Override + public Plan withLogicalProperties(Optional logicalProperties) { + return new LogicalUnion(qualifier, outputs, hasBuildAgg, hasPushedFilter, + Optional.empty(), logicalProperties, children); + } + + @Override + public Plan withNewOutputs(List newOutputs) { + return new LogicalUnion(qualifier, newOutputs, hasBuildAgg, hasPushedFilter, + Optional.empty(), Optional.empty(), children); + } + + public boolean hasBuildAgg() { + return hasBuildAgg; + } + + public Plan withHasBuildAgg() { + return new LogicalUnion(qualifier, outputs, true, hasPushedFilter, + Optional.empty(), Optional.empty(), children); + } + + public boolean hasPushedFilter() { + return hasPushedFilter; + } + + public Plan withHasPushedFilter() { + return new LogicalUnion(qualifier, outputs, hasBuildAgg, true, + Optional.empty(), Optional.empty(), children); + } + + @Override + public Plan withNewChildren(List children) { + return withChildren(children); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java new file mode 100644 index 0000000000..939d1355e3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.StatsDeriveResult; + +import java.util.List; +import java.util.Optional; + +/** + * Physical Except. + */ +public class PhysicalExcept extends PhysicalSetOperation { + + public PhysicalExcept(Qualifier qualifier, + LogicalProperties logicalProperties, + List inputs) { + super(PlanType.PHYSICAL_EXCEPT, qualifier, logicalProperties, inputs); + } + + public PhysicalExcept(Qualifier qualifier, + Optional groupExpression, + LogicalProperties logicalProperties, + List inputs) { + super(PlanType.PHYSICAL_EXCEPT, qualifier, groupExpression, logicalProperties, inputs); + } + + public PhysicalExcept(Qualifier qualifier, Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult, + List inputs) { + super(PlanType.PHYSICAL_EXCEPT, qualifier, + groupExpression, logicalProperties, physicalProperties, statsDeriveResult, inputs); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalExcept(this, context); + } + + @Override + public String toString() { + return Utils.toSqlString("PhysicalExcept", + "qualifier", qualifier, + "stats", statsDeriveResult); + } + + @Override + public PhysicalExcept withChildren(List children) { + return new PhysicalExcept(qualifier, getLogicalProperties(), children); + } + + @Override + public PhysicalExcept withGroupExpression( + Optional groupExpression) { + return new PhysicalExcept(qualifier, groupExpression, getLogicalProperties(), children); + } + + @Override + public PhysicalExcept withLogicalProperties( + Optional logicalProperties) { + return new PhysicalExcept(qualifier, Optional.empty(), logicalProperties.get(), children); + } + + @Override + public PhysicalExcept withPhysicalPropertiesAndStats( + PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) { + return new PhysicalExcept(qualifier, Optional.empty(), + getLogicalProperties(), physicalProperties, statsDeriveResult, children); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java new file mode 100644 index 0000000000..be3b9d0316 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.StatsDeriveResult; + +import java.util.List; +import java.util.Optional; + +/** + * Physical Intersect. + */ +public class PhysicalIntersect extends PhysicalSetOperation { + + public PhysicalIntersect(Qualifier qualifier, + LogicalProperties logicalProperties, + List inputs) { + super(PlanType.PHYSICAL_INTERSECT, qualifier, logicalProperties, inputs); + } + + public PhysicalIntersect(Qualifier qualifier, + Optional groupExpression, + LogicalProperties logicalProperties, + List inputs) { + super(PlanType.PHYSICAL_INTERSECT, qualifier, groupExpression, logicalProperties, inputs); + } + + public PhysicalIntersect(Qualifier qualifier, + Optional groupExpression, LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult, + List inputs) { + super(PlanType.PHYSICAL_INTERSECT, qualifier, + groupExpression, logicalProperties, physicalProperties, statsDeriveResult, inputs); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalIntersect(this, context); + } + + @Override + public String toString() { + return Utils.toSqlString("PhysicalIntersect", + "qualifier", qualifier, + "stats", statsDeriveResult); + } + + @Override + public PhysicalIntersect withChildren(List children) { + return new PhysicalIntersect(qualifier, getLogicalProperties(), children); + } + + @Override + public PhysicalIntersect withGroupExpression( + Optional groupExpression) { + return new PhysicalIntersect(qualifier, groupExpression, getLogicalProperties(), children); + } + + @Override + public PhysicalIntersect withLogicalProperties( + Optional logicalProperties) { + return new PhysicalIntersect(qualifier, Optional.empty(), logicalProperties.get(), children); + } + + @Override + public PhysicalIntersect withPhysicalPropertiesAndStats( + PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) { + return new PhysicalIntersect(qualifier, + Optional.empty(), getLogicalProperties(), physicalProperties, statsDeriveResult, children); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java index 7f783a438c..02885fee30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java @@ -42,12 +42,16 @@ import java.util.Optional; */ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelation { private final ImmutableList projects; + private final boolean buildUnionNode; - public PhysicalOneRowRelation(List projects, LogicalProperties logicalProperties) { - this(projects, Optional.empty(), logicalProperties, null, null); + public PhysicalOneRowRelation(List projects, boolean buildUnionNode, + LogicalProperties logicalProperties) { + this(projects, buildUnionNode, Optional.empty(), logicalProperties, null, null); } - private PhysicalOneRowRelation(List projects, Optional groupExpression, + private PhysicalOneRowRelation(List projects, + boolean buildUnionNode, + Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) { super(PlanType.PHYSICAL_ONE_ROW_RELATION, groupExpression, logicalProperties, physicalProperties, @@ -55,6 +59,7 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelati Preconditions.checkArgument(projects.stream().allMatch(Expression::isConstant), "OneRowRelation must consist of some constant expression"); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); + this.buildUnionNode = buildUnionNode; } @Override @@ -74,20 +79,21 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelati @Override public Plan withGroupExpression(Optional groupExpression) { - return new PhysicalOneRowRelation(projects, groupExpression, + return new PhysicalOneRowRelation(projects, buildUnionNode, groupExpression, logicalPropertiesSupplier.get(), physicalProperties, statsDeriveResult); } @Override public Plan withLogicalProperties(Optional logicalProperties) { - return new PhysicalOneRowRelation(projects, Optional.empty(), + return new PhysicalOneRowRelation(projects, buildUnionNode, Optional.empty(), logicalProperties.get(), physicalProperties, statsDeriveResult); } @Override public String toString() { return Utils.toSqlString("PhysicalOneRowRelation", - "expressions", projects + "expressions", projects, + "buildUnionNode", buildUnionNode ); } @@ -100,18 +106,23 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelati return false; } PhysicalOneRowRelation that = (PhysicalOneRowRelation) o; - return Objects.equals(projects, that.projects); + return Objects.equals(projects, that.projects) + && buildUnionNode == that.buildUnionNode; } @Override public int hashCode() { - return Objects.hash(projects); + return Objects.hash(projects, buildUnionNode); } @Override public PhysicalOneRowRelation withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) { - return new PhysicalOneRowRelation(projects, Optional.empty(), + return new PhysicalOneRowRelation(projects, buildUnionNode, Optional.empty(), logicalPropertiesSupplier.get(), physicalProperties, statsDeriveResult); } + + public boolean notBuildUnionNode() { + return !buildUnionNode; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java new file mode 100644 index 0000000000..f8c86ac781 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.StatsDeriveResult; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Physical SetOperation. + */ +public abstract class PhysicalSetOperation extends AbstractPhysicalPlan implements SetOperation { + protected final Qualifier qualifier; + + public PhysicalSetOperation(PlanType planType, + Qualifier qualifier, + LogicalProperties logicalProperties, + List inputs) { + super(planType, Optional.empty(), logicalProperties, inputs.toArray(new Plan[0])); + this.qualifier = qualifier; + } + + public PhysicalSetOperation(PlanType planType, + Qualifier qualifier, + Optional groupExpression, + LogicalProperties logicalProperties, + List inputs) { + super(planType, groupExpression, logicalProperties, inputs.toArray(new Plan[0])); + this.qualifier = qualifier; + } + + public PhysicalSetOperation(PlanType planType, + Qualifier qualifier, + Optional groupExpression, LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult, List inputs) { + super(planType, groupExpression, logicalProperties, + physicalProperties, statsDeriveResult, inputs.toArray(new Plan[0])); + this.qualifier = qualifier; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalSetOperation(this, context); + } + + @Override + public String toString() { + return Utils.toSqlString("PhysicalSetOperation", + "qualifier", qualifier, + "stats", statsDeriveResult); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PhysicalSetOperation that = (PhysicalSetOperation) o; + return Objects.equals(qualifier, that.qualifier); + } + + @Override + public int hashCode() { + return Objects.hash(qualifier); + } + + @Override + public List getExpressions() { + return ImmutableList.of(); + } + + @Override + public Qualifier getQualifier() { + return qualifier; + } + + @Override + public List getFirstOutput() { + return child(0).getOutput(); + } + + @Override + public List getChildOutput(int i) { + return child(i).getOutput(); + } + + @Override + public List getOutputs() { + return getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public int getArity() { + return children.size(); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java new file mode 100644 index 0000000000..3ef2d63a23 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.StatsDeriveResult; + +import java.util.List; +import java.util.Optional; + +/** + * Physical Union. + */ +public class PhysicalUnion extends PhysicalSetOperation { + + public PhysicalUnion(Qualifier qualifier, + LogicalProperties logicalProperties, + List inputs) { + super(PlanType.PHYSICAL_UNION, qualifier, logicalProperties, inputs); + } + + public PhysicalUnion(Qualifier qualifier, + Optional groupExpression, + LogicalProperties logicalProperties, + List inputs) { + super(PlanType.PHYSICAL_UNION, qualifier, groupExpression, logicalProperties, inputs); + } + + public PhysicalUnion(Qualifier qualifier, + Optional groupExpression, LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult, List inputs) { + super(PlanType.PHYSICAL_UNION, qualifier, + groupExpression, logicalProperties, physicalProperties, statsDeriveResult, inputs); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalUnion(this, context); + } + + @Override + public String toString() { + return Utils.toSqlString("PhysicalUnion", + "qualifier", qualifier, + "stats", statsDeriveResult); + } + + @Override + public PhysicalUnion withChildren(List children) { + return new PhysicalUnion(qualifier, getLogicalProperties(), children); + } + + @Override + public PhysicalUnion withGroupExpression( + Optional groupExpression) { + return new PhysicalUnion(qualifier, groupExpression, getLogicalProperties(), children); + } + + @Override + public PhysicalUnion withLogicalProperties( + Optional logicalProperties) { + return new PhysicalUnion(qualifier, Optional.empty(), logicalProperties.get(), children); + } + + @Override + public PhysicalUnion withPhysicalPropertiesAndStats( + PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) { + return new PhysicalUnion(qualifier, Optional.empty(), + getLogicalProperties(), physicalProperties, statsDeriveResult, children); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java index 07ca552e8d..961cc9f5ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java @@ -31,8 +31,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -41,18 +43,22 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalLocalQuickSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; @@ -62,9 +68,11 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; /** * Base class for the processing of logical and physical plan. @@ -192,6 +200,23 @@ public abstract class PlanVisitor { return visit(having, context); } + public R visitLogicalSetOperation( + LogicalSetOperation logicalSetOperation, C context) { + return visit(logicalSetOperation, context); + } + + public R visitLogicalUnion(LogicalUnion union, C context) { + return visitLogicalSetOperation(union, context); + } + + public R visitLogicalExcept(LogicalExcept except, C context) { + return visitLogicalSetOperation(except, context); + } + + public R visitLogicalIntersect(LogicalIntersect intersect, C context) { + return visitLogicalSetOperation(intersect, context); + } + // ******************************* // Physical plans // ******************************* @@ -265,6 +290,22 @@ public abstract class PlanVisitor { return visit(filter, context); } + public R visitPhysicalSetOperation(PhysicalSetOperation setOperation, C context) { + return visit(setOperation, context); + } + + public R visitPhysicalUnion(PhysicalUnion union, C context) { + return visitPhysicalSetOperation(union, context); + } + + public R visitPhysicalExcept(PhysicalExcept except, C context) { + return visitPhysicalSetOperation(except, context); + } + + public R visitPhysicalIntersect(PhysicalIntersect intersect, C context) { + return visitPhysicalSetOperation(intersect, context); + } + // ******************************* // Physical enforcer // ******************************* diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java index c9fe36c41c..ad904a5ba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java @@ -26,7 +26,7 @@ import org.apache.doris.thrift.TPlanNodeType; import java.util.List; public class ExceptNode extends SetOperationNode { - protected ExceptNode(PlanNodeId id, TupleId tupleId) { + public ExceptNode(PlanNodeId id, TupleId tupleId) { super(id, tupleId, "EXCEPT"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java index b6a4e676cb..b4f1b1a271 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java @@ -25,7 +25,7 @@ import org.apache.doris.thrift.TPlanNodeType; import java.util.List; public class IntersectNode extends SetOperationNode { - protected IntersectNode(PlanNodeId id, TupleId tupleId) { + public IntersectNode(PlanNodeId id, TupleId tupleId) { super(id, tupleId, "INTERSECT"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java index be4782188b..3e939e8513 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java @@ -118,6 +118,10 @@ public abstract class SetOperationNode extends PlanNode { constExprLists.add(exprs); } + public void addResultExprLists(List exprs) { + resultExprLists.add(exprs); + } + /** * Returns true if this UnionNode has only constant exprs. */ @@ -446,7 +450,10 @@ public abstract class SetOperationNode extends PlanNode { return numInstances; } - public void finalizeForNereids(TupleDescriptor tupleDescriptor, List constExprSlots) { + /** + * just for Nereids. + */ + public void finalizeForNereids(List constExprSlots, List resultExprSlots) { materializedConstExprLists.clear(); for (List exprList : constExprLists) { Preconditions.checkState(exprList.size() == constExprSlots.size()); @@ -458,5 +465,21 @@ public abstract class SetOperationNode extends PlanNode { } materializedConstExprLists.add(newExprList); } + + materializedResultExprLists.clear(); + Preconditions.checkState(resultExprLists.size() == children.size()); + for (int i = 0; i < resultExprLists.size(); ++i) { + List exprList = resultExprLists.get(i); + List newExprList = Lists.newArrayList(); + Preconditions.checkState(exprList.size() == resultExprSlots.size()); + for (int j = 0; j < exprList.size(); ++j) { + if (resultExprSlots.get(j).isMaterialized()) { + newExprList.add(exprList.get(j)); + } + } + materializedResultExprLists.add(newExprList); + } + Preconditions.checkState( + materializedResultExprLists.size() == getChildren().size()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java index f6a124b81d..f23e64c3fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java @@ -131,6 +131,15 @@ public class StatsDeriveResult { for (Entry entry : slotIdToColumnStats.entrySet()) { statsDeriveResult.addColumnStats(entry.getKey(), entry.getValue().updateByLimit(limit, rowCount)); } + // When the table is first created, rowCount is empty. + // This leads to NPE if there is SetOperation outside the limit. + // Therefore, when rowCount is empty, slotIdToColumnStats is also imported, + // but the possible problem is that the first query statistics are not derived accurately. + if (statsDeriveResult.slotIdToColumnStats.isEmpty()) { + for (Entry entry : slotIdToColumnStats.entrySet()) { + statsDeriveResult.addColumnStats(entry.getKey(), entry.getValue()); + } + } return statsDeriveResult; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index e43e62f85d..d11a82d3c3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -242,4 +242,17 @@ public class NereidsParserTest extends ParserTestBase { .sum(); Assertions.assertEquals(doubleCount, 1); } + + @Test + public void parseSetOperation() { + String union = "select * from t1 union select * from t2 union all select * from t3"; + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(union); + System.out.println(logicalPlan.treeString()); + + String union1 = "select * from t1 union (select * from t2 union all select * from t3)"; + NereidsParser nereidsParser1 = new NereidsParser(); + LogicalPlan logicalPlan1 = nereidsParser1.parseSingle(union1); + System.out.println(logicalPlan1.treeString()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java index 840402bb72..ec6d6bf913 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java @@ -362,7 +362,7 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements P @Test public void testCountDistinctValueColumn() { - singleTableTest("select k1, count(distinct v1) from from t group by k1", scan -> { + singleTableTest("select k1, count(distinct v1) from t group by k1", scan -> { Assertions.assertFalse(scan.isPreAggregation()); Assertions.assertEquals("Count distinct is only valid for key columns, but meet count(DISTINCT v1).", scan.getReasonOfPreAggregation()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java index 7234cf73b1..b46ea7dcef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java @@ -68,7 +68,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().build()); + List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); cascadesContext.topDownRewrite(rules); Plan actual = cascadesContext.getMemo().copyOut(); @@ -82,7 +82,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().build()); + List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); cascadesContext.topDownRewrite(rules); Plan actual = cascadesContext.getMemo().copyOut(); @@ -96,7 +96,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().build()); + List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); cascadesContext.topDownRewrite(rules); Plan actual = cascadesContext.getMemo().copyOut(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java index 5ddbce94c4..b9de02ec0a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java @@ -89,7 +89,7 @@ public class PlanToStringTest { LogicalProject plan = new LogicalProject<>(ImmutableList.of( new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList())), child); - Assertions.assertTrue(plan.toString().matches("LogicalProject \\( projects=\\[a#\\d+], excepts=\\[] \\)")); + Assertions.assertTrue(plan.toString().matches("LogicalProject \\( projects=\\[a#\\d+], excepts=\\[], canEliminate=true \\)")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/SetOperationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/SetOperationTest.java new file mode 100644 index 0000000000..fa7fcddc3f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/SetOperationTest.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Test; + +public class SetOperationTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + useDatabase("test"); + + createTable("CREATE TABLE `t1` (\n" + + " `k1` bigint(20) NULL,\n" + + " `k2` bigint(20) NULL,\n" + + " `k3` bigint(20) not NULL,\n" + + " `k4` bigint(20) not NULL,\n" + + " `k5` bigint(20) NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); + + createTable("CREATE TABLE `t2` (\n" + + " `k1` bigint(20) NULL,\n" + + " `k2` varchar(20) NULL,\n" + + " `k3` bigint(20) not NULL,\n" + + " `k4` bigint(20) not NULL,\n" + + " `k5` bigint(20) NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); + + createTable("CREATE TABLE `t3` (\n" + + " `k1` varchar(20) NULL,\n" + + " `k2` varchar(20) NULL,\n" + + " `k3` bigint(20) not NULL,\n" + + " `k4` bigint(20) not NULL,\n" + + " `k5` bigint(20) NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`)\n" + + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); + } + + // union + @Test + public void testUnion1() { + PlanChecker.from(connectContext) + .checkPlannerResult("select k1, k2 from t1 union select k1, k2 from t3;"); + } + + @Test + public void testUnion2() { + PlanChecker.from(connectContext) + .checkPlannerResult("select k1, k2 from t1 union all select k1, k2 from t3;"); + } + + @Test + public void testUnion3() { + PlanChecker.from(connectContext) + .checkPlannerResult("select k1 from t1 union select k1 from t3 union select 1;"); + } + + @Test + public void testUnion4() { + PlanChecker.from(connectContext) + .checkPlannerResult("select 1 a, 2 b union all select 3, 4 union all select 10 e, 20 f;"); + } + + @Test + public void testUnion5() { + PlanChecker.from(connectContext) + .checkPlannerResult("select 1, 2 union all select 1, 2 union all select 10 e, 20 f;"); + } +} diff --git a/regression-test/data/nereids_syntax_p0/set_operation.out b/regression-test/data/nereids_syntax_p0/set_operation.out new file mode 100644 index 0000000000..a5f9e478d0 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/set_operation.out @@ -0,0 +1,441 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +1 0 +1 1 +1 2 +1 3 +2 0 +2 1 +2 2 +2 4 +2 6 +3 0 +3 2 +3 3 +3 6 +3 9 +4 0 +4 3 + +-- !select2 -- +2 0 +2 1 +2 2 +2 3 +3 0 +3 2 +3 4 +3 6 +4 0 +4 3 +4 6 +4 9 + +-- !select3 -- +1 0 0 +1 1 1 +1 1 2 +1 1 3 +2 2 0 +2 2 2 +2 2 4 +2 2 6 +3 0 0 +3 3 3 +3 3 6 +3 3 9 +a b 1 +a c 1 +a d 1 +b b 1 +b c 2 +b d 2 +c b 2 +c c 2 +c d 3 +d b 3 +d c 3 +d d 3 + +-- !select4 -- +1 0 0 +1 1 1 +1 1 2 +1 1 3 +2 2 0 +2 2 2 +2 2 4 +2 2 6 +3 0 0 +3 3 3 +3 3 6 +3 3 9 +a b 1 +a c 1 +a d 1 +b b 1 +b c 2 +b d 2 +c b 2 +c c 2 +c d 3 +d b 3 +d c 3 +d d 3 + +-- !select5 -- +1 0 +1 1 +1 2 +1 3 +2 0 +2 0 +2 1 +2 1 +2 1 +2 2 +2 4 +2 6 +3 0 +3 2 +3 2 +3 2 +3 2 +3 3 +3 6 +3 9 +4 0 +4 3 +4 3 +4 3 + +-- !select6 -- +2 0 +2 0 +2 1 +2 1 +2 1 +2 1 +2 2 +2 3 +3 0 +3 2 +3 2 +3 2 +3 2 +3 2 +3 4 +3 6 +4 0 +4 0 +4 3 +4 3 +4 3 +4 3 +4 6 +4 9 + +-- !select7 -- +1 0 0 +1 1 1 +1 1 2 +1 1 3 +2 2 0 +2 2 2 +2 2 4 +2 2 6 +3 0 0 +3 3 3 +3 3 6 +3 3 9 +a b 1 +a c 1 +a d 1 +b b 1 +b c 2 +b d 2 +c b 2 +c c 2 +c d 3 +d b 3 +d c 3 +d d 3 + +-- !select8 -- +1 0 0 +1 1 1 +1 1 2 +1 1 3 +2 2 0 +2 2 2 +2 2 4 +2 2 6 +3 0 0 +3 3 3 +3 3 6 +3 3 9 +a b 1 +a c 1 +a d 1 +b b 1 +b c 2 +b d 2 +c b 2 +c c 2 +c d 3 +d b 3 +d c 3 +d d 3 + +-- !select9 -- +2 1 +3 2 +4 0 +4 3 + +-- !select10 -- +2 2 +2 3 +3 0 +3 4 +3 6 +4 6 +4 9 + +-- !select11 -- +a b 1 +a c 1 +a d 1 +b b 1 +b c 2 +b d 2 +c b 2 +c c 2 +c d 3 +d b 3 +d c 3 +d d 3 + +-- !select12 -- +1 0 0 +1 1 1 +1 1 2 +1 1 3 +2 2 0 +2 2 2 +2 2 4 +2 2 6 +3 0 0 +3 3 3 +3 3 6 +3 3 9 + +-- !select13 -- +2 0 + +-- !select14 -- +2 0 +2 1 +3 2 +4 0 +4 3 + +-- !select15 -- + +-- !select16 -- + +-- !select17 -- +1 0 +1 2 +1 3 +1 a +1 b +2 0 +2 4 +2 6 +2 b +2 c +3 0 +3 6 +3 9 +3 c +3 d + +-- !select18 -- +0 1 +0 3 +1 0 +1 1 +1 1 +1 2 +1 3 +1 a +1 b +2 0 +2 2 +2 2 +2 4 +2 6 +2 b +2 c +3 0 +3 3 +3 3 +3 6 +3 9 +3 c +3 d + +-- !select19 -- +0 1 +0 3 +1 0 +1 1 +1 2 +1 3 +1 a +1 b +2 0 +2 2 +2 4 +2 6 +2 b +2 c +3 0 +3 3 +3 6 +3 9 +3 c +3 d + +-- !select20 -- +1 0 +1 1 +1 1 +1 1 +1 a +1 a +1 a +1 b +2 2 +2 2 +2 2 +2 2 +2 b +2 b +2 c +2 c +3 0 +3 3 +3 3 +3 3 +3 c +3 d +3 d +3 d + +-- !select21 -- +1 0 +1 1 +1 a +1 b +2 2 +2 b +2 c +3 0 +3 3 +3 c +3 d + +-- !select24 -- +1 2 +10 20 +3 4 + +-- !select25 -- +1 3 +2 3 +2 8 +3 9 + +-- !select26 -- +1 3 +2 3 +2 8 +3 9 + +-- !select27 -- +1 a \N 10.0 +1 a \N 10.0 +2 b \N 20.0 + +-- !select28 -- +1 0.0 \N 4 +1 1.0 1 1 +1 1.0 1 3 +1 1.0 2 3 +1 1.0 3 4 +1 2.0 1 2 +1 3.0 1 3 +10 10.0 hello world +2 0.0 2 0 +2 2.0 2 2 +2 4.0 2 4 +2 6.0 2 6 +20 20.0 wangjuoo4 beautiful +3 3.0 3 3 +3 6.0 3 6 +3 9.0 3 9 + +-- !select29 -- +1 0.0 \N 4 +1 1.0 1 1 +1 1.0 1 3 +1 1.0 2 3 +1 1.0 3 4 +1 2.0 1 2 +1 3.0 1 3 +10 10.0 hello world +2 0.0 2 0 +2 2.0 2 2 +2 4.0 2 4 +2 6.0 2 6 +20 20.0 wangjuoo4 beautiful +3 3.0 3 3 +3 6.0 3 6 +3 9.0 3 9 + +-- !union30 -- +1.00 2.0 +1.01 2.0 +0.00 0.0 + +-- !union31 -- +1 2 +hell0 + +-- !union32 -- +1.0 2.0 + +-- !union33 -- +1.0 2.0 + +-- !union34 -- +1.0 2.0 +1.0 2.0 +1.0 2.0 + +-- !union35 -- +1.0 2.0 +1.0 2.0 + +-- !union36 -- +1.0 2.0 + +-- !union38 -- +2016-07-01 +2016-07-02 + +-- !union36 -- +1 2 + diff --git a/regression-test/suites/nereids_syntax_p0/set_operation.groovy b/regression-test/suites/nereids_syntax_p0/set_operation.groovy new file mode 100644 index 0000000000..8edfaa33cb --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/set_operation.groovy @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_nereids_set_operation") { + + sql "SET enable_nereids_planner=true" + sql "SET enable_vectorized_engine=true" + + sql "DROP TABLE IF EXISTS setOperationTable" + sql "DROP TABLE IF EXISTS setOperationTableNotNullable" + + sql """ + CREATE TABLE `setOperationTable` ( + `k1` bigint(20) NULL, + `k2` bigint(20) NULL, + `k3` bigint(20) NULL, + `k4` bigint(20) not null, + `k5` varchar(10), + `k6` varchar(10) + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k2`) BUCKETS 1 + PROPERTIES ('replication_num' = '1') + """ + + sql """ + CREATE TABLE `setOperationTableNotNullable` ( + `k1` bigint(20) NOT NULL, + `k2` bigint(20) NOT NULL, + `k3` bigint(20) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k2`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + """ + + sql """ + INSERT INTO setOperationTable VALUES + (1, 1, 1, 3, 'a', 'b'), + (1, 1, 2, 3, 'a', 'c'), + (1, 1, 3, 4, 'a' , 'd'), + (1, 0, null, 4, 'b' , 'b'), + (2, 2, 2, 5, 'b', 'c'), + (2, 2, 4, 5, 'b' , 'd'), + (2, 2, 6, 4, 'c', 'b'), + (2, 2, null, 4, 'c', 'c'), + (3, 3, 3, 3, 'c', 'd'), + (3, 3, 6, 3, 'd', 'b'), + (3, 3, 9, 4, 'd', 'c'), + (3, 0, null, 5, 'd', 'd') + """ + + sql """ + insert into setOperationTableNotNullable values + (1, 0, 0), + (1, 1, 3), + (1, 1, 2), + (1, 1, 1), + (2, 2, 0), + (2, 2, 6), + (2, 2, 4), + (2, 2, 2), + (3, 0, 0), + (3, 3, 9), + (3, 3, 6), + (3, 3, 3); + """ + + sql "SET enable_fallback_to_original_planner=false" + + // union + order_qt_select1 "select k1+1, k2 from setOperationTable union select k1, k3 from setOperationTableNotNullable;"; + order_qt_select2 "select k1+1, k3 from setOperationTableNotNullable union select k1+1, k2 from setOperationTable;"; + + order_qt_select3 "select k5, k6, k1 from setOperationTable union select k1, k2, k3 from setOperationTableNotNullable"; + order_qt_select4 "select k1, k2, k3 from setOperationTableNotNullable union select k5, k6, k1 from setOperationTable"; + + order_qt_select5 "select k1+1, k2 from setOperationTable union all select k1, k3 from setOperationTableNotNullable;"; + order_qt_select6 "select k1+1, k3 from setOperationTableNotNullable union all select k1+1, k2 from setOperationTable;"; + + order_qt_select7 "select k5, k6, k1 from setOperationTable union all select k1, k2, k3 from setOperationTableNotNullable"; + order_qt_select8 "select k1, k2, k3 from setOperationTableNotNullable union all select k5, k6, k1 from setOperationTable"; + + + // except + order_qt_select9 "select k1+1, k2 from setOperationTable except select k1, k3 from setOperationTableNotNullable;"; + order_qt_select10 "select k1+1, k3 from setOperationTableNotNullable except select k1+1, k2 from setOperationTable;"; + + order_qt_select11 "select k5, k6, k1 from setOperationTable except select k1, k2, k3 from setOperationTableNotNullable"; + order_qt_select12 "select k1, k2, k3 from setOperationTableNotNullable except select k5, k6, k1 from setOperationTable"; + + //intersect + order_qt_select13 "select k1+1, k2 from setOperationTable intersect select k1, k3 from setOperationTableNotNullable;"; + order_qt_select14 "select k1+1, k3 from setOperationTableNotNullable intersect select k1+1, k2 from setOperationTable;"; + + order_qt_select15 "select k5, k6, k1 from setOperationTable intersect select k1, k2, k3 from setOperationTableNotNullable"; + order_qt_select16 "select k1, k2, k3 from setOperationTableNotNullable intersect select k5, k6, k1 from setOperationTable"; + + // mix + order_qt_select17 """ + select k1, k3 from setOperationTableNotNullable union all + select k1, k5 from setOperationTable except + select k2, k1 from setOperationTableNotNullable + """ + + order_qt_select18 """ + select k1, k3 from setOperationTableNotNullable union all + (select k1, k5 from setOperationTable union + select k2, k1 from setOperationTableNotNullable) + """ + + order_qt_select19 """ + (select k1, k3 from setOperationTableNotNullable union all + select k1, k5 from setOperationTable) union + select k2, k1 from setOperationTableNotNullable + """ + + order_qt_select20 """ + select * from (select k1, k2 from setOperationTableNotNullable union all select k1, k5 from setOperationTable) t; + """ + + order_qt_select21 """ + select * from (select k1, k2 from setOperationTableNotNullable union select k1, k5 from setOperationTable) t; + """ + + order_qt_select24 """select * from (select 1 a, 2 b + union all select 3, 4 + union all select 10, 20) t where a0) + """ + + order_qt_select29 """ + select * from ( + (select 10, 10.0, 'hello', 'world') union all + (select k1, k2, k3, k4 from setOperationTable where k1=1) union all + (select 20, 20.0, 'wangjuoo4', 'beautiful') union all + (select k2, k3, k1, k3 from setOperationTableNotNullable where k2>0)) t + """ + + // test_union_basic + qt_union30 """select 1, 2 union select 1.01, 2.0 union (select 0.0001, 0.0000001)""" + qt_union31 """select 1, 2 union (select "hell0", "")""" + qt_union32 """select 1, 2 union select 1.0, 2.0 union (select 1.00000000, 2.00000)""" + qt_union33 """select 1, 2 union all select 1.0, 2.0 union (select 1.00000000, 2.00000) """ + qt_union34 """select 1, 2 union all select 1.0, 2.0 union all (select 1.00000000, 2.00000) """ + qt_union35 """select 1, 2 union select 1.0, 2.0 union all (select 1.00000000, 2.00000) """ + qt_union36 """select 1, 2 union distinct select 1.0, 2.0 union distinct (select 1.00000000, 2.00000) """ + qt_union38 """select "2016-07-01" union (select "2016-07-02")""" + + // test_union_bug + // PALO-3617 + qt_union36 """select * from (select 1 as a, 2 as b union select 3, 3) c where a = 1""" + + // cast类型 + def res5 = sql"""(select k1, k2 from setOperationTable) union (select k2, cast(k1 as int) from setOperationTable) + order by k1, k2""" + def res6 = sql"""(select k1, k2 from setOperationTable) union (select k2, cast(k1 as int) from setOperationTable order by k2) + order by k1, k2""" + check2_doris(res5, res6) + def res7 = sql"""(select k1, k2 from setOperationTable) union (select k2, cast(k3 as int) from setOperationTable) order by k1, k2""" + + def res8 = sql"""(select k1, k2 from setOperationTable) union (select k2, cast(k3 as int) from setOperationTable order by k2) order + by k1, k2""" + check2_doris(res7, res8) + // 不同类型不同个数 + test { + sql """select k1, k2 from setOperationTable union select k1, k3, k4 from setOperationTable order by k1, k2""" + check {result, exception, startTime, endTime -> + assertTrue(exception != null) + logger.info(exception.message) + } + } +}