database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids) implement setOperation (#15020)

Browse Source 
The pr implements the SetOperation.

- Adapt to the EliminateUnnecessaryProject rule to ensure that the project under SetOperation is not deleted.
- Add predicate pushdown of SetOperation
- Optimization: Merge multiple SetOperations with the same type and the same qualifier
- Optimization: merge oneRowRelation and union
This commit is contained in:
zhengshiJ
2022-12-20 15:14:29 +08:00
committed by GitHub
parent fdb54a346d
commit d9550c311e
46 changed files with 2940 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View File

@ -22,12 +22,6 @@ parser grammar DorisParser;
options { tokenVocab = DorisLexer; }
@members {
/**
* When false, INTERSECT is given the greater precedence over the other set
* operations (UNION, EXCEPT and MINUS) as per the SQL standard.
*/
public boolean legacy_setops_precedence_enabled = false;
/**
* When false, a literal with an exponent would be converted into
* double type rather than decimal type.
@ -37,7 +31,7 @@ options { tokenVocab = DorisLexer; }
/**
* When true, the behavior of keywords follows ANSI SQL standard.
*/
public boolean SQL_standard_keyword_behavior = false;
public boolean SQL_standard_keyword_behavior = true;
}
multiStatements
@ -77,6 +71,13 @@ query
queryTerm
: queryPrimary #queryTermDefault
| left=queryTerm operator=(UNION | EXCEPT | INTERSECT)
setQuantifier? right=queryTerm #setOperation
;
setQuantifier
: DISTINCT
| ALL
;
queryPrimary
@ -462,6 +463,7 @@ ansiNonReserved
| FUNCTIONS
| GLOBAL
| GROUPING
| GRAPH
| HOUR
| IF
| IGNORE
@ -520,6 +522,7 @@ ansiNonReserved
| PIVOT
| PLACING
| PLAN
| POLICY
| POSITION
| PRECEDING
| PRINCIPALS
@ -599,6 +602,7 @@ ansiNonReserved
| UPDATE
| USE
| VALUES
| VERBOSE
| VERSION
| VIEW
| VIEWS

20
fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@ -43,17 +44,21 @@ import java.util.Optional;
* e.g. select 100, 'value'
*/
public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRowRelation {
private final RelationId id;
private final List<NamedExpression> projects;
public UnboundOneRowRelation(List<NamedExpression> projects) {
this(projects, Optional.empty(), Optional.empty());
public UnboundOneRowRelation(RelationId id, List<NamedExpression> projects) {
this(id, projects, Optional.empty(), Optional.empty());
}
private UnboundOneRowRelation(List<NamedExpression> projects, Optional<GroupExpression> groupExpression,
private UnboundOneRowRelation(RelationId id,
List<NamedExpression> projects,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
super(PlanType.LOGICAL_UNBOUND_ONE_ROW_RELATION, groupExpression, logicalProperties);
Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(Slot.class)),
"OneRowRelation can not contains any slot");
this.id = id;
this.projects = ImmutableList.copyOf(projects);
}
@ -74,12 +79,12 @@ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRo
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new UnboundOneRowRelation(projects, groupExpression, Optional.of(logicalPropertiesSupplier.get()));
return new UnboundOneRowRelation(id, projects, groupExpression, Optional.of(logicalPropertiesSupplier.get()));
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new UnboundOneRowRelation(projects, Optional.empty(), logicalProperties);
return new UnboundOneRowRelation(id, projects, Optional.empty(), logicalProperties);
}
@Override
@ -95,6 +100,7 @@ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRo
@Override
public String toString() {
return Utils.toSqlString("UnboundOneRowRelation",
"relationId", id,
"projects", projects
);
}
@ -111,11 +117,11 @@ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRo
return false;
}
UnboundOneRowRelation that = (UnboundOneRowRelation) o;
return Objects.equals(projects, that.projects);
return Objects.equals(id, that.id) && Objects.equals(projects, that.projects);
}
@Override
public int hashCode() {
return Objects.hash(projects);
return Objects.hash(id, projects);
}
}

217
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -45,6 +45,7 @@ import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.AggregateExpression;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -63,9 +64,11 @@ import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
@ -74,20 +77,26 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.JoinUtils;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.planner.AggregationNode;
import org.apache.doris.planner.AssertNumRowsNode;
import org.apache.doris.planner.DataPartition;
import org.apache.doris.planner.EmptySetNode;
import org.apache.doris.planner.ExceptNode;
import org.apache.doris.planner.ExchangeNode;
import org.apache.doris.planner.HashJoinNode;
import org.apache.doris.planner.HashJoinNode.DistributionMode;
import org.apache.doris.planner.IntersectNode;
import org.apache.doris.planner.JoinNodeBase;
import org.apache.doris.planner.NestedLoopJoinNode;
import org.apache.doris.planner.OlapScanNode;
@ -96,6 +105,7 @@ import org.apache.doris.planner.PlanNode;
import org.apache.doris.planner.RepeatNode;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.planner.SelectNode;
import org.apache.doris.planner.SetOperationNode;
import org.apache.doris.planner.SortNode;
import org.apache.doris.planner.UnionNode;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
@ -104,6 +114,7 @@ import org.apache.doris.thrift.TPushAggOp;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -115,6 +126,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -364,6 +376,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
@Override
public PlanFragment visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation,
PlanTranslatorContext context) {
if (oneRowRelation.notBuildUnionNode()) {
return null;
}
List<Slot> slots = oneRowRelation.getLogicalProperties().getOutput();
TupleDescriptor oneRowTuple = generateTupleDesc(slots, null, context);
@ -382,7 +398,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
UnionNode unionNode = new UnionNode(context.nextPlanNodeId(), oneRowTuple.getId());
unionNode.setCardinality(1L);
unionNode.addConstExprList(legacyExprs);
unionNode.finalizeForNereids(oneRowTuple, oneRowTuple.getSlots());
unionNode.finalizeForNereids(oneRowTuple.getSlots(), new ArrayList<>());
PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), unionNode, DataPartition.UNPARTITIONED);
context.addPlanFragment(planFragment);
@ -1061,6 +1077,12 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
join.getFilterConjuncts().addAll(ExpressionUtils.extractConjunction(filter.getPredicates()));
}
PlanFragment inputFragment = filter.child(0).accept(this, context);
// Union contains oneRowRelation --> inputFragment = null
if (inputFragment == null) {
return inputFragment;
}
PlanNode planNode = inputFragment.getPlanRoot();
if (planNode instanceof ExchangeNode || planNode instanceof SortNode || planNode instanceof UnionNode) {
// the three nodes don't support conjuncts, need create a SelectNode to filter data
@ -1086,6 +1108,12 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
@Override
public PlanFragment visitPhysicalLimit(PhysicalLimit<? extends Plan> physicalLimit, PlanTranslatorContext context) {
PlanFragment inputFragment = physicalLimit.child(0).accept(this, context);
// Union contains oneRowRelation
if (inputFragment == null) {
return inputFragment;
}
PlanNode child = inputFragment.getPlanRoot();
// physical plan: limit --> sort
@ -1154,6 +1182,163 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return currentFragment;
}
/**
* Returns a new fragment with a UnionNode as its root. The data partition of the
* returned fragment and how the data of the child fragments is consumed depends on the
* data partitions of the child fragments:
* - All child fragments are unpartitioned or partitioned: The returned fragment has an
* UNPARTITIONED or RANDOM data partition, respectively. The UnionNode absorbs the
* plan trees of all child fragments.
* - Mixed partitioned/unpartitioned child fragments: The returned fragment is
* RANDOM partitioned. The plan trees of all partitioned child fragments are absorbed
* into the UnionNode. All unpartitioned child fragments are connected to the
* UnionNode via a RANDOM exchange, and remain unchanged otherwise.
*/
@Override
public PlanFragment visitPhysicalSetOperation(
PhysicalSetOperation setOperation, PlanTranslatorContext context) {
List<PlanFragment> childrenFragments = new ArrayList<>();
Map<Plan, PlanFragment> childNodeToFragment = new HashMap<>();
for (Plan plan : setOperation.children()) {
PlanFragment planFragment = plan.accept(this, context);
if (planFragment != null) {
childrenFragments.add(planFragment);
}
childNodeToFragment.put(plan, planFragment);
}
PlanFragment setOperationFragment;
SetOperationNode setOperationNode;
List<Slot> allSlots = new Builder<Slot>()
.addAll(setOperation.getOutput())
.build();
TupleDescriptor setTuple = generateTupleDesc(allSlots, null, context);
List<SlotDescriptor> outputSLotDescs = new ArrayList<>(setTuple.getSlots());
// create setOperationNode
if (setOperation instanceof PhysicalUnion) {
setOperationNode = new UnionNode(
context.nextPlanNodeId(), setTuple.getId());
} else if (setOperation instanceof PhysicalExcept) {
setOperationNode = new ExceptNode(
context.nextPlanNodeId(), setTuple.getId());
} else if (setOperation instanceof PhysicalIntersect) {
setOperationNode = new IntersectNode(
context.nextPlanNodeId(), setTuple.getId());
} else {
throw new RuntimeException("not support");
}
SetOperationResult setOperationResult = collectSetOperationResult(setOperation, childNodeToFragment);
for (List<Expression> expressions : setOperationResult.getResultExpressions()) {
List<Expr> resultExprs = expressions
.stream()
.map(expr -> ExpressionTranslator.translate(expr, context))
.collect(ImmutableList.toImmutableList());
setOperationNode.addResultExprLists(resultExprs);
}
for (List<Expression> expressions : setOperationResult.getConstExpressions()) {
List<Expr> constExprs = expressions
.stream()
.map(expr -> ExpressionTranslator.translate(expr, context))
.collect(ImmutableList.toImmutableList());
setOperationNode.addConstExprList(constExprs);
}
for (PlanFragment childFragment : childrenFragments) {
if (childFragment != null) {
setOperationNode.addChild(childFragment.getPlanRoot());
}
}
setOperationNode.finalizeForNereids(outputSLotDescs, outputSLotDescs);
// create setOperationFragment
// If all child fragments are unpartitioned, return a single unpartitioned fragment
// with a UnionNode that merges all child fragments.
if (allChildFragmentsUnPartitioned(childrenFragments)) {
setOperationFragment = new PlanFragment(
context.nextFragmentId(), setOperationNode, DataPartition.UNPARTITIONED);
// Absorb the plan trees of all childFragments into unionNode
// and fix up the fragment tree in the process.
for (int i = 0; i < childrenFragments.size(); ++i) {
setOperationFragment.setFragmentInPlanTree(setOperationNode.getChild(i));
setOperationFragment.addChildren(childrenFragments.get(i).getChildren());
}
} else {
setOperationFragment = new PlanFragment(context.nextFragmentId(), setOperationNode,
new DataPartition(TPartitionType.HASH_PARTITIONED,
setOperationNode.getMaterializedResultExprLists().get(0)));
for (int i = 0; i < childrenFragments.size(); ++i) {
PlanFragment childFragment = childrenFragments.get(i);
// Connect the unpartitioned child fragments to SetOperationNode via a random exchange.
connectChildFragmentNotCheckExchangeNode(
setOperationNode, i, setOperationFragment, childFragment, context);
childFragment.setOutputPartition(
DataPartition.hashPartitioned(setOperationNode.getMaterializedResultExprLists().get(i)));
}
}
context.addPlanFragment(setOperationFragment);
return setOperationFragment;
}
private List<Expression> castCommonDataTypeOutputs(List<Slot> outputs, List<Slot> childOutputs) {
List<Expression> newChildOutputs = new ArrayList<>();
for (int i = 0; i < outputs.size(); ++i) {
Slot right = childOutputs.get(i);
DataType tightestCommonType = outputs.get(i).getDataType();
Expression newRight = TypeCoercionUtils.castIfNotSameType(right, tightestCommonType);
newChildOutputs.add(newRight);
}
return ImmutableList.copyOf(newChildOutputs);
}
private SetOperationResult collectSetOperationResult(
PhysicalSetOperation setOperation, Map<Plan, PlanFragment> childPlanToFragment) {
List<List<Expression>> resultExprs = new ArrayList<>();
List<List<Expression>> constExprs = new ArrayList<>();
List<Slot> outputs = setOperation.getOutput();
for (Plan child : setOperation.children()) {
List<Expression> castCommonDataTypeOutputs = castCommonDataTypeOutputs(outputs, child.getOutput());
if (child.anyMatch(PhysicalOneRowRelation.class::isInstance) && childPlanToFragment.get(child) == null) {
constExprs.add(collectConstExpressions(castCommonDataTypeOutputs, child));
} else {
resultExprs.add(castCommonDataTypeOutputs);
}
}
return new SetOperationResult(resultExprs, constExprs);
}
private List<Expression> collectConstExpressions(
List<Expression> castExpressions, Plan child) {
List<Expression> newCastExpressions = new ArrayList<>();
for (int i = 0; i < castExpressions.size(); ++i) {
Expression expression = castExpressions.get(i);
if (expression instanceof Cast) {
newCastExpressions.add(expression.withChildren(
(collectPhysicalOneRowRelation(child).getProjects().get(i).children())));
} else {
newCastExpressions.add(
(collectPhysicalOneRowRelation(child).getProjects().get(i)));
}
}
return newCastExpressions;
}
private PhysicalOneRowRelation collectPhysicalOneRowRelation(Plan child) {
return (PhysicalOneRowRelation)
((ImmutableSet) child.collect(PhysicalOneRowRelation.class::isInstance)).asList().get(0);
}
private boolean allChildFragmentsUnPartitioned(List<PlanFragment> childrenFragments) {
boolean allChildFragmentsUnPartitioned = true;
for (PlanFragment child : childrenFragments) {
allChildFragmentsUnPartitioned = allChildFragmentsUnPartitioned && !child.isPartitioned();
}
return allChildFragmentsUnPartitioned;
}
private void extractExecSlot(Expr root, Set<Integer> slotRefList) {
if (root instanceof SlotRef) {
slotRefList.add(((SlotRef) root).getDesc().getId().asInt());
@ -1229,6 +1414,18 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
childFragment.setDestination((ExchangeNode) exchange);
}
private void connectChildFragmentNotCheckExchangeNode(PlanNode parent, int childIdx,
PlanFragment parentFragment, PlanFragment childFragment,
PlanTranslatorContext context) {
PlanNode exchange = new ExchangeNode(
context.nextPlanNodeId(), childFragment.getPlanRoot(), false);
exchange.setNumInstances(childFragment.getPlanRoot().getNumInstances());
childFragment.setPlanRoot(exchange.getChild(0));
exchange.setFragment(parentFragment);
parent.setChild(childIdx, exchange);
childFragment.setDestination((ExchangeNode) exchange);
}
/**
* Return unpartitioned fragment that merges the input fragment's output via
* an ExchangeNode.
@ -1433,4 +1630,22 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return Optional.empty();
}
}
private static class SetOperationResult {
private final List<List<Expression>> resultExpressions;
private final List<List<Expression>> constExpressions;
public SetOperationResult(List<List<Expression>> resultExpressions, List<List<Expression>> constExpressions) {
this.resultExpressions = ImmutableList.copyOf(resultExpressions);
this.constExpressions = ImmutableList.copyOf(constExpressions);
}
public List<List<Expression>> getConstExpressions() {
return constExpressions;
}
public List<List<Expression>> getResultExpressions() {
return resultExpressions;
}
}
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java
View File

@ -30,6 +30,7 @@ import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput;
import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy;
import org.apache.doris.nereids.rules.analysis.Scope;
import org.apache.doris.nereids.rules.analysis.UserAuthentication;
import org.apache.doris.nereids.rules.rewrite.logical.HideOneRowRelationUnderUnion;
import com.google.common.collect.ImmutableList;
@ -59,7 +60,8 @@ public class AnalyzeRulesJob extends BatchRulesJob {
new BindFunction(),
new ProjectToGlobalAggregate(),
new ResolveOrdinalInOrderByAndGroupBy(),
new ReplaceExpressionByChildOutput()
new ReplaceExpressionByChildOutput(),
new HideOneRowRelationUnderUnion()
)),
topDownBatch(ImmutableList.of(
new FillUpMissingSlots(),

4
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization
import org.apache.doris.nereids.rules.expression.rewrite.ExpressionOptimization;
import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate;
import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.BuildAggForUnion;
import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter;
@ -39,6 +40,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin;
import org.apache.doris.nereids.rules.rewrite.logical.InferPredicates;
import org.apache.doris.nereids.rules.rewrite.logical.LimitPushDown;
import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.logical.MergeSetOperations;
import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition;
import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet;
@ -102,6 +104,8 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob {
.add(topDownBatch(ImmutableList.of(new EliminateOrderByConstant())))
.add(topDownBatch(ImmutableList.of(new EliminateUnnecessaryProject())))
.add(topDownBatch(ImmutableList.of(new EliminateAggregate())))
.add(bottomUpBatch(ImmutableList.of(new MergeSetOperations())))
.add(topDownBatch(ImmutableList.of(new BuildAggForUnion())))
// this rule batch must keep at the end of rewrite to do some plan check
.add(bottomUpBatch(ImmutableList.of(new CheckAfterRewrite())))
.build();

41
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -71,12 +71,14 @@ import org.apache.doris.nereids.DorisParser.RelationContext;
import org.apache.doris.nereids.DorisParser.SelectClauseContext;
import org.apache.doris.nereids.DorisParser.SelectColumnClauseContext;
import org.apache.doris.nereids.DorisParser.SelectHintContext;
import org.apache.doris.nereids.DorisParser.SetOperationContext;
import org.apache.doris.nereids.DorisParser.SingleStatementContext;
import org.apache.doris.nereids.DorisParser.SortClauseContext;
import org.apache.doris.nereids.DorisParser.SortItemContext;
import org.apache.doris.nereids.DorisParser.StarContext;
import org.apache.doris.nereids.DorisParser.StatementDefaultContext;
import org.apache.doris.nereids.DorisParser.StringLiteralContext;
import org.apache.doris.nereids.DorisParser.SubqueryContext;
import org.apache.doris.nereids.DorisParser.SubqueryExpressionContext;
import org.apache.doris.nereids.DorisParser.TableAliasContext;
import org.apache.doris.nereids.DorisParser.TableNameContext;
@ -169,6 +171,7 @@ import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
@ -176,8 +179,10 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@ -186,6 +191,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.logical.RelationUtil;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.IntegerType;
@ -317,6 +323,39 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
});
}
@Override
public LogicalPlan visitSetOperation(SetOperationContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {
LogicalPlan leftQuery = plan(ctx.left);
LogicalPlan rightQuery = plan(ctx.right);
Qualifier qualifier;
if (ctx.setQuantifier() == null || ctx.setQuantifier().DISTINCT() != null) {
qualifier = Qualifier.DISTINCT;
} else {
qualifier = Qualifier.ALL;
}
List<Plan> newChildren = new ImmutableList.Builder<Plan>()
.add(leftQuery)
.add(rightQuery)
.build();
if (ctx.UNION() != null) {
return new LogicalUnion(qualifier, newChildren);
} else if (ctx.EXCEPT() != null) {
return new LogicalExcept(qualifier, newChildren);
} else if (ctx.INTERSECT() != null) {
return new LogicalIntersect(qualifier, newChildren);
}
throw new ParseException("not support", ctx);
});
}
@Override
public LogicalPlan visitSubquery(SubqueryContext ctx) {
return ParserUtils.withOrigin(ctx, () -> visitQuery(ctx.query()));
}
@Override
public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {
@ -1076,7 +1115,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return ParserUtils.withOrigin(selectCtx, () -> {
// fromClause does not exists.
List<NamedExpression> projects = getNamedExpressions(selectCtx.namedExpressionSeq());
return new UnboundOneRowRelation(projects);
return new UnboundOneRowRelation(RelationUtil.newRelationId(), projects);
});
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java
View File

@ -73,4 +73,8 @@ public class PatternDescriptor<INPUT_TYPE extends Plan> {
MatchedMultiAction<INPUT_TYPE, OUTPUT_TYPE> matchedAction) {
return new PatternMatcher<>(pattern, defaultPromise, matchedAction);
}
public Pattern<INPUT_TYPE> getPattern() {
return pattern;
}
}

98
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java
View File

@ -25,15 +25,21 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.UnaryPlan;
import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalBinary;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnary;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.physical.PhysicalBinary;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnary;
import java.util.Arrays;
/**
* An interface provided some PatternDescriptor.
* Child Interface(RuleFactory) can use to declare a pattern shape, then convert to a rule.
@ -174,6 +180,98 @@ public interface Patterns {
return new PatternDescriptor(new TypePattern(LogicalRelation.class), defaultPromise());
}
/**
* create a logicalSetOperation pattern.
*/
default PatternDescriptor<LogicalSetOperation>
logicalSetOperation(
PatternDescriptor... children) {
return new PatternDescriptor(
new TypePattern(LogicalSetOperation.class,
Arrays.stream(children)
.map(PatternDescriptor::getPattern)
.toArray(Pattern[]::new)),
defaultPromise());
}
/**
* create a logicalSetOperation group.
*/
default PatternDescriptor<LogicalSetOperation> logicalSetOperation() {
return new PatternDescriptor(
new TypePattern(LogicalSetOperation.class, multiGroup().pattern),
defaultPromise());
}
/**
* create a logicalUnion pattern.
*/
default PatternDescriptor<LogicalUnion>
logicalUnion(
PatternDescriptor... children) {
return new PatternDescriptor(
new TypePattern(LogicalUnion.class,
Arrays.stream(children)
.map(PatternDescriptor::getPattern)
.toArray(Pattern[]::new)),
defaultPromise());
}
/**
* create a logicalUnion group.
*/
default PatternDescriptor<LogicalUnion> logicalUnion() {
return new PatternDescriptor(
new TypePattern(LogicalUnion.class, multiGroup().pattern),
defaultPromise());
}
/**
* create a logicalExcept pattern.
*/
default PatternDescriptor<LogicalExcept>
logicalExcept(
PatternDescriptor... children) {
return new PatternDescriptor(
new TypePattern(LogicalExcept.class,
Arrays.stream(children)
.map(PatternDescriptor::getPattern)
.toArray(Pattern[]::new)),
defaultPromise());
}
/**
* create a logicalExcept group.
*/
default PatternDescriptor<LogicalExcept> logicalExcept() {
return new PatternDescriptor(
new TypePattern(LogicalExcept.class, multiGroup().pattern),
defaultPromise());
}
/**
* create a logicalUnion pattern.
*/
default PatternDescriptor<LogicalIntersect>
logicalIntersect(
PatternDescriptor... children) {
return new PatternDescriptor(
new TypePattern(LogicalIntersect.class,
Arrays.stream(children)
.map(PatternDescriptor::getPattern)
.toArray(Pattern[]::new)),
defaultPromise());
}
/**
* create a logicalUnion group.
*/
default PatternDescriptor<LogicalIntersect> logicalIntersect() {
return new PatternDescriptor(
new TypePattern(LogicalIntersect.class, multiGroup().pattern),
defaultPromise());
}
/* abstract physical plan patterns */
/**

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View File

@ -28,7 +28,9 @@ import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose
import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject;
import org.apache.doris.nereids.rules.implementation.LogicalAssertNumRowsToPhysicalAssertNumRows;
import org.apache.doris.nereids.rules.implementation.LogicalEmptyRelationToPhysicalEmptyRelation;
import org.apache.doris.nereids.rules.implementation.LogicalExceptToPhysicalExcept;
import org.apache.doris.nereids.rules.implementation.LogicalFilterToPhysicalFilter;
import org.apache.doris.nereids.rules.implementation.LogicalIntersectToPhysicalIntersect;
import org.apache.doris.nereids.rules.implementation.LogicalJoinToHashJoin;
import org.apache.doris.nereids.rules.implementation.LogicalJoinToNestedLoopJoin;
import org.apache.doris.nereids.rules.implementation.LogicalLimitToPhysicalLimit;
@ -39,6 +41,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalRepeatToPhysicalRepe
import org.apache.doris.nereids.rules.implementation.LogicalSortToPhysicalQuickSort;
import org.apache.doris.nereids.rules.implementation.LogicalTVFRelationToPhysicalTVFRelation;
import org.apache.doris.nereids.rules.implementation.LogicalTopNToPhysicalTopN;
import org.apache.doris.nereids.rules.implementation.LogicalUnionToPhysicalUnion;
import org.apache.doris.nereids.rules.rewrite.AggregateStrategies;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateOuterJoin;
import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters;
@ -49,6 +52,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughAggre
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughRepeat;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughSetOperation;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownJoinOtherCondition;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughLimit;
@ -83,6 +87,7 @@ public class RuleSet {
new PushdownExpressionsInHashCondition(),
new PushdownFilterThroughAggregation(),
new PushdownFilterThroughRepeat(),
new PushdownFilterThroughSetOperation(),
new PushdownProjectThroughLimit(),
new EliminateOuterJoin(),
new MergeProjects(),
@ -104,6 +109,9 @@ public class RuleSet {
.add(new LogicalEmptyRelationToPhysicalEmptyRelation())
.add(new LogicalTVFRelationToPhysicalTVFRelation())
.add(new AggregateStrategies())
.add(new LogicalUnionToPhysicalUnion())
.add(new LogicalExceptToPhysicalExcept())
.add(new LogicalIntersectToPhysicalIntersect())
.build();
public static final List<Rule> LEFT_DEEP_TREE_JOIN_REORDER = planRuleFactories()

11
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -40,6 +40,7 @@ public enum RuleType {
BINDING_REPEAT_SLOT(RuleTypeClass.REWRITE),
BINDING_HAVING_SLOT(RuleTypeClass.REWRITE),
BINDING_SORT_SLOT(RuleTypeClass.REWRITE),
BINDING_SORT_SET_OPERATION_SLOT(RuleTypeClass.REWRITE),
BINDING_LIMIT_SLOT(RuleTypeClass.REWRITE),
BINDING_ONE_ROW_RELATION_FUNCTION(RuleTypeClass.REWRITE),
BINDING_PROJECT_FUNCTION(RuleTypeClass.REWRITE),
@ -51,6 +52,7 @@ public enum RuleType {
BINDING_SORT_FUNCTION(RuleTypeClass.REWRITE),
BINDING_JOIN_FUNCTION(RuleTypeClass.REWRITE),
BINDING_UNBOUND_TVF_RELATION_FUNCTION(RuleTypeClass.REWRITE),
BINDING_SET_OPERATION_SLOT(RuleTypeClass.REWRITE),
REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT(RuleTypeClass.REWRITE),
@ -89,6 +91,7 @@ public enum RuleType {
DISTINCT_AGGREGATE_DISASSEMBLE(RuleTypeClass.REWRITE),
ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE),
MERGE_PROJECTS(RuleTypeClass.REWRITE),
MARK_NECESSARY_PROJECT(RuleTypeClass.REWRITE),
LOGICAL_SUB_QUERY_ALIAS_TO_LOGICAL_PROJECT(RuleTypeClass.REWRITE),
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
@ -115,6 +118,7 @@ public enum RuleType {
PUSH_FILTER_INSIDE_JOIN(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_PROJECT(RuleTypeClass.REWRITE),
PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE),
// column prune rules,
COLUMN_PRUNE_AGGREGATION_CHILD(RuleTypeClass.REWRITE),
COLUMN_PRUNE_FILTER_CHILD(RuleTypeClass.REWRITE),
@ -151,6 +155,10 @@ public enum RuleType {
OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE),
PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE),
EXTRACT_SINGLE_TABLE_EXPRESSION_FROM_DISJUNCTION(RuleTypeClass.REWRITE),
HIDE_ONE_ROW_RELATION_UNDER_UNION(RuleTypeClass.REWRITE),
MERGE_SET_OPERATION(RuleTypeClass.REWRITE),
BUILD_AGG_FOR_UNION(RuleTypeClass.REWRITE),
REWRITE_SENTINEL(RuleTypeClass.REWRITE),
// limit push down
PUSH_LIMIT_THROUGH_JOIN(RuleTypeClass.REWRITE),
@ -203,6 +211,9 @@ public enum RuleType {
TWO_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI(RuleTypeClass.IMPLEMENTATION),
TWO_PHASE_AGGREGATE_WITH_MULTI_DISTINCT(RuleTypeClass.IMPLEMENTATION),
THREE_PHASE_AGGREGATE_WITH_DISTINCT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_UNION_TO_PHYSICAL_UNION(RuleTypeClass.IMPLEMENTATION),
LOGICAL_EXCEPT_TO_PHYSICAL_EXCEPT(RuleTypeClass.IMPLEMENTATION),
LOGICAL_INTERSECT_TO_PHYSICAL_INTERSECT(RuleTypeClass.IMPLEMENTATION),
IMPLEMENTATION_SENTINEL(RuleTypeClass.IMPLEMENTATION),
LOGICAL_SEMI_JOIN_SEMI_JOIN_TRANPOSE_PROJECT(RuleTypeClass.EXPLORATION),

43
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java
View File

@ -49,14 +49,18 @@ import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.LeafPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import org.apache.doris.planner.PlannerContext;
@ -246,6 +250,22 @@ public class BindSlotReference implements AnalysisRuleFactory {
return new LogicalSort<>(sortItemList, sort.child());
})
),
RuleType.BINDING_SORT_SET_OPERATION_SLOT.build(
logicalSort(logicalSetOperation()).when(Plan::canBind).thenApply(ctx -> {
LogicalSort<LogicalSetOperation> sort = ctx.root;
List<OrderKey> sortItemList = sort.getOrderKeys()
.stream()
.map(orderKey -> {
Expression item = bind(orderKey.getExpr(), sort.children(), sort, ctx.cascadesContext);
if (item.containsType(UnboundSlot.class)) {
item = bind(item, sort.child().children(), sort, ctx.cascadesContext);
}
return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst());
}).collect(Collectors.toList());
return new LogicalSort<>(sortItemList, sort.child());
})
),
RuleType.BINDING_HAVING_SLOT.build(
logicalHaving(any()).when(Plan::canBind).thenApply(ctx -> {
LogicalHaving<Plan> having = ctx.root;
@ -272,6 +292,29 @@ public class BindSlotReference implements AnalysisRuleFactory {
return new LogicalOneRowRelation(projects);
})
),
RuleType.BINDING_SET_OPERATION_SLOT.build(
logicalSetOperation().when(Plan::canBind).then(setOperation -> {
// check whether the left and right child output columns are the same
if (setOperation.child(0).getOutput().size() != setOperation.child(1).getOutput().size()) {
throw new AnalysisException("Operands have unequal number of columns:\n"
+ "'" + setOperation.child(0).getOutput() + "' has "
+ setOperation.child(0).getOutput().size() + " column(s)\n"
+ "'" + setOperation.child(1).getOutput() + "' has "
+ setOperation.child(1).getOutput().size() + " column(s)");
}
// INTERSECT and EXCEPT does not support ALL qualifie
if (setOperation.getQualifier() == Qualifier.ALL
&& (setOperation instanceof LogicalExcept || setOperation instanceof LogicalIntersect)) {
throw new AnalysisException("INTERSECT and EXCEPT does not support ALL qualifie");
}
List<List<Expression>> castExpressions = setOperation.collectCastExpressions();
List<NamedExpression> newOutputs = setOperation.buildNewOutputs(castExpressions.get(0));
return setOperation.withNewOutputs(newOutputs);
})
),
RuleType.BINDING_NON_LEAF_LOGICAL_PLAN.build(
logicalPlan()

36
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExceptToPhysicalExcept.java Normal file
View File

@ -0,0 +1,36 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.implementation;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept;
/**
* Implementation rule that convert logical Except to Physical Except.
*/
public class LogicalExceptToPhysicalExcept extends OneImplementationRuleFactory {
@Override
public Rule build() {
return logicalExcept().then(except ->
new PhysicalExcept(except.getQualifier(),
except.getLogicalProperties(),
except.children())
).toRule(RuleType.LOGICAL_EXCEPT_TO_PHYSICAL_EXCEPT);
}
}

36
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIntersectToPhysicalIntersect.java Normal file
View File

@ -0,0 +1,36 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.implementation;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
/**
* Implementation rule that convert logical Intersect to Physical Intersect.
*/
public class LogicalIntersectToPhysicalIntersect extends OneImplementationRuleFactory {
@Override
public Rule build() {
return logicalIntersect().then(intersect ->
new PhysicalIntersect(intersect.getQualifier(),
intersect.getLogicalProperties(),
intersect.children())
).toRule(RuleType.LOGICAL_INTERSECT_TO_PHYSICAL_INTERSECT);
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java
View File

@ -28,7 +28,8 @@ public class LogicalOneRowRelationToPhysicalOneRowRelation extends OneImplementa
@Override
public Rule build() {
return logicalOneRowRelation()
.then(relation -> new PhysicalOneRowRelation(relation.getProjects(), relation.getLogicalProperties()))
.then(relation -> new PhysicalOneRowRelation(
relation.getProjects(), relation.buildUnionNode(), relation.getLogicalProperties()))
.toRule(RuleType.LOGICAL_ONE_ROW_RELATION_TO_PHYSICAL_ONE_ROW_RELATION);
}
}

36
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalUnionToPhysicalUnion.java Normal file
View File

@ -0,0 +1,36 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.implementation;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
/**
* Implementation rule that convert logical Union to Physical Union.
*/
public class LogicalUnionToPhysicalUnion extends OneImplementationRuleFactory {
@Override
public Rule build() {
return logicalUnion().then(union ->
new PhysicalUnion(union.getQualifier(),
union.getLogicalProperties(),
union.children())
).toRule(RuleType.LOGICAL_UNION_TO_PHYSICAL_UNION);
}
}

43
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/BuildAggForUnion.java Normal file
View File

@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import java.util.Optional;
/**
* For distinct union, add agg node.
*/
public class BuildAggForUnion extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalUnion().whenNot(LogicalUnion::hasBuildAgg).then(union -> {
if (union.getQualifier() == Qualifier.DISTINCT) {
return new LogicalAggregate(union.getOutputs(), union.getOutputs(),
true, Optional.empty(), union.withHasBuildAgg());
}
return union;
}).toRule(RuleType.BUILD_AGG_FOR_UNION);
}
}

57
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java
View File

@ -19,33 +19,54 @@ package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* remove the project that output same with its child to avoid we get two consecutive projects in best plan.
* for more information, please see <a href="https://github.com/apache/doris/pull/13886">this PR</a>
*/
public class EliminateUnnecessaryProject extends OneRewriteRuleFactory {
public class EliminateUnnecessaryProject implements RewriteRuleFactory {
@Override
public Rule build() {
return logicalProject(any())
.when(project -> project.getOutputSet().equals(project.child().getOutputSet()))
.thenApply(ctx -> {
int rootGroupId = ctx.cascadesContext.getMemo().getRoot().getGroupId().asInt();
LogicalProject<Plan> project = ctx.root;
// if project is root, we need to ensure the output order is same.
if (project.getGroupExpression().get().getOwnerGroup().getGroupId().asInt() == rootGroupId) {
if (project.getOutput().equals(project.child().getOutput())) {
return project.child();
public List<Rule> buildRules() {
return ImmutableList.of(
RuleType.MARK_NECESSARY_PROJECT.build(
logicalSetOperation(logicalProject(), group())
.thenApply(ctx -> {
LogicalProject project = (LogicalProject) ctx.root.child(0);
return ctx.root.withChildren(project.withEliminate(false), ctx.root.child(1));
})
),
RuleType.MARK_NECESSARY_PROJECT.build(
logicalSetOperation(group(), logicalProject())
.thenApply(ctx -> {
LogicalProject project = (LogicalProject) ctx.root.child(1);
return ctx.root.withChildren(ctx.root.child(0), project.withEliminate(false));
})
),
RuleType.ELIMINATE_UNNECESSARY_PROJECT.build(
logicalProject(any())
.when(LogicalProject::canEliminate)
.when(project -> project.getOutputSet().equals(project.child().getOutputSet()))
.thenApply(ctx -> {
int rootGroupId = ctx.cascadesContext.getMemo().getRoot().getGroupId().asInt();
LogicalProject<Plan> project = ctx.root;
// if project is root, we need to ensure the output order is same.
if (project.getGroupExpression().get().getOwnerGroup().getGroupId().asInt()
== rootGroupId) {
if (project.getOutput().equals(project.child().getOutput())) {
return project.child();
} else {
return null;
}
} else {
return null;
return project.child();
}
} else {
return project.child();
}
}).toRule(RuleType.ELIMINATE_UNNECESSARY_PROJECT);
})));
}
}

74
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java Normal file
View File

@ -0,0 +1,74 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.analysis.AnalysisRuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* 1. Include oneRowRelation in the union, hide the oneRowRelation, and reduce the generation of union nodes.
* eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2;
* before:
* logicalUnion()
* / \
* logicalUnion() logicalProject
* / \
* logicalProject logicalOneRowRelation(BuildUnionNode:true)
* eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2;
*
* after:
* logicalUnion()
* / \
* logicalUnion() logicalProject
* / \
* logicalProject logicalOneRowRelation(BuildUnionNode:false)
*/
public class HideOneRowRelationUnderUnion implements AnalysisRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build(
logicalUnion(logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode), group())
.then(union -> {
List<Plan> newChildren = new ImmutableList.Builder<Plan>()
.add(((LogicalOneRowRelation) union.child(0)).withBuildUnionNode(false))
.add(union.child(1))
.build();
return union.withChildren(newChildren);
})
),
RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build(
logicalUnion(group(), logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode))
.then(union -> {
List<Plan> children = new ImmutableList.Builder<Plan>()
.add(union.child(0))
.add(((LogicalOneRowRelation) union.child(1)).withBuildUnionNode(false))
.build();
return union.withChildren(children);
})
)
);
}
}

98
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java Normal file
View File

@ -0,0 +1,98 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* optimization.
* Merge nodes of the same type and same qualifier.
*
* eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2;
* before:
* logicalUnion()
* / \
* logicalUnion() logicalProject
* / \
* logicalProject logicalOneRowRelation
*
* after:
* 2. MERGE_SET_OPERATION
* logicalUnion()
* / \ \
* logicalProject logicalOneRowRelation logicalProject
*/
public class MergeSetOperations implements RewriteRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
RuleType.MERGE_SET_OPERATION.build(
logicalSetOperation(logicalSetOperation(), group()).thenApply(ctx -> {
LogicalSetOperation parentSetOperation = ctx.root;
LogicalSetOperation childSetOperation = (LogicalSetOperation) parentSetOperation.child(0);
if (isSameClass(parentSetOperation, childSetOperation)
&& isSameQualifierOrChildQualifierIsAll(parentSetOperation, childSetOperation)) {
List<Plan> newChildren = new ImmutableList.Builder<Plan>()
.addAll(childSetOperation.children())
.add(parentSetOperation.child(1))
.build();
return parentSetOperation.withChildren(newChildren);
}
return parentSetOperation;
})
),
RuleType.MERGE_SET_OPERATION.build(
logicalSetOperation(group(), logicalSetOperation()).thenApply(ctx -> {
LogicalSetOperation parentSetOperation = ctx.root;
LogicalSetOperation childSetOperation = (LogicalSetOperation) parentSetOperation.child(1);
if (isSameClass(parentSetOperation, childSetOperation)
&& isSameQualifierOrChildQualifierIsAll(parentSetOperation, childSetOperation)) {
List<Plan> newChildren = new ImmutableList.Builder<Plan>()
.add(parentSetOperation.child(0))
.addAll(childSetOperation.children())
.build();
return parentSetOperation.withNewChildren(newChildren);
}
return parentSetOperation;
})
)
);
}
private boolean isSameQualifierOrChildQualifierIsAll(LogicalSetOperation parentSetOperation,
LogicalSetOperation childSetOperation) {
return parentSetOperation.getQualifier() == childSetOperation.getQualifier()
|| childSetOperation.getQualifier() == Qualifier.ALL;
}
private boolean isSameClass(LogicalSetOperation parentSetOperation,
LogicalSetOperation childSetOperation) {
return parentSetOperation.getClass().isAssignableFrom(childSetOperation.getClass());
}
}

74
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSetOperation.java Normal file
View File

@ -0,0 +1,74 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Convert the expression in the filter into the output column corresponding to the child node and push it down.
*/
public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(logicalSetOperation()).then(filter -> {
LogicalSetOperation setOperation = filter.child();
if (setOperation instanceof LogicalUnion && ((LogicalUnion) setOperation).hasPushedFilter()) {
return filter;
}
List<Plan> newChildren = new ArrayList<>();
for (Plan child : setOperation.children()) {
Map<Expression, Expression> replaceMap = new HashMap<>();
for (int i = 0; i < setOperation.getOutputs().size(); ++i) {
NamedExpression output = setOperation.getOutputs().get(i);
replaceMap.put(output, child.getOutput().get(i));
}
List<Expression> newFilterPredicates = Lists.newArrayList();
ExpressionUtils.extractConjunction(filter.getPredicates()).forEach(conjunct -> {
newFilterPredicates.add(ExpressionUtils.replace(conjunct, replaceMap));
});
newChildren.add(new LogicalFilter<>(ExpressionUtils.and(newFilterPredicates), child));
}
if (setOperation instanceof LogicalUnion && setOperation.getQualifier() == Qualifier.DISTINCT) {
return new LogicalFilter<>(filter.getPredicates(),
((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren));
}
return setOperation.withNewChildren(newChildren);
}).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_SET_OPERATION);
}
}

104
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
View File

@ -35,11 +35,14 @@ import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation;
import org.apache.doris.nereids.trees.plans.algebra.Project;
import org.apache.doris.nereids.trees.plans.algebra.Repeat;
import org.apache.doris.nereids.trees.plans.algebra.Scan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.algebra.TopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
@ -49,12 +52,15 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLocalQuickSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
@ -66,6 +72,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
@ -186,6 +193,24 @@ public class StatsCalculator extends DefaultPlanVisitor<StatsDeriveResult, Void>
return computeAssertNumRows(assertNumRows.getAssertNumRowsElement().getDesiredNumOfRows());
}
@Override
public StatsDeriveResult visitLogicalUnion(
LogicalUnion union, Void context) {
return computeUnion(union);
}
@Override
public StatsDeriveResult visitLogicalExcept(
LogicalExcept except, Void context) {
return computeExcept();
}
@Override
public StatsDeriveResult visitLogicalIntersect(
LogicalIntersect intersect, Void context) {
return computeIntersect(intersect);
}
@Override
public StatsDeriveResult visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelation, Void context) {
return computeEmptyRelation(emptyRelation);
@ -274,6 +299,21 @@ public class StatsCalculator extends DefaultPlanVisitor<StatsDeriveResult, Void>
return computeAssertNumRows(assertNumRows.getAssertNumRowsElement().getDesiredNumOfRows());
}
@Override
public StatsDeriveResult visitPhysicalUnion(PhysicalUnion union, Void context) {
return computeUnion(union);
}
@Override
public StatsDeriveResult visitPhysicalExcept(PhysicalExcept except, Void context) {
return computeExcept();
}
@Override
public StatsDeriveResult visitPhysicalIntersect(PhysicalIntersect intersect, Void context) {
return computeIntersect(intersect);
}
private StatsDeriveResult computeAssertNumRows(long desiredNumOfRows) {
StatsDeriveResult statsDeriveResult = groupExpression.childStatistics(0);
statsDeriveResult.updateByLimit(1);
@ -434,4 +474,68 @@ public class StatsCalculator extends DefaultPlanVisitor<StatsDeriveResult, Void>
int rowCount = 0;
return new StatsDeriveResult(rowCount, columnStatsMap);
}
private StatsDeriveResult computeUnion(SetOperation setOperation) {
StatsDeriveResult leftStatsResult = groupExpression.childStatistics(0);
Map<Id, ColumnStatistic> leftStatsSlotIdToColumnStats = leftStatsResult.getSlotIdToColumnStats();
Map<Id, ColumnStatistic> newColumnStatsMap = new HashMap<>();
double rowCount = leftStatsResult.getRowCount();
for (int j = 0; j < setOperation.getArity() - 1; ++j) {
StatsDeriveResult rightStatsResult = groupExpression.childStatistics(j + 1);
Map<Id, ColumnStatistic> rightStatsSlotIdToColumnStats = rightStatsResult.getSlotIdToColumnStats();
for (int i = 0; i < setOperation.getOutputs().size(); ++i) {
Slot leftSlot = getLeftSlot(j, i, setOperation);
Slot rightSlot = setOperation.getChildOutput(j + 1).get(i);
ColumnStatistic leftStats = getLeftStats(j, leftSlot, leftStatsSlotIdToColumnStats, newColumnStatsMap);
ColumnStatistic rightStats = rightStatsSlotIdToColumnStats.get(rightSlot.getExprId());
newColumnStatsMap.put(setOperation.getOutputs().get(i).getExprId(), new ColumnStatistic(
leftStats.count + rightStats.count,
leftStats.ndv + rightStats.ndv,
leftStats.avgSizeByte,
leftStats.numNulls + rightStats.numNulls,
leftStats.dataSize + rightStats.dataSize,
Math.min(leftStats.minValue, rightStats.minValue),
Math.max(leftStats.maxValue, rightStats.maxValue),
1.0 / (leftStats.ndv + rightStats.ndv),
leftStats.minExpr,
leftStats.maxExpr,
leftStats.isUnKnown));
}
rowCount = Math.min(rowCount, rightStatsResult.getRowCount());
}
return new StatsDeriveResult(rowCount, newColumnStatsMap);
}
private Slot getLeftSlot(int fistSetOperation, int outputSlotIdx, SetOperation setOperation) {
return fistSetOperation == 0
? setOperation.getFirstOutput().get(outputSlotIdx)
: setOperation.getOutputs().get(outputSlotIdx).toSlot();
}
private ColumnStatistic getLeftStats(int fistSetOperation,
Slot leftSlot,
Map<Id, ColumnStatistic> leftStatsSlotIdToColumnStats,
Map<Id, ColumnStatistic> newColumnStatsMap) {
return fistSetOperation == 0
? leftStatsSlotIdToColumnStats.get(leftSlot.getExprId())
: newColumnStatsMap.get(leftSlot.getExprId());
}
private StatsDeriveResult computeExcept() {
return groupExpression.childStatistics(0);
}
private StatsDeriveResult computeIntersect(SetOperation setOperation) {
StatsDeriveResult leftStatsResult = groupExpression.childStatistics(0);
double rowCount = leftStatsResult.getRowCount();
for (int i = 1; i < setOperation.getArity(); ++i) {
rowCount = Math.min(rowCount, groupExpression.childStatistics(i).getRowCount());
}
return new StatsDeriveResult(
rowCount, leftStatsResult.getSlotIdToColumnStats());
}
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
View File

@ -48,6 +48,9 @@ public enum PlanType {
LOGICAL_HAVING,
LOGICAL_MULTI_JOIN,
LOGICAL_CHECK_POLICY,
LOGICAL_UNION,
LOGICAL_EXCEPT,
LOGICAL_INTERSECT,
GROUP_PLAN,
// physical plan
@ -68,5 +71,8 @@ public enum PlanType {
PHYSICAL_NESTED_LOOP_JOIN,
PHYSICAL_EXCHANGE,
PHYSICAL_DISTRIBUTION,
PHYSICAL_ASSERT_NUM_ROWS;
PHYSICAL_ASSERT_NUM_ROWS,
PHYSICAL_UNION,
PHYSICAL_EXCEPT,
PHYSICAL_INTERSECT
}

46
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/SetOperation.java Normal file
View File

@ -0,0 +1,46 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.algebra;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import java.util.List;
/**
* Common interface for logical/physical SetOperation.
*/
public interface SetOperation {
/**
* SetOperation qualifier type.
*/
enum Qualifier {
ALL,
DISTINCT
}
Qualifier getQualifier();
List<Slot> getFirstOutput();
List<Slot> getChildOutput(int i);
List<NamedExpression> getOutputs();
int getArity();
}

88
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java Normal file
View File

@ -0,0 +1,88 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import java.util.List;
import java.util.Optional;
/**
* Logical Except.
*/
public class LogicalExcept extends LogicalSetOperation {
public LogicalExcept(Qualifier qualifier, List<Plan> inputs) {
super(PlanType.LOGICAL_EXCEPT, qualifier, inputs);
}
public LogicalExcept(Qualifier qualifier, List<NamedExpression> outputs, List<Plan> inputs) {
super(PlanType.LOGICAL_EXCEPT, qualifier, outputs, inputs);
}
public LogicalExcept(Qualifier qualifier, List<NamedExpression> outputs,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<Plan> inputs) {
super(PlanType.LOGICAL_EXCEPT, qualifier, outputs, groupExpression, logicalProperties, inputs);
}
@Override
public String toString() {
return Utils.toSqlString("LogicalExcept",
"qualifier", qualifier,
"outputs", outputs);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalExcept(this, context);
}
@Override
public LogicalExcept withChildren(List<Plan> children) {
return new LogicalExcept(qualifier, outputs, children);
}
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalExcept(qualifier, outputs, groupExpression,
Optional.of(getLogicalProperties()), children);
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new LogicalExcept(qualifier, outputs,
Optional.empty(), logicalProperties, children);
}
@Override
public Plan withNewOutputs(List<NamedExpression> newOutputs) {
return new LogicalExcept(qualifier, newOutputs, Optional.empty(), Optional.empty(), children);
}
@Override
public Plan withNewChildren(List<Plan> children) {
return withChildren(children);
}
}

90
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java Normal file
View File

@ -0,0 +1,90 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import java.util.List;
import java.util.Optional;
/**
* Logical Intersect.
*/
public class LogicalIntersect extends LogicalSetOperation {
public LogicalIntersect(Qualifier qualifier, List<Plan> inputs) {
super(PlanType.LOGICAL_INTERSECT, qualifier, inputs);
}
public LogicalIntersect(Qualifier qualifier, List<NamedExpression> outputs,
List<Plan> inputs) {
super(PlanType.LOGICAL_INTERSECT, qualifier, outputs, inputs);
}
public LogicalIntersect(Qualifier qualifier, List<NamedExpression> outputs,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<Plan> inputs) {
super(PlanType.LOGICAL_INTERSECT, qualifier, outputs, groupExpression, logicalProperties, inputs);
}
@Override
public String toString() {
return Utils.toSqlString("LogicalIntersect",
"qualifier", qualifier,
"outputs", outputs);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalIntersect(this, context);
}
@Override
public LogicalIntersect withChildren(List<Plan> children) {
return new LogicalIntersect(qualifier, outputs, children);
}
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalIntersect(qualifier, outputs, groupExpression,
Optional.of(getLogicalProperties()), children);
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new LogicalIntersect(qualifier, outputs,
Optional.empty(), logicalProperties, children);
}
@Override
public Plan withNewOutputs(List<NamedExpression> newOutputs) {
return new LogicalIntersect(qualifier, newOutputs,
Optional.empty(), Optional.empty(), children);
}
@Override
public Plan withNewChildren(List<Plan> children) {
return withChildren(children);
}
}

31
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
View File

@ -41,17 +41,21 @@ import java.util.Optional;
*/
public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation {
private final ImmutableList<NamedExpression> projects;
private final boolean buildUnionNode;
public LogicalOneRowRelation(List<NamedExpression> projects) {
this(projects, Optional.empty(), Optional.empty());
this(projects, true, Optional.empty(), Optional.empty());
}
private LogicalOneRowRelation(List<NamedExpression> projects, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
private LogicalOneRowRelation(List<NamedExpression> projects,
boolean buildUnionNode,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties) {
super(PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties);
Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(Slot.class)),
"OneRowRelation can not contains any slot");
this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null"));
this.buildUnionNode = buildUnionNode;
}
@Override
@ -71,12 +75,13 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalOneRowRelation(projects, groupExpression, Optional.of(logicalPropertiesSupplier.get()));
return new LogicalOneRowRelation(projects, buildUnionNode,
groupExpression, Optional.of(logicalPropertiesSupplier.get()));
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new LogicalOneRowRelation(projects, Optional.empty(), logicalProperties);
return new LogicalOneRowRelation(projects, buildUnionNode, Optional.empty(), logicalProperties);
}
@Override
@ -89,7 +94,8 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation
@Override
public String toString() {
return Utils.toSqlString("LogicalOneRowRelation",
"projects", projects
"projects", projects,
"buildUnionNode", buildUnionNode
);
}
@ -105,11 +111,20 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation
return false;
}
LogicalOneRowRelation that = (LogicalOneRowRelation) o;
return Objects.equals(projects, that.projects);
return Objects.equals(projects, that.projects)
&& Objects.equals(buildUnionNode, that.buildUnionNode);
}
@Override
public int hashCode() {
return Objects.hash(projects);
return Objects.hash(projects, buildUnionNode);
}
public boolean buildUnionNode() {
return buildUnionNode;
}
public Plan withBuildUnionNode(boolean buildUnionNode) {
return new LogicalOneRowRelation(projects, buildUnionNode, Optional.empty(), Optional.empty());
}
}

43
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
View File

@ -44,12 +44,20 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
private final ImmutableList<NamedExpression> projects;
private final ImmutableList<NamedExpression> excepts;
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, CHILD_TYPE child) {
this(projects, excepts, Optional.empty(), Optional.empty(), child);
}
// For project nodes under union, erasure cannot be configured, so add this flag.
private final boolean canEliminate;
public LogicalProject(List<NamedExpression> projects, CHILD_TYPE child) {
this(projects, Collections.emptyList(), child);
this(projects, Collections.emptyList(), true, child);
}
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, CHILD_TYPE child) {
this(projects, excepts, true, child);
}
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
boolean canEliminate, CHILD_TYPE child) {
this(projects, excepts, canEliminate, Optional.empty(), Optional.empty(), child);
}
/**
@ -57,12 +65,13 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
*
* @param projects project list
*/
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, boolean canEliminate,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
CHILD_TYPE child) {
super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties, child);
this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null"));
this.excepts = ImmutableList.copyOf(excepts);
this.canEliminate = canEliminate;
}
/**
@ -90,7 +99,8 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
public String toString() {
return Utils.toSqlString("LogicalProject",
"projects", projects,
"excepts", excepts
"excepts", excepts,
"canEliminate", canEliminate
);
}
@ -113,27 +123,38 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
return false;
}
LogicalProject that = (LogicalProject) o;
return projects.equals(that.projects);
return projects.equals(that.projects)
&& excepts.equals(that.excepts)
&& canEliminate == that.canEliminate;
}
@Override
public int hashCode() {
return Objects.hash(projects);
return Objects.hash(projects, canEliminate);
}
@Override
public LogicalUnary<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new LogicalProject<>(projects, excepts, children.get(0));
return new LogicalProject<>(projects, excepts, canEliminate, children.get(0));
}
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalProject<>(projects, excepts, groupExpression, Optional.of(getLogicalProperties()), child());
return new LogicalProject<>(projects, excepts, canEliminate,
groupExpression, Optional.of(getLogicalProperties()), child());
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new LogicalProject<>(projects, excepts, Optional.empty(), logicalProperties, child());
return new LogicalProject<>(projects, excepts, canEliminate, Optional.empty(), logicalProperties, child());
}
public boolean canEliminate() {
return canEliminate;
}
public Plan withEliminate(boolean isEliminate) {
return new LogicalProject<>(projects, excepts, isEliminate, child());
}
}

222
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java Normal file
View File

@ -0,0 +1,222 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
/**
* Logical SetOperation.
* The type can have any number of children.
* After parse, there will only be two children.
* But after rewriting rules such as merging of the same nodes and elimination of oneRowRelation,
* there will be multiple or no children.
*
* eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2;
*/
public abstract class LogicalSetOperation extends AbstractLogicalPlan implements SetOperation {
// eg value: qualifier:DISTINCT
protected final Qualifier qualifier;
// The newly created output column, used to display the output.
// eg value: outputs:[k1, k2]
protected final List<NamedExpression> outputs;
public LogicalSetOperation(PlanType planType, Qualifier qualifier, List<Plan> inputs) {
super(planType, inputs.toArray(new Plan[0]));
this.qualifier = qualifier;
this.outputs = ImmutableList.of();
}
public LogicalSetOperation(PlanType planType, Qualifier qualifier,
List<NamedExpression> outputs,
List<Plan> inputs) {
super(planType, inputs.toArray(new Plan[0]));
this.qualifier = qualifier;
this.outputs = ImmutableList.copyOf(outputs);
}
public LogicalSetOperation(PlanType planType, Qualifier qualifier, List<NamedExpression> outputs,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<Plan> inputs) {
super(planType, groupExpression, logicalProperties, inputs.toArray(new Plan[0]));
this.qualifier = qualifier;
this.outputs = ImmutableList.copyOf(outputs);
}
@Override
public List<Slot> computeOutput() {
return outputs.stream()
.map(NamedExpression::toSlot)
.collect(ImmutableList.toImmutableList());
}
public List<List<Expression>> collectCastExpressions() {
return castCommonDataTypeOutputs(resetNullableForLeftOutputs());
}
/**
* Generate new output for SetOperation.
*/
public List<NamedExpression> buildNewOutputs(List<Expression> leftCastExpressions) {
ImmutableList.Builder<NamedExpression> newOutputs = new Builder<>();
for (Expression expression : leftCastExpressions) {
if (expression instanceof Cast) {
newOutputs.add(new SlotReference(
((Cast) expression).child().toSql(), expression.getDataType(),
((Cast) expression).child().nullable()));
} else if (expression instanceof Slot) {
newOutputs.add(new SlotReference(
expression.toSql(), expression.getDataType(), expression.nullable()));
}
}
return newOutputs.build();
}
// If the right child is nullable, need to ensure that the left child is also nullable
private List<Slot> resetNullableForLeftOutputs() {
Preconditions.checkState(children.size() == 2);
List<Slot> resetNullableForLeftOutputs = new ArrayList<>();
for (int i = 0; i < child(1).getOutput().size(); ++i) {
if (child(1).getOutput().get(i).nullable() && !child(0).getOutput().get(i).nullable()) {
resetNullableForLeftOutputs.add(child(0).getOutput().get(i).withNullable(true));
} else {
resetNullableForLeftOutputs.add(child(0).getOutput().get(i));
}
}
return ImmutableList.copyOf(resetNullableForLeftOutputs);
}
private List<List<Expression>> castCommonDataTypeOutputs(List<Slot> resetNullableForLeftOutputs) {
List<Expression> newLeftOutputs = new ArrayList<>();
List<Expression> newRightOutpus = new ArrayList<>();
// Ensure that the output types of the left and right children are consistent and expand upward.
for (int i = 0; i < resetNullableForLeftOutputs.size(); ++i) {
boolean hasPushed = false;
Slot left = resetNullableForLeftOutputs.get(i);
Slot right = child(1).getOutput().get(i);
if (TypeCoercionUtils.canHandleTypeCoercion(left.getDataType(), right.getDataType())) {
Optional<DataType> tightestCommonType =
TypeCoercionUtils.findTightestCommonType(left.getDataType(), right.getDataType());
if (tightestCommonType.isPresent()) {
Expression newLeft = TypeCoercionUtils.castIfNotSameType(left, tightestCommonType.get());
Expression newRight = TypeCoercionUtils.castIfNotSameType(right, tightestCommonType.get());
newLeftOutputs.add(newLeft);
newRightOutpus.add(newRight);
hasPushed = true;
}
}
if (!hasPushed) {
newLeftOutputs.add(left);
newRightOutpus.add(right);
}
}
List<List<Expression>> resultExpressions = new ArrayList<>();
resultExpressions.add(newLeftOutputs);
resultExpressions.add(newRightOutpus);
return ImmutableList.copyOf(resultExpressions);
}
@Override
public String toString() {
return Utils.toSqlString("LogicalSetOperation",
"qualifier", qualifier,
"outputs", outputs);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LogicalSetOperation that = (LogicalSetOperation) o;
return Objects.equals(qualifier, that.qualifier)
&& Objects.equals(outputs, that.outputs);
}
@Override
public int hashCode() {
return Objects.hash(qualifier, outputs);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalSetOperation(this, context);
}
@Override
public List<? extends Expression> getExpressions() {
return ImmutableList.of();
}
@Override
public Qualifier getQualifier() {
return qualifier;
}
@Override
public List<Slot> getFirstOutput() {
return child(0).getOutput();
}
@Override
public List<Slot> getChildOutput(int i) {
return child(i).getOutput();
}
@Override
public List<NamedExpression> getOutputs() {
return outputs;
}
public abstract Plan withNewOutputs(List<NamedExpression> newOutputs);
@Override
public int getArity() {
return children.size();
}
public abstract Plan withNewChildren(List<Plan> children);
}

145
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java Normal file
View File

@ -0,0 +1,145 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
/**
* Logical Union.
*/
public class LogicalUnion extends LogicalSetOperation {
// When the union is DISTINCT, an additional LogicalAggregation needs to be created,
// so add this flag to judge whether agg has been created to avoid repeated creation
private final boolean hasBuildAgg;
// When there is an agg on the union and there is a filter on the agg,
// it is necessary to keep the filter on the agg and push the filter down to each child of the union.
private final boolean hasPushedFilter;
public LogicalUnion(Qualifier qualifier, List<Plan> inputs) {
super(PlanType.LOGICAL_UNION, qualifier, inputs);
this.hasBuildAgg = false;
this.hasPushedFilter = false;
}
public LogicalUnion(Qualifier qualifier, List<NamedExpression> outputs,
boolean hasBuildAgg, boolean hasPushedFilter,
List<Plan> inputs) {
super(PlanType.LOGICAL_UNION, qualifier, outputs, inputs);
this.hasBuildAgg = hasBuildAgg;
this.hasPushedFilter = hasPushedFilter;
}
public LogicalUnion(Qualifier qualifier, List<NamedExpression> outputs,
boolean hasBuildAgg, boolean hasPushedFilter,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<Plan> inputs) {
super(PlanType.LOGICAL_UNION, qualifier, outputs, groupExpression, logicalProperties, inputs);
this.hasBuildAgg = hasBuildAgg;
this.hasPushedFilter = hasPushedFilter;
}
@Override
public String toString() {
return Utils.toSqlString("LogicalUnion",
"qualifier", qualifier,
"outputs", outputs,
"hasBuildAgg", hasBuildAgg,
"hasPushedFilter", hasPushedFilter);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LogicalUnion that = (LogicalUnion) o;
return super.equals(that)
&& hasBuildAgg == that.hasBuildAgg
&& hasPushedFilter == that.hasPushedFilter;
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), hasBuildAgg, hasPushedFilter);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalUnion(this, context);
}
@Override
public LogicalUnion withChildren(List<Plan> children) {
return new LogicalUnion(qualifier, outputs, hasBuildAgg, hasPushedFilter, children);
}
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalUnion(qualifier, outputs, hasBuildAgg, hasPushedFilter, groupExpression,
Optional.of(getLogicalProperties()), children);
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new LogicalUnion(qualifier, outputs, hasBuildAgg, hasPushedFilter,
Optional.empty(), logicalProperties, children);
}
@Override
public Plan withNewOutputs(List<NamedExpression> newOutputs) {
return new LogicalUnion(qualifier, newOutputs, hasBuildAgg, hasPushedFilter,
Optional.empty(), Optional.empty(), children);
}
public boolean hasBuildAgg() {
return hasBuildAgg;
}
public Plan withHasBuildAgg() {
return new LogicalUnion(qualifier, outputs, true, hasPushedFilter,
Optional.empty(), Optional.empty(), children);
}
public boolean hasPushedFilter() {
return hasPushedFilter;
}
public Plan withHasPushedFilter() {
return new LogicalUnion(qualifier, outputs, hasBuildAgg, true,
Optional.empty(), Optional.empty(), children);
}
@Override
public Plan withNewChildren(List<Plan> children) {
return withChildren(children);
}
}

93
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java Normal file
View File

@ -0,0 +1,93 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.StatsDeriveResult;
import java.util.List;
import java.util.Optional;
/**
* Physical Except.
*/
public class PhysicalExcept extends PhysicalSetOperation {
public PhysicalExcept(Qualifier qualifier,
LogicalProperties logicalProperties,
List<Plan> inputs) {
super(PlanType.PHYSICAL_EXCEPT, qualifier, logicalProperties, inputs);
}
public PhysicalExcept(Qualifier qualifier,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties,
List<Plan> inputs) {
super(PlanType.PHYSICAL_EXCEPT, qualifier, groupExpression, logicalProperties, inputs);
}
public PhysicalExcept(Qualifier qualifier, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties,
PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult,
List<Plan> inputs) {
super(PlanType.PHYSICAL_EXCEPT, qualifier,
groupExpression, logicalProperties, physicalProperties, statsDeriveResult, inputs);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitPhysicalExcept(this, context);
}
@Override
public String toString() {
return Utils.toSqlString("PhysicalExcept",
"qualifier", qualifier,
"stats", statsDeriveResult);
}
@Override
public PhysicalExcept withChildren(List<Plan> children) {
return new PhysicalExcept(qualifier, getLogicalProperties(), children);
}
@Override
public PhysicalExcept withGroupExpression(
Optional<GroupExpression> groupExpression) {
return new PhysicalExcept(qualifier, groupExpression, getLogicalProperties(), children);
}
@Override
public PhysicalExcept withLogicalProperties(
Optional<LogicalProperties> logicalProperties) {
return new PhysicalExcept(qualifier, Optional.empty(), logicalProperties.get(), children);
}
@Override
public PhysicalExcept withPhysicalPropertiesAndStats(
PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) {
return new PhysicalExcept(qualifier, Optional.empty(),
getLogicalProperties(), physicalProperties, statsDeriveResult, children);
}
}

93
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java Normal file
View File

@ -0,0 +1,93 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.StatsDeriveResult;
import java.util.List;
import java.util.Optional;
/**
* Physical Intersect.
*/
public class PhysicalIntersect extends PhysicalSetOperation {
public PhysicalIntersect(Qualifier qualifier,
LogicalProperties logicalProperties,
List<Plan> inputs) {
super(PlanType.PHYSICAL_INTERSECT, qualifier, logicalProperties, inputs);
}
public PhysicalIntersect(Qualifier qualifier,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties,
List<Plan> inputs) {
super(PlanType.PHYSICAL_INTERSECT, qualifier, groupExpression, logicalProperties, inputs);
}
public PhysicalIntersect(Qualifier qualifier,
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult,
List<Plan> inputs) {
super(PlanType.PHYSICAL_INTERSECT, qualifier,
groupExpression, logicalProperties, physicalProperties, statsDeriveResult, inputs);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitPhysicalIntersect(this, context);
}
@Override
public String toString() {
return Utils.toSqlString("PhysicalIntersect",
"qualifier", qualifier,
"stats", statsDeriveResult);
}
@Override
public PhysicalIntersect withChildren(List<Plan> children) {
return new PhysicalIntersect(qualifier, getLogicalProperties(), children);
}
@Override
public PhysicalIntersect withGroupExpression(
Optional<GroupExpression> groupExpression) {
return new PhysicalIntersect(qualifier, groupExpression, getLogicalProperties(), children);
}
@Override
public PhysicalIntersect withLogicalProperties(
Optional<LogicalProperties> logicalProperties) {
return new PhysicalIntersect(qualifier, Optional.empty(), logicalProperties.get(), children);
}
@Override
public PhysicalIntersect withPhysicalPropertiesAndStats(
PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) {
return new PhysicalIntersect(qualifier,
Optional.empty(), getLogicalProperties(), physicalProperties, statsDeriveResult, children);
}
}

29
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java
View File

@ -42,12 +42,16 @@ import java.util.Optional;
*/
public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelation {
private final ImmutableList<NamedExpression> projects;
private final boolean buildUnionNode;
public PhysicalOneRowRelation(List<NamedExpression> projects, LogicalProperties logicalProperties) {
this(projects, Optional.empty(), logicalProperties, null, null);
public PhysicalOneRowRelation(List<NamedExpression> projects, boolean buildUnionNode,
LogicalProperties logicalProperties) {
this(projects, buildUnionNode, Optional.empty(), logicalProperties, null, null);
}
private PhysicalOneRowRelation(List<NamedExpression> projects, Optional<GroupExpression> groupExpression,
private PhysicalOneRowRelation(List<NamedExpression> projects,
boolean buildUnionNode,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties,
StatsDeriveResult statsDeriveResult) {
super(PlanType.PHYSICAL_ONE_ROW_RELATION, groupExpression, logicalProperties, physicalProperties,
@ -55,6 +59,7 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelati
Preconditions.checkArgument(projects.stream().allMatch(Expression::isConstant),
"OneRowRelation must consist of some constant expression");
this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null"));
this.buildUnionNode = buildUnionNode;
}
@Override
@ -74,20 +79,21 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelati
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new PhysicalOneRowRelation(projects, groupExpression,
return new PhysicalOneRowRelation(projects, buildUnionNode, groupExpression,
logicalPropertiesSupplier.get(), physicalProperties, statsDeriveResult);
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new PhysicalOneRowRelation(projects, Optional.empty(),
return new PhysicalOneRowRelation(projects, buildUnionNode, Optional.empty(),
logicalProperties.get(), physicalProperties, statsDeriveResult);
}
@Override
public String toString() {
return Utils.toSqlString("PhysicalOneRowRelation",
"expressions", projects
"expressions", projects,
"buildUnionNode", buildUnionNode
);
}
@ -100,18 +106,23 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelati
return false;
}
PhysicalOneRowRelation that = (PhysicalOneRowRelation) o;
return Objects.equals(projects, that.projects);
return Objects.equals(projects, that.projects)
&& buildUnionNode == that.buildUnionNode;
}
@Override
public int hashCode() {
return Objects.hash(projects);
return Objects.hash(projects, buildUnionNode);
}
@Override
public PhysicalOneRowRelation withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
StatsDeriveResult statsDeriveResult) {
return new PhysicalOneRowRelation(projects, Optional.empty(),
return new PhysicalOneRowRelation(projects, buildUnionNode, Optional.empty(),
logicalPropertiesSupplier.get(), physicalProperties, statsDeriveResult);
}
public boolean notBuildUnionNode() {
return !buildUnionNode;
}
}

132
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java Normal file
View File

@ -0,0 +1,132 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.StatsDeriveResult;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
/**
* Physical SetOperation.
*/
public abstract class PhysicalSetOperation extends AbstractPhysicalPlan implements SetOperation {
protected final Qualifier qualifier;
public PhysicalSetOperation(PlanType planType,
Qualifier qualifier,
LogicalProperties logicalProperties,
List<Plan> inputs) {
super(planType, Optional.empty(), logicalProperties, inputs.toArray(new Plan[0]));
this.qualifier = qualifier;
}
public PhysicalSetOperation(PlanType planType,
Qualifier qualifier,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties,
List<Plan> inputs) {
super(planType, groupExpression, logicalProperties, inputs.toArray(new Plan[0]));
this.qualifier = qualifier;
}
public PhysicalSetOperation(PlanType planType,
Qualifier qualifier,
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult, List<Plan> inputs) {
super(planType, groupExpression, logicalProperties,
physicalProperties, statsDeriveResult, inputs.toArray(new Plan[0]));
this.qualifier = qualifier;
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitPhysicalSetOperation(this, context);
}
@Override
public String toString() {
return Utils.toSqlString("PhysicalSetOperation",
"qualifier", qualifier,
"stats", statsDeriveResult);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
PhysicalSetOperation that = (PhysicalSetOperation) o;
return Objects.equals(qualifier, that.qualifier);
}
@Override
public int hashCode() {
return Objects.hash(qualifier);
}
@Override
public List<? extends Expression> getExpressions() {
return ImmutableList.of();
}
@Override
public Qualifier getQualifier() {
return qualifier;
}
@Override
public List<Slot> getFirstOutput() {
return child(0).getOutput();
}
@Override
public List<Slot> getChildOutput(int i) {
return child(i).getOutput();
}
@Override
public List<NamedExpression> getOutputs() {
return getOutput().stream()
.map(NamedExpression.class::cast)
.collect(ImmutableList.toImmutableList());
}
@Override
public int getArity() {
return children.size();
}
}

92
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java Normal file
View File

@ -0,0 +1,92 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.StatsDeriveResult;
import java.util.List;
import java.util.Optional;
/**
* Physical Union.
*/
public class PhysicalUnion extends PhysicalSetOperation {
public PhysicalUnion(Qualifier qualifier,
LogicalProperties logicalProperties,
List<Plan> inputs) {
super(PlanType.PHYSICAL_UNION, qualifier, logicalProperties, inputs);
}
public PhysicalUnion(Qualifier qualifier,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties,
List<Plan> inputs) {
super(PlanType.PHYSICAL_UNION, qualifier, groupExpression, logicalProperties, inputs);
}
public PhysicalUnion(Qualifier qualifier,
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties,
PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult, List<Plan> inputs) {
super(PlanType.PHYSICAL_UNION, qualifier,
groupExpression, logicalProperties, physicalProperties, statsDeriveResult, inputs);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitPhysicalUnion(this, context);
}
@Override
public String toString() {
return Utils.toSqlString("PhysicalUnion",
"qualifier", qualifier,
"stats", statsDeriveResult);
}
@Override
public PhysicalUnion withChildren(List<Plan> children) {
return new PhysicalUnion(qualifier, getLogicalProperties(), children);
}
@Override
public PhysicalUnion withGroupExpression(
Optional<GroupExpression> groupExpression) {
return new PhysicalUnion(qualifier, groupExpression, getLogicalProperties(), children);
}
@Override
public PhysicalUnion withLogicalProperties(
Optional<LogicalProperties> logicalProperties) {
return new PhysicalUnion(qualifier, Optional.empty(), logicalProperties.get(), children);
}
@Override
public PhysicalUnion withPhysicalPropertiesAndStats(
PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) {
return new PhysicalUnion(qualifier, Optional.empty(),
getLogicalProperties(), physicalProperties, statsDeriveResult, children);
}
}

41
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
View File

@ -31,8 +31,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
@ -41,18 +43,22 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLocalQuickSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
@ -62,9 +68,11 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
/**
* Base class for the processing of logical and physical plan.
@ -192,6 +200,23 @@ public abstract class PlanVisitor<R, C> {
return visit(having, context);
}
public R visitLogicalSetOperation(
LogicalSetOperation logicalSetOperation, C context) {
return visit(logicalSetOperation, context);
}
public R visitLogicalUnion(LogicalUnion union, C context) {
return visitLogicalSetOperation(union, context);
}
public R visitLogicalExcept(LogicalExcept except, C context) {
return visitLogicalSetOperation(except, context);
}
public R visitLogicalIntersect(LogicalIntersect intersect, C context) {
return visitLogicalSetOperation(intersect, context);
}
// *******************************
// Physical plans
// *******************************
@ -265,6 +290,22 @@ public abstract class PlanVisitor<R, C> {
return visit(filter, context);
}
public R visitPhysicalSetOperation(PhysicalSetOperation setOperation, C context) {
return visit(setOperation, context);
}
public R visitPhysicalUnion(PhysicalUnion union, C context) {
return visitPhysicalSetOperation(union, context);
}
public R visitPhysicalExcept(PhysicalExcept except, C context) {
return visitPhysicalSetOperation(except, context);
}
public R visitPhysicalIntersect(PhysicalIntersect intersect, C context) {
return visitPhysicalSetOperation(intersect, context);
}
// *******************************
// Physical enforcer
// *******************************

2
fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java
View File

@ -26,7 +26,7 @@ import org.apache.doris.thrift.TPlanNodeType;
import java.util.List;
public class ExceptNode extends SetOperationNode {
protected ExceptNode(PlanNodeId id, TupleId tupleId) {
public ExceptNode(PlanNodeId id, TupleId tupleId) {
super(id, tupleId, "EXCEPT");
}

2
fe/fe-core/src/main/java/org/apache/doris/planner/IntersectNode.java
View File

@ -25,7 +25,7 @@ import org.apache.doris.thrift.TPlanNodeType;
import java.util.List;
public class IntersectNode extends SetOperationNode {
protected IntersectNode(PlanNodeId id, TupleId tupleId) {
public IntersectNode(PlanNodeId id, TupleId tupleId) {
super(id, tupleId, "INTERSECT");
}

25
fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
View File

@ -118,6 +118,10 @@ public abstract class SetOperationNode extends PlanNode {
constExprLists.add(exprs);
}
public void addResultExprLists(List<Expr> exprs) {
resultExprLists.add(exprs);
}
/**
* Returns true if this UnionNode has only constant exprs.
*/
@ -446,7 +450,10 @@ public abstract class SetOperationNode extends PlanNode {
return numInstances;
}
public void finalizeForNereids(TupleDescriptor tupleDescriptor, List<SlotDescriptor> constExprSlots) {
/**
* just for Nereids.
*/
public void finalizeForNereids(List<SlotDescriptor> constExprSlots, List<SlotDescriptor> resultExprSlots) {
materializedConstExprLists.clear();
for (List<Expr> exprList : constExprLists) {
Preconditions.checkState(exprList.size() == constExprSlots.size());
@ -458,5 +465,21 @@ public abstract class SetOperationNode extends PlanNode {
}
materializedConstExprLists.add(newExprList);
}
materializedResultExprLists.clear();
Preconditions.checkState(resultExprLists.size() == children.size());
for (int i = 0; i < resultExprLists.size(); ++i) {
List<Expr> exprList = resultExprLists.get(i);
List<Expr> newExprList = Lists.newArrayList();
Preconditions.checkState(exprList.size() == resultExprSlots.size());
for (int j = 0; j < exprList.size(); ++j) {
if (resultExprSlots.get(j).isMaterialized()) {
newExprList.add(exprList.get(j));
}
}
materializedResultExprLists.add(newExprList);
}
Preconditions.checkState(
materializedResultExprLists.size() == getChildren().size());
}
}

9
fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
View File

@ -131,6 +131,15 @@ public class StatsDeriveResult {
for (Entry<Id, ColumnStatistic> entry : slotIdToColumnStats.entrySet()) {
statsDeriveResult.addColumnStats(entry.getKey(), entry.getValue().updateByLimit(limit, rowCount));
}
// When the table is first created, rowCount is empty.
// This leads to NPE if there is SetOperation outside the limit.
// Therefore, when rowCount is empty, slotIdToColumnStats is also imported,
// but the possible problem is that the first query statistics are not derived accurately.
if (statsDeriveResult.slotIdToColumnStats.isEmpty()) {
for (Entry<Id, ColumnStatistic> entry : slotIdToColumnStats.entrySet()) {
statsDeriveResult.addColumnStats(entry.getKey(), entry.getValue());
}
}
return statsDeriveResult;
}

13
fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
View File

@ -242,4 +242,17 @@ public class NereidsParserTest extends ParserTestBase {
.sum();
Assertions.assertEquals(doubleCount, 1);
}
@Test
public void parseSetOperation() {
String union = "select * from t1 union select * from t2 union all select * from t3";
NereidsParser nereidsParser = new NereidsParser();
LogicalPlan logicalPlan = nereidsParser.parseSingle(union);
System.out.println(logicalPlan.treeString());
String union1 = "select * from t1 union (select * from t2 union all select * from t3)";
NereidsParser nereidsParser1 = new NereidsParser();
LogicalPlan logicalPlan1 = nereidsParser1.parseSingle(union1);
System.out.println(logicalPlan1.treeString());
}
}

2
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java
View File

@ -362,7 +362,7 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements P
@Test
public void testCountDistinctValueColumn() {
singleTableTest("select k1, count(distinct v1) from from t group by k1", scan -> {
singleTableTest("select k1, count(distinct v1) from t group by k1", scan -> {
Assertions.assertFalse(scan.isPreAggregation());
Assertions.assertEquals("Count distinct is only valid for key columns, but meet count(DISTINCT v1).",
scan.getReasonOfPreAggregation());

6
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java
View File

@ -68,7 +68,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService {
.build();
CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject);
List<Rule> rules = Lists.newArrayList(new EliminateUnnecessaryProject().build());
List<Rule> rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules());
cascadesContext.topDownRewrite(rules);
Plan actual = cascadesContext.getMemo().copyOut();
@ -82,7 +82,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService {
.build();
CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject);
List<Rule> rules = Lists.newArrayList(new EliminateUnnecessaryProject().build());
List<Rule> rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules());
cascadesContext.topDownRewrite(rules);
Plan actual = cascadesContext.getMemo().copyOut();
@ -96,7 +96,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService {
.build();
CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject);
List<Rule> rules = Lists.newArrayList(new EliminateUnnecessaryProject().build());
List<Rule> rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules());
cascadesContext.topDownRewrite(rules);
Plan actual = cascadesContext.getMemo().copyOut();

2
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
View File

@ -89,7 +89,7 @@ public class PlanToStringTest {
LogicalProject<Plan> plan = new LogicalProject<>(ImmutableList.of(
new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList())), child);
Assertions.assertTrue(plan.toString().matches("LogicalProject \\( projects=\\[a#\\d+], excepts=\\[] \\)"));
Assertions.assertTrue(plan.toString().matches("LogicalProject \\( projects=\\[a#\\d+], excepts=\\[], canEliminate=true \\)"));
}
@Test

113
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/SetOperationTest.java Normal file
View File

@ -0,0 +1,113 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.jupiter.api.Test;
public class SetOperationTest extends TestWithFeService {
@Override
protected void runBeforeAll() throws Exception {
createDatabase("test");
useDatabase("test");
createTable("CREATE TABLE `t1` (\n"
+ " `k1` bigint(20) NULL,\n"
+ " `k2` bigint(20) NULL,\n"
+ " `k3` bigint(20) not NULL,\n"
+ " `k4` bigint(20) not NULL,\n"
+ " `k5` bigint(20) NULL\n"
+ ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`k1`)\n"
+ "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 1\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"disable_auto_compaction\" = \"false\"\n"
+ ");");
createTable("CREATE TABLE `t2` (\n"
+ " `k1` bigint(20) NULL,\n"
+ " `k2` varchar(20) NULL,\n"
+ " `k3` bigint(20) not NULL,\n"
+ " `k4` bigint(20) not NULL,\n"
+ " `k5` bigint(20) NULL\n"
+ ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`k1`)\n"
+ "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 1\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"disable_auto_compaction\" = \"false\"\n"
+ ");");
createTable("CREATE TABLE `t3` (\n"
+ " `k1` varchar(20) NULL,\n"
+ " `k2` varchar(20) NULL,\n"
+ " `k3` bigint(20) not NULL,\n"
+ " `k4` bigint(20) not NULL,\n"
+ " `k5` bigint(20) NULL\n"
+ ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`k1`)\n"
+ "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 1\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"disable_auto_compaction\" = \"false\"\n"
+ ");");
}
// union
@Test
public void testUnion1() {
PlanChecker.from(connectContext)
.checkPlannerResult("select k1, k2 from t1 union select k1, k2 from t3;");
}
@Test
public void testUnion2() {
PlanChecker.from(connectContext)
.checkPlannerResult("select k1, k2 from t1 union all select k1, k2 from t3;");
}
@Test
public void testUnion3() {
PlanChecker.from(connectContext)
.checkPlannerResult("select k1 from t1 union select k1 from t3 union select 1;");
}
@Test
public void testUnion4() {
PlanChecker.from(connectContext)
.checkPlannerResult("select 1 a, 2 b union all select 3, 4 union all select 10 e, 20 f;");
}
@Test
public void testUnion5() {
PlanChecker.from(connectContext)
.checkPlannerResult("select 1, 2 union all select 1, 2 union all select 10 e, 20 f;");
}
}

441
regression-test/data/nereids_syntax_p0/set_operation.out Normal file
View File

@ -0,0 +1,441 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select1 --
1 0
1 1
1 2
1 3
2 0
2 1
2 2
2 4
2 6
3 0
3 2
3 3
3 6
3 9
4 0
4 3
-- !select2 --
2 0
2 1
2 2
2 3
3 0
3 2
3 4
3 6
4 0
4 3
4 6
4 9
-- !select3 --
1 0 0
1 1 1
1 1 2
1 1 3
2 2 0
2 2 2
2 2 4
2 2 6
3 0 0
3 3 3
3 3 6
3 3 9
a b 1
a c 1
a d 1
b b 1
b c 2
b d 2
c b 2
c c 2
c d 3
d b 3
d c 3
d d 3
-- !select4 --
1 0 0
1 1 1
1 1 2
1 1 3
2 2 0
2 2 2
2 2 4
2 2 6
3 0 0
3 3 3
3 3 6
3 3 9
a b 1
a c 1
a d 1
b b 1
b c 2
b d 2
c b 2
c c 2
c d 3
d b 3
d c 3
d d 3
-- !select5 --
1 0
1 1
1 2
1 3
2 0
2 0
2 1
2 1
2 1
2 2
2 4
2 6
3 0
3 2
3 2
3 2
3 2
3 3
3 6
3 9
4 0
4 3
4 3
4 3
-- !select6 --
2 0
2 0
2 1
2 1
2 1
2 1
2 2
2 3
3 0
3 2
3 2
3 2
3 2
3 2
3 4
3 6
4 0
4 0
4 3
4 3
4 3
4 3
4 6
4 9
-- !select7 --
1 0 0
1 1 1
1 1 2
1 1 3
2 2 0
2 2 2
2 2 4
2 2 6
3 0 0
3 3 3
3 3 6
3 3 9
a b 1
a c 1
a d 1
b b 1
b c 2
b d 2
c b 2
c c 2
c d 3
d b 3
d c 3
d d 3
-- !select8 --
1 0 0
1 1 1
1 1 2
1 1 3
2 2 0
2 2 2
2 2 4
2 2 6
3 0 0
3 3 3
3 3 6
3 3 9
a b 1
a c 1
a d 1
b b 1
b c 2
b d 2
c b 2
c c 2
c d 3
d b 3
d c 3
d d 3
-- !select9 --
2 1
3 2
4 0
4 3
-- !select10 --
2 2
2 3
3 0
3 4
3 6
4 6
4 9
-- !select11 --
a b 1
a c 1
a d 1
b b 1
b c 2
b d 2
c b 2
c c 2
c d 3
d b 3
d c 3
d d 3
-- !select12 --
1 0 0
1 1 1
1 1 2
1 1 3
2 2 0
2 2 2
2 2 4
2 2 6
3 0 0
3 3 3
3 3 6
3 3 9
-- !select13 --
2 0
-- !select14 --
2 0
2 1
3 2
4 0
4 3
-- !select15 --
-- !select16 --
-- !select17 --
1 0
1 2
1 3
1 a
1 b
2 0
2 4
2 6
2 b
2 c
3 0
3 6
3 9
3 c
3 d
-- !select18 --
0 1
0 3
1 0
1 1
1 1
1 2
1 3
1 a
1 b
2 0
2 2
2 2
2 4
2 6
2 b
2 c
3 0
3 3
3 3
3 6
3 9
3 c
3 d
-- !select19 --
0 1
0 3
1 0
1 1
1 2
1 3
1 a
1 b
2 0
2 2
2 4
2 6
2 b
2 c
3 0
3 3
3 6
3 9
3 c
3 d
-- !select20 --
1 0
1 1
1 1
1 1
1 a
1 a
1 a
1 b
2 2
2 2
2 2
2 2
2 b
2 b
2 c
2 c
3 0
3 3
3 3
3 3
3 c
3 d
3 d
3 d
-- !select21 --
1 0
1 1
1 a
1 b
2 2
2 b
2 c
3 0
3 3
3 c
3 d
-- !select24 --
1 2
10 20
3 4
-- !select25 --
1 3
2 3
2 8
3 9
-- !select26 --
1 3
2 3
2 8
3 9
-- !select27 --
1 a \N 10.0
1 a \N 10.0
2 b \N 20.0
-- !select28 --
1 0.0 \N 4
1 1.0 1 1
1 1.0 1 3
1 1.0 2 3
1 1.0 3 4
1 2.0 1 2
1 3.0 1 3
10 10.0 hello world
2 0.0 2 0
2 2.0 2 2
2 4.0 2 4
2 6.0 2 6
20 20.0 wangjuoo4 beautiful
3 3.0 3 3
3 6.0 3 6
3 9.0 3 9
-- !select29 --
1 0.0 \N 4
1 1.0 1 1
1 1.0 1 3
1 1.0 2 3
1 1.0 3 4
1 2.0 1 2
1 3.0 1 3
10 10.0 hello world
2 0.0 2 0
2 2.0 2 2
2 4.0 2 4
2 6.0 2 6
20 20.0 wangjuoo4 beautiful
3 3.0 3 3
3 6.0 3 6
3 9.0 3 9
-- !union30 --
1.00 2.0
1.01 2.0
0.00 0.0
-- !union31 --
1 2
hell0
-- !union32 --
1.0 2.0
-- !union33 --
1.0 2.0
-- !union34 --
1.0 2.0
1.0 2.0
1.0 2.0
-- !union35 --
1.0 2.0
1.0 2.0
-- !union36 --
1.0 2.0
-- !union38 --
2016-07-01
2016-07-02
-- !union36 --
1 2

216
regression-test/suites/nereids_syntax_p0/set_operation.groovy Normal file
View File

@ -0,0 +1,216 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_nereids_set_operation") {
sql "SET enable_nereids_planner=true"
sql "SET enable_vectorized_engine=true"
sql "DROP TABLE IF EXISTS setOperationTable"
sql "DROP TABLE IF EXISTS setOperationTableNotNullable"
sql """
CREATE TABLE `setOperationTable` (
`k1` bigint(20) NULL,
`k2` bigint(20) NULL,
`k3` bigint(20) NULL,
`k4` bigint(20) not null,
`k5` varchar(10),
`k6` varchar(10)
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
DISTRIBUTED BY HASH(`k2`) BUCKETS 1
PROPERTIES ('replication_num' = '1')
"""
sql """
CREATE TABLE `setOperationTableNotNullable` (
`k1` bigint(20) NOT NULL,
`k2` bigint(20) NOT NULL,
`k3` bigint(20) NOT NULL
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k2`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
);
"""
sql """
INSERT INTO setOperationTable VALUES
(1, 1, 1, 3, 'a', 'b'),
(1, 1, 2, 3, 'a', 'c'),
(1, 1, 3, 4, 'a' , 'd'),
(1, 0, null, 4, 'b' , 'b'),
(2, 2, 2, 5, 'b', 'c'),
(2, 2, 4, 5, 'b' , 'd'),
(2, 2, 6, 4, 'c', 'b'),
(2, 2, null, 4, 'c', 'c'),
(3, 3, 3, 3, 'c', 'd'),
(3, 3, 6, 3, 'd', 'b'),
(3, 3, 9, 4, 'd', 'c'),
(3, 0, null, 5, 'd', 'd')
"""
sql """
insert into setOperationTableNotNullable values
(1, 0, 0),
(1, 1, 3),
(1, 1, 2),
(1, 1, 1),
(2, 2, 0),
(2, 2, 6),
(2, 2, 4),
(2, 2, 2),
(3, 0, 0),
(3, 3, 9),
(3, 3, 6),
(3, 3, 3);
"""
sql "SET enable_fallback_to_original_planner=false"
// union
order_qt_select1 "select k1+1, k2 from setOperationTable union select k1, k3 from setOperationTableNotNullable;";
order_qt_select2 "select k1+1, k3 from setOperationTableNotNullable union select k1+1, k2 from setOperationTable;";
order_qt_select3 "select k5, k6, k1 from setOperationTable union select k1, k2, k3 from setOperationTableNotNullable";
order_qt_select4 "select k1, k2, k3 from setOperationTableNotNullable union select k5, k6, k1 from setOperationTable";
order_qt_select5 "select k1+1, k2 from setOperationTable union all select k1, k3 from setOperationTableNotNullable;";
order_qt_select6 "select k1+1, k3 from setOperationTableNotNullable union all select k1+1, k2 from setOperationTable;";
order_qt_select7 "select k5, k6, k1 from setOperationTable union all select k1, k2, k3 from setOperationTableNotNullable";
order_qt_select8 "select k1, k2, k3 from setOperationTableNotNullable union all select k5, k6, k1 from setOperationTable";
// except
order_qt_select9 "select k1+1, k2 from setOperationTable except select k1, k3 from setOperationTableNotNullable;";
order_qt_select10 "select k1+1, k3 from setOperationTableNotNullable except select k1+1, k2 from setOperationTable;";
order_qt_select11 "select k5, k6, k1 from setOperationTable except select k1, k2, k3 from setOperationTableNotNullable";
order_qt_select12 "select k1, k2, k3 from setOperationTableNotNullable except select k5, k6, k1 from setOperationTable";
//intersect
order_qt_select13 "select k1+1, k2 from setOperationTable intersect select k1, k3 from setOperationTableNotNullable;";
order_qt_select14 "select k1+1, k3 from setOperationTableNotNullable intersect select k1+1, k2 from setOperationTable;";
order_qt_select15 "select k5, k6, k1 from setOperationTable intersect select k1, k2, k3 from setOperationTableNotNullable";
order_qt_select16 "select k1, k2, k3 from setOperationTableNotNullable intersect select k5, k6, k1 from setOperationTable";
// mix
order_qt_select17 """
select k1, k3 from setOperationTableNotNullable union all
select k1, k5 from setOperationTable except
select k2, k1 from setOperationTableNotNullable
"""
order_qt_select18 """
select k1, k3 from setOperationTableNotNullable union all
(select k1, k5 from setOperationTable union
select k2, k1 from setOperationTableNotNullable)
"""
order_qt_select19 """
(select k1, k3 from setOperationTableNotNullable union all
select k1, k5 from setOperationTable) union
select k2, k1 from setOperationTableNotNullable
"""
order_qt_select20 """
select * from (select k1, k2 from setOperationTableNotNullable union all select k1, k5 from setOperationTable) t;
"""
order_qt_select21 """
select * from (select k1, k2 from setOperationTableNotNullable union select k1, k5 from setOperationTable) t;
"""
order_qt_select24 """select * from (select 1 a, 2 b
union all select 3, 4
union all select 10, 20) t where a<b order by a, b"""
order_qt_select25 """
select k1, sum(k2) from setOperationTableNotNullable group by k1
union distinct (select 2,3)
"""
order_qt_select26 """
(select 2,3)
union distinct
select k1, sum(k2) from setOperationTableNotNullable group by k1
union distinct (select 2,3)
"""
order_qt_select27 """
(select 1, 'a', NULL, 10.0)
union all (select 2, 'b', NULL, 20.0)
union all (select 1, 'a', NULL, 10.0)
"""
order_qt_select28 """
(select 10, 10.0, 'hello', 'world') union all
(select k1, k2, k3, k4 from setOperationTable where k1=1) union all
(select 20, 20.0, 'wangjuoo4', 'beautiful') union all
(select k2, k3, k1, k3 from setOperationTableNotNullable where k2>0)
"""
order_qt_select29 """
select * from (
(select 10, 10.0, 'hello', 'world') union all
(select k1, k2, k3, k4 from setOperationTable where k1=1) union all
(select 20, 20.0, 'wangjuoo4', 'beautiful') union all
(select k2, k3, k1, k3 from setOperationTableNotNullable where k2>0)) t
"""
// test_union_basic
qt_union30 """select 1, 2 union select 1.01, 2.0 union (select 0.0001, 0.0000001)"""
qt_union31 """select 1, 2 union (select "hell0", "")"""
qt_union32 """select 1, 2 union select 1.0, 2.0 union (select 1.00000000, 2.00000)"""
qt_union33 """select 1, 2 union all select 1.0, 2.0 union (select 1.00000000, 2.00000) """
qt_union34 """select 1, 2 union all select 1.0, 2.0 union all (select 1.00000000, 2.00000) """
qt_union35 """select 1, 2 union select 1.0, 2.0 union all (select 1.00000000, 2.00000) """
qt_union36 """select 1, 2 union distinct select 1.0, 2.0 union distinct (select 1.00000000, 2.00000) """
qt_union38 """select "2016-07-01" union (select "2016-07-02")"""
// test_union_bug
// PALO-3617
qt_union36 """select * from (select 1 as a, 2 as b union select 3, 3) c where a = 1"""
// cast类型
def res5 = sql"""(select k1, k2 from setOperationTable) union (select k2, cast(k1 as int) from setOperationTable)
order by k1, k2"""
def res6 = sql"""(select k1, k2 from setOperationTable) union (select k2, cast(k1 as int) from setOperationTable order by k2)
order by k1, k2"""
check2_doris(res5, res6)
def res7 = sql"""(select k1, k2 from setOperationTable) union (select k2, cast(k3 as int) from setOperationTable) order by k1, k2"""
def res8 = sql"""(select k1, k2 from setOperationTable) union (select k2, cast(k3 as int) from setOperationTable order by k2) order
by k1, k2"""
check2_doris(res7, res8)
// 不同类型不同个数
test {
sql """select k1, k2 from setOperationTable union select k1, k3, k4 from setOperationTable order by k1, k2"""
check {result, exception, startTime, endTime ->
assertTrue(exception != null)
logger.info(exception.message)
}
}
}