database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](Nereids) refactor column pruning (#17579)

Browse Source 
This pr refactor the column pruning by the visitor, the good sides
1. easy to provide ability of column pruning for new plan by implement the interface `OutputPrunable` if the plan contains output field or do nothing if not contains output field, don't need to add new rule like `PruneXxxChildColumns`, few scenarios need to override the visit function to write special logic, like prune the LogicalSetOperation and Aggregate
2. support shrink output field in some plans, this can skip some useless operations so improvement

example:
```sql
select id 
from (
  select id, sum(age)
  from student
  group by id
)a
```

we should prune the useless `sum (age)` in the aggregate.
before refactor:
```
LogicalProject ( distinct=false, projects=[id#0], excepts=[], canEliminate=true )
+--LogicalSubQueryAlias ( qualifier=[a] )
   +--LogicalAggregate ( groupByExpr=[id#0], outputExpr=[id#0, sum(age#2) AS `sum(age)`#4], hasRepeat=false )
      +--LogicalProject ( distinct=false, projects=[id#0, age#2], excepts=[], canEliminate=true )
         +--LogicalOlapScan ( qualified=default_cluster:test.student, indexName=<index_not_selected>, selectedIndexId=10007, preAgg=ON )
```

after refactor:
```
LogicalProject ( distinct=false, projects=[id#0], excepts=[], canEliminate=true )
+--LogicalSubQueryAlias ( qualifier=[a] )
   +--LogicalAggregate ( groupByExpr=[id#0], outputExpr=[id#0], hasRepeat=false )
      +--LogicalProject ( distinct=false, projects=[id#0], excepts=[], canEliminate=true )
         +--LogicalOlapScan ( qualified=default_cluster:test.student, indexName=<index_not_selected>, selectedIndexId=10007, preAgg=ON )
```
This commit is contained in:
924060929
2023-03-24 09:00:48 +08:00
committed by GitHub
parent 678314d657
commit d3e7f12ada
44 changed files with 627 additions and 582 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java
View File

@ -844,7 +844,7 @@ public final class AggregateInfo extends AggregateInfoBase {
// why output and intermediate may have different materialized slots?
// because some slot is materialized by materializeSrcExpr method directly
// in that case, only output slots is materialized
// assume output tuple has correct marterialized infomation
// assume output tuple has correct materialized information
// we update intermediate tuple and materializedSlots based on output tuple
materializedSlots.clear();
ArrayList<SlotDescriptor> outputSlots = outputTupleDesc.getSlots();

6
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -1671,7 +1671,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.map(e -> ExpressionTranslator.translate(e, context))
.collect(Collectors.toCollection(ArrayList::new));
TupleDescriptor tupleDescriptor = generateTupleDesc(generate.getGeneratorOutput(), null, context);
List<SlotId> outputSlotIds = Stream.concat(currentFragment.getPlanRoot().getTupleIds().stream(),
List<TupleId> childOutputTupleIds = currentFragment.getPlanRoot().getOutputTupleIds();
if (childOutputTupleIds == null || childOutputTupleIds.isEmpty()) {
childOutputTupleIds = currentFragment.getPlanRoot().getTupleIds();
}
List<SlotId> outputSlotIds = Stream.concat(childOutputTupleIds.stream(),
Stream.of(tupleDescriptor.getId()))
.map(id -> context.getTupleDesc(id).getSlots())
.flatMap(List::stream)

110
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.jobs.batch;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.RewriteJob;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet;
@ -56,6 +57,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.logical.MergeSetOperations;
import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.NormalizeSort;
import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition;
import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet;
import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin;
@ -63,6 +65,8 @@ import org.apache.doris.nereids.rules.rewrite.logical.PushdownLimit;
import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
import org.apache.doris.nereids.rules.rewrite.logical.SplitLimit;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
@ -89,7 +93,7 @@ public class NereidsRewriter extends BatchRewriteJob {
// ExtractSingleTableExpressionFromDisjunction conflict to InPredicateToEqualToRule
// in the ExpressionNormalization, so must invoke in another job, or else run into
// deep loop
// dead loop
topDown(
new ExtractSingleTableExpressionFromDisjunction()
)
@ -117,10 +121,11 @@ public class NereidsRewriter extends BatchRewriteJob {
// The rule modification needs to be done after the subquery is unnested,
// because for scalarSubQuery, the connection condition is stored in apply in the analyzer phase,
// but when normalizeAggregate is performed, the members in apply cannot be obtained,
// but when normalizeAggregate/normalizeSort is performed, the members in apply cannot be obtained,
// resulting in inconsistent output results and results in apply
topDown(
new NormalizeAggregate()
new NormalizeAggregate(),
new NormalizeSort()
),
topDown(
@ -138,50 +143,50 @@ public class NereidsRewriter extends BatchRewriteJob {
),
topic("Rewrite join",
// infer not null filter, then push down filter, and then reorder join(cross join to inner join)
topDown(
new InferFilterNotNull(),
new InferJoinNotNull()
),
// ReorderJoin depends PUSH_DOWN_FILTERS
// the PUSH_DOWN_FILTERS depends on lots of rules, e.g. merge project, eliminate outer,
// sometimes transform the bottom plan make some rules usable which can apply to the top plan,
// but top-down traverse can not cover this case in one iteration, so bottom-up is more
// efficient because it can find the new plans and apply transform wherever it is
bottomUp(RuleSet.PUSH_DOWN_FILTERS),
// infer not null filter, then push down filter, and then reorder join(cross join to inner join)
topDown(
new InferFilterNotNull(),
new InferJoinNotNull()
),
// ReorderJoin depends PUSH_DOWN_FILTERS
// the PUSH_DOWN_FILTERS depends on lots of rules, e.g. merge project, eliminate outer,
// sometimes transform the bottom plan make some rules usable which can apply to the top plan,
// but top-down traverse can not cover this case in one iteration, so bottom-up is more
// efficient because it can find the new plans and apply transform wherever it is
bottomUp(RuleSet.PUSH_DOWN_FILTERS),
topDown(
new MergeFilters(),
new ReorderJoin(),
new PushFilterInsideJoin(),
new FindHashConditionForJoin(),
new ConvertInnerOrCrossJoin(),
new EliminateNullAwareLeftAntiJoin()
),
topDown(
new EliminateDedupJoinCondition()
)
topDown(
new MergeFilters(),
new ReorderJoin(),
new PushFilterInsideJoin(),
new FindHashConditionForJoin(),
new ConvertInnerOrCrossJoin(),
new EliminateNullAwareLeftAntiJoin()
),
topDown(
new EliminateDedupJoinCondition()
)
),
topic("Column pruning and infer predicate",
topDown(new ColumnPruning()),
custom(RuleType.COLUMN_PRUNING, () -> new ColumnPruning()),
custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()),
custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()),
// column pruning create new project, so we should use PUSH_DOWN_FILTERS
// to change filter-project to project-filter
bottomUp(RuleSet.PUSH_DOWN_FILTERS),
// column pruning create new project, so we should use PUSH_DOWN_FILTERS
// to change filter-project to project-filter
bottomUp(RuleSet.PUSH_DOWN_FILTERS),
// after eliminate outer join in the PUSH_DOWN_FILTERS, we can infer more predicate and push down
custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()),
// after eliminate outer join in the PUSH_DOWN_FILTERS, we can infer more predicate and push down
custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()),
bottomUp(RuleSet.PUSH_DOWN_FILTERS),
bottomUp(RuleSet.PUSH_DOWN_FILTERS),
// after eliminate outer join, we can move some filters to join.otherJoinConjuncts,
// this can help to translate plan to backend
topDown(
new PushFilterInsideJoin()
)
// after eliminate outer join, we can move some filters to join.otherJoinConjuncts,
// this can help to translate plan to backend
topDown(
new PushFilterInsideJoin()
)
),
// this rule should invoke after ColumnPruning
@ -189,20 +194,35 @@ public class NereidsRewriter extends BatchRewriteJob {
// we need to execute this rule at the end of rewrite
// to avoid two consecutive same project appear when we do optimization.
topic("Others optimization", topDown(
topic("Others optimization",
bottomUp(ImmutableList.<RuleFactory>builder().addAll(ImmutableList.of(
new EliminateNotNull(),
new EliminateLimit(),
new EliminateFilter(),
new PruneOlapScanPartition(),
new SelectMaterializedIndexWithAggregate(),
new SelectMaterializedIndexWithoutAggregate(),
new PruneOlapScanTablet(),
new EliminateAggregate(),
new MergeSetOperations(),
new PushdownLimit(),
new SplitLimit(),
new BuildAggForUnion()
)),
// after eliminate filter, the project maybe can push down again,
// so we add push down rules
)).addAll(RuleSet.PUSH_DOWN_FILTERS).build())
),
// TODO: I think these rules should be implementation rules, and generate alternative physical plans.
topic("Table/MV/Physical optimization",
topDown(
// TODO: the logical plan should not contains any phase information,
// we should refactor like AggregateStrategies, e.g. LimitStrategies,
// generate one PhysicalLimit if current distribution is gather or two
// PhysicalLimits with gather exchange
new SplitLimit(),
new SelectMaterializedIndexWithAggregate(),
new SelectMaterializedIndexWithoutAggregate(),
new PruneOlapScanTablet(),
new PruneOlapScanPartition()
)
),
// this rule batch must keep at the end of rewrite to do some plan check
topic("Final rewrite and check", bottomUp(

10
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -88,6 +88,7 @@ public enum RuleType {
// rewrite rules
NORMALIZE_AGGREGATE(RuleTypeClass.REWRITE),
NORMALIZE_SORT(RuleTypeClass.REWRITE),
NORMALIZE_REPEAT(RuleTypeClass.REWRITE),
EXTRACT_AND_NORMALIZE_WINDOW_EXPRESSIONS(RuleTypeClass.REWRITE),
CHECK_AND_STANDARDIZE_WINDOW_FUNCTION_AND_FRAME(RuleTypeClass.REWRITE),
@ -128,14 +129,7 @@ public enum RuleType {
PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE),
PUSHDOWN_ALIAS_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE),
// column prune rules,
COLUMN_PRUNE_AGGREGATION_CHILD(RuleTypeClass.REWRITE),
COLUMN_PRUNE_FILTER_CHILD(RuleTypeClass.REWRITE),
PRUNE_ONE_ROW_RELATION_COLUMN(RuleTypeClass.REWRITE),
COLUMN_PRUNE_SORT_CHILD(RuleTypeClass.REWRITE),
COLUMN_PRUNE_SORT(RuleTypeClass.REWRITE),
COLUMN_PRUNE_JOIN_CHILD(RuleTypeClass.REWRITE),
COLUMN_PRUNE_REPEAT_CHILD(RuleTypeClass.REWRITE),
COLUMN_PRUNING(RuleTypeClass.REWRITE),
// expression of plan rewrite
REWRITE_ONE_ROW_RELATION_EXPRESSION(RuleTypeClass.REWRITE),
REWRITE_PROJECT_EXPRESSION(RuleTypeClass.REWRITE),

272
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruning.java
View File

@ -17,33 +17,271 @@
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.PlanRuleFactory;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RulePromise;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning.PruneContext;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.logical.OutputPrunable;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
* column prune rule set.
* ColumnPruning.
*
* you should implement OutputPrunable for your plan to provide the ability of column pruning
*
* functions:
*
* 1. prune/shrink output field for OutputPrunable, e.g.
*
* project(projects=[k1, sum(v1)]) project(projects=[k1, sum(v1)])
* | -> |
* agg(groupBy=[k1], output=[k1, sum(v1), sum(v2)] agg(groupBy=[k1], output=[k1, sum(v1)])
*
* 2. add project for the plan which prune children's output failed, e.g. the filter not record
* the output, and we can not prune/shrink output field for the filter, so we should add project on filter.
*
* agg(groupBy=[a]) agg(groupBy=[a])
* | |
* filter(b > 10) -> project(a)
* | |
* plan filter(b > 10)
* |
* plan
*/
public class ColumnPruning implements PlanRuleFactory {
public class ColumnPruning extends DefaultPlanRewriter<PruneContext> implements CustomRewriter {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
new PruneFilterChildColumns().build(),
new PruneAggChildColumns().build(),
new PruneJoinChildrenColumns().build(),
new PruneSortColumns().build(),
new PruneSortChildColumns().build(),
new MergeProjects().build(),
new PruneRepeatChildColumns().build()
);
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
return plan.accept(this, new PruneContext(plan.getOutputSet(), null));
}
@Override
public RulePromise defaultPromise() {
return RulePromise.REWRITE;
public Plan visit(Plan plan, PruneContext context) {
if (plan instanceof OutputPrunable) {
// the case 1 in the class comment
// two steps: prune current output and prune children
OutputPrunable outputPrunable = (OutputPrunable) plan;
plan = pruneOutput(plan, outputPrunable.getOutputs(), outputPrunable::pruneOutputs, context);
return pruneChildren(plan);
} else {
// e.g.
//
// project(a)
// |
// | require: [a]
// v
// filter(b > 1) <- process currently
// |
// | require: [a, b]
// v
// child plan
//
// the filter is not OutputPrunable, we should pass through the parent required slots
// (slot a, which in the context.requiredSlots) and the used slots currently(slot b) to child plan.
return pruneChildren(plan, context.requiredSlots);
}
}
// union can not prune children by the common logic, we must override visit method to write special code.
@Override
public Plan visitLogicalUnion(LogicalUnion union, PruneContext context) {
LogicalUnion prunedOutputUnion = pruneOutput(union, union.getOutputs(), union::pruneOutputs, context);
// start prune children of union
List<Slot> originOutput = union.getOutput();
Set<Slot> prunedOutput = prunedOutputUnion.getOutputSet();
Set<Integer> prunedOutputIndexes = IntStream.range(0, originOutput.size())
.filter(index -> prunedOutput.contains(originOutput.get(index)))
.boxed()
.collect(ImmutableSet.toImmutableSet());
AtomicBoolean changed = new AtomicBoolean(false);
List<Plan> prunedChildren = prunedOutputUnion.children().stream()
.map(child -> {
List<Slot> childOutput = child.getOutput();
Set<Slot> prunedChildOutput = prunedOutputIndexes.stream()
.map(childOutput::get)
.collect(ImmutableSet.toImmutableSet());
Plan prunedChild = doPruneChild(prunedOutputUnion, child, prunedChildOutput);
if (prunedChild != child) {
changed.set(true);
}
return prunedChild;
})
.collect(ImmutableList.toImmutableList());
if (!changed.get()) {
return prunedOutputUnion;
}
return prunedOutputUnion.withChildren(prunedChildren);
}
// we should keep the output of LogicalSetOperation and all the children
@Override
public Plan visitLogicalExcept(LogicalExcept except, PruneContext context) {
return skipPruneThisAndFirstLevelChildren(except);
}
@Override
public Plan visitLogicalIntersect(LogicalIntersect intersect, PruneContext context) {
return skipPruneThisAndFirstLevelChildren(intersect);
}
// the backend not support filter(project(agg)), so we can not prune the key set in the agg,
// only prune the agg functions here
@Override
public Plan visitLogicalAggregate(LogicalAggregate<? extends Plan> aggregate, PruneContext context) {
return pruneAggregate(aggregate, context);
}
// same as aggregate
@Override
public Plan visitLogicalRepeat(LogicalRepeat<? extends Plan> repeat, PruneContext context) {
return pruneAggregate(repeat, context);
}
private Plan pruneAggregate(Aggregate agg, PruneContext context) {
// first try to prune group by and aggregate functions
Aggregate prunedOutputAgg = pruneOutput(agg, agg.getOutputs(), agg::pruneOutputs, context);
List<Expression> groupByExpressions = prunedOutputAgg.getGroupByExpressions();
List<NamedExpression> outputExpressions = prunedOutputAgg.getOutputExpressions();
// then fill up group by
Aggregate fillUpOutputRepeat = fillUpGroupByToOutput(groupByExpressions, outputExpressions)
.map(fullOutput -> prunedOutputAgg.withAggOutput(fullOutput))
.orElse(prunedOutputAgg);
return pruneChildren(fillUpOutputRepeat);
}
private Plan skipPruneThisAndFirstLevelChildren(Plan plan) {
Set<Slot> requireAllOutputOfChildren = plan.children()
.stream()
.flatMap(child -> child.getOutputSet().stream())
.collect(Collectors.toSet());
return pruneChildren(plan, requireAllOutputOfChildren);
}
private static Optional<List<NamedExpression>> fillUpGroupByToOutput(
List<Expression> groupBy, List<NamedExpression> output) {
if (output.containsAll(groupBy)) {
return Optional.empty();
}
List<NamedExpression> aggFunctions = Lists.newArrayList(output);
aggFunctions.removeAll(groupBy);
return Optional.of(ImmutableList.<NamedExpression>builder()
.addAll((List) groupBy)
.addAll(aggFunctions)
.build());
}
public static final <P extends Plan> P pruneOutput(P plan, List<NamedExpression> originOutput,
Function<List<NamedExpression>, P> withPrunedOutput, PruneContext context) {
Optional<List<NamedExpression>> prunedOutputs = pruneOutput(originOutput, context);
return prunedOutputs.map(withPrunedOutput).orElse(plan);
}
/** prune output */
public static Optional<List<NamedExpression>> pruneOutput(
List<NamedExpression> originOutput, PruneContext context) {
List<NamedExpression> prunedOutputs = originOutput.stream()
.filter(output -> context.requiredSlots.contains(output.toSlot()))
.collect(ImmutableList.toImmutableList());
if (prunedOutputs.isEmpty()) {
NamedExpression minimumColumn = ExpressionUtils.selectMinimumColumn(originOutput);
prunedOutputs = ImmutableList.of(minimumColumn);
}
return prunedOutputs.equals(originOutput)
? Optional.empty()
: Optional.of(prunedOutputs);
}
private final <P extends Plan> P pruneChildren(P plan) {
return pruneChildren(plan, ImmutableSet.of());
}
private final <P extends Plan> P pruneChildren(P plan, Set<Slot> parentRequiredSlots) {
if (plan.arity() == 0) {
// leaf
return plan;
}
Set<Slot> currentUsedSlots = plan.getInputSlots();
Set<Slot> childrenRequiredSlots = parentRequiredSlots.isEmpty()
? currentUsedSlots
: ImmutableSet.<Slot>builder()
.addAll(parentRequiredSlots)
.addAll(currentUsedSlots)
.build();
List<Plan> newChildren = new ArrayList<>();
boolean hasNewChildren = false;
for (Plan child : plan.children()) {
Set<Slot> childOutputSet = child.getOutputSet();
Set<Slot> childRequiredSlots = Sets.intersection(childrenRequiredSlots, childOutputSet);
if (childRequiredSlots.isEmpty()) {
childRequiredSlots = ImmutableSet.of(ExpressionUtils.selectMinimumColumn(childOutputSet));
}
Plan prunedChild = doPruneChild(plan, child, childRequiredSlots);
if (prunedChild != child) {
hasNewChildren = true;
}
newChildren.add(prunedChild);
}
return hasNewChildren ? (P) plan.withChildren(newChildren) : plan;
}
private Plan doPruneChild(Plan plan, Plan child, Set<Slot> childRequiredSlots) {
boolean isProject = plan instanceof LogicalProject;
Plan prunedChild = child.accept(this, new PruneContext(childRequiredSlots, plan));
// the case 2 in the class comment, prune child's output failed
if (!isProject && !Sets.difference(prunedChild.getOutputSet(), childRequiredSlots).isEmpty()) {
prunedChild = new LogicalProject<>(ImmutableList.copyOf(childRequiredSlots), prunedChild);
}
return prunedChild;
}
/** PruneContext */
public static class PruneContext {
public Set<Slot> requiredSlots;
public Optional<Plan> parent;
public PruneContext(Set<Slot> requiredSlots, Plan parent) {
this.requiredSlots = requiredSlots;
this.parent = Optional.ofNullable(parent);
}
}
}

17
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortColumns.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeSort.java
View File

@ -27,19 +27,24 @@ import com.google.common.collect.ImmutableList;
import java.util.stream.Collectors;
/**
the sort node will create new slots for order by keys if the order by keys is not in the output
so need create a project above sort node to prune the unnecessary order by keys
* the sort node will create new slots for order by keys if the order by keys is not in the output
* so need create a project above sort node to prune the unnecessary order by keys. This means the
* Tuple slots size is difference to PhysicalSort.output.size. If not prune and hide the order key,
* the upper plan node will see the temporary slots and treat as output, and then translate failed.
* This is trick, we should add sort output tuple to ensure the tuple slot size is equals, but it
* has large workload. I think we should refactor the PhysicalPlanTranslator in the future, and
* process PhysicalProject(output)/PhysicalDistribute more general.
*/
public class PruneSortColumns extends OneRewriteRuleFactory {
public class NormalizeSort extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalSort()
.when(sort -> !sort.isOrderKeysPruned() && !sort.getOutputSet()
.when(sort -> !sort.isNormalized() && !sort.getOutputSet()
.containsAll(sort.getOrderKeys().stream()
.map(orderKey -> orderKey.getExpr()).collect(Collectors.toSet())))
.then(sort -> {
return new LogicalProject(sort.getOutput(), ImmutableList.of(), false,
sort.withOrderKeysPruned(true));
}).toRule(RuleType.COLUMN_PRUNE_SORT);
sort.withNormalize(true));
}).toRule(RuleType.NORMALIZE_SORT);
}
}

89
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java
View File

@ -1,89 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Set;
/**
* prune its child output according to agg.
* pattern: agg()
* table a: k1,k2,k3,v1
* select k1,sum(v1) from a group by k1
* plan tree:
* agg
* |
* scan(k1,k2,k3,v1)
* transformed:
* agg
* |
* project(k1,v1)
* |
* scan(k1,k2,k3,v1)
*/
public class PruneAggChildColumns extends OneRewriteRuleFactory {
@Override
public Rule build() {
return RuleType.COLUMN_PRUNE_AGGREGATION_CHILD.build(logicalAggregate().then(agg -> {
List<Slot> childOutput = agg.child().getOutput();
if (isAggregateWithConstant(agg) && agg.getGroupByExpressions().isEmpty()) {
Slot slot = ExpressionUtils.selectMinimumColumn(childOutput);
if (childOutput.size() == 1 && childOutput.get(0).equals(slot)) {
return agg;
}
return agg.withChildren(ImmutableList.of(new LogicalProject<>(ImmutableList.of(slot), agg.child())));
}
Set<Slot> aggInputSlots = agg.getInputSlots();
List<NamedExpression> prunedOutputs = childOutput.stream().filter(aggInputSlots::contains)
.collect(ImmutableList.toImmutableList());
if (prunedOutputs.size() == agg.child().getOutput().size()) {
return agg;
}
return agg.withChildren(ImmutableList.of(new LogicalProject<>(prunedOutputs, agg.child())));
}));
}
/**
* For these aggregate function with constant param. Such as:
* count(*), count(1), sum(1)..etc.
* @return null, if there exists an aggregation function that its parameters contains non-constant expr.
* else return a slot with min data type.
*/
private boolean isAggregateWithConstant(LogicalAggregate<Plan> agg) {
for (NamedExpression output : agg.getOutputExpressions()) {
if (output.anyMatch(SlotReference.class::isInstance)) {
return false;
}
}
return true;
}
}

71
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java
View File

@ -1,71 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* prune filter output.
* pattern: project(filter())
* table a: k1,k2,k3,v1
* select k1 from a where k2 > 3
* plan tree:
* project(k1)
* |
* filter(k2 > 3)
* |
* scan(k1,k2,k3,v1)
* transformed:
* 　project(k1)
* |
* filter(k2 > 3)
* |
* project(k1,k2)
* |
* scan(k1,k2,k3,v1)
*/
public class PruneFilterChildColumns extends AbstractPushDownProjectRule<LogicalFilter<Plan>> {
public PruneFilterChildColumns() {
setRuleType(RuleType.COLUMN_PRUNE_FILTER_CHILD);
setTarget(logicalFilter());
}
@Override
protected Plan pushDownProject(LogicalFilter<Plan> filter, Set<Slot> references) {
Set<Slot> filterInputSlots = filter.getInputSlots();
Set<Slot> required = Stream.concat(references.stream(), filterInputSlots.stream()).collect(Collectors.toSet());
if (required.containsAll(filter.child().getOutput())) {
return filter;
}
return filter.withChildren(
ImmutableList.of(new LogicalProject<>(Lists.newArrayList(required), filter.child()))
);
}
}

98
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java
View File

@ -1,98 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* prune join children output.
* pattern: project(join())
* table a: k1,k2,k3,v1
* table b: k1,k2,v1,v2
* select a.k1,b.k2 from a join b on a.k1 = b.k1 where a.k3 > 1
* plan tree:
* project(a.k1,b.k2)
* |
* join(k1,k2,k3,v1,k1,k2,v1,v2)
* / \
* scan(a) scan(b)
* transformed:
* project(a.k1,b.k2)
* |
* join(k1,k2,k3,v1,k1,k2,v1,v2)
* / \
* project(a.k1,a.k3) project(b.k2,b.k1)
* | |
* scan scan
*/
public class PruneJoinChildrenColumns
extends AbstractPushDownProjectRule<LogicalJoin<Plan, Plan>> {
public PruneJoinChildrenColumns() {
setRuleType(RuleType.COLUMN_PRUNE_JOIN_CHILD);
setTarget(logicalJoin());
}
@Override
protected Plan pushDownProject(LogicalJoin<Plan, Plan> joinPlan,
Set<Slot> references) {
Set<ExprId> exprIds = Stream.of(references, joinPlan.getInputSlots())
.flatMap(Set::stream)
.map(NamedExpression::getExprId)
.collect(Collectors.toSet());
List<NamedExpression> leftInputs = joinPlan.left().getOutput().stream()
.filter(r -> exprIds.contains(r.getExprId())).collect(ImmutableList.toImmutableList());
List<NamedExpression> rightInputs = joinPlan.right().getOutput().stream()
.filter(r -> exprIds.contains(r.getExprId())).collect(ImmutableList.toImmutableList());
if (leftInputs.isEmpty()) {
leftInputs = ImmutableList.of(ExpressionUtils.selectMinimumColumn(joinPlan.left().getOutput()));
}
if (rightInputs.isEmpty()) {
rightInputs = ImmutableList.of(ExpressionUtils.selectMinimumColumn(joinPlan.right().getOutput()));
}
Plan leftPlan = joinPlan.left();
Plan rightPlan = joinPlan.right();
if (leftInputs.size() != leftPlan.getOutput().size()) {
leftPlan = new LogicalProject<>(leftInputs, leftPlan);
}
if (rightInputs.size() != rightPlan.getOutput().size()) {
rightPlan = new LogicalProject<>(rightInputs, rightPlan);
}
return joinPlan.withChildren(ImmutableList.of(leftPlan, rightPlan));
}
}

63
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneRepeatChildColumns.java
View File

@ -1,63 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Set;
/**
* prune its child output according to repeat.
* pattern: repeat()
* table a: k1,k2,k3,v1
* select k1,sum(v1) from a group by grouping sets ((k1))
* plan tree:
* repeat
* |
* scan(k1,k2,k3,v1)
* transformed:
* repeat
* |
* project(k1,v1)
* |
* scan(k1,k2,k3,v1)
*/
public class PruneRepeatChildColumns extends OneRewriteRuleFactory {
@Override
public Rule build() {
return RuleType.COLUMN_PRUNE_REPEAT_CHILD.build(logicalRepeat().then(repeat -> {
List<Slot> childOutput = repeat.child().getOutput();
Set<Slot> groupByInputSlots = repeat.getInputSlots();
List<NamedExpression> prunedOutputs = childOutput.stream().filter(groupByInputSlots::contains)
.collect(ImmutableList.toImmutableList());
if (prunedOutputs.size() == repeat.child().getOutput().size()) {
return repeat;
}
return repeat.withChildren(ImmutableList.of(new LogicalProject<>(prunedOutputs, repeat.child())));
}));
}
}

54
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java
View File

@ -1,54 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import com.google.common.collect.ImmutableList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* prune join children output.
* pattern: project(sort())
*/
public class PruneSortChildColumns extends AbstractPushDownProjectRule<LogicalSort<Plan>> {
public PruneSortChildColumns() {
setRuleType(RuleType.COLUMN_PRUNE_SORT_CHILD);
setTarget(logicalSort());
}
@Override
protected Plan pushDownProject(LogicalSort<Plan> sortPlan, Set<Slot> references) {
Set<Slot> sortSlots = sortPlan.getOutputSet();
Set<Slot> required = Stream.concat(references.stream(), sortSlots.stream()).collect(Collectors.toSet());
if (required.containsAll(sortPlan.child().getOutput())) {
return sortPlan;
}
return sortPlan.withChildren(
ImmutableList.of(new LogicalProject<>(ImmutableList.copyOf(required), sortPlan.child()))
);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java
View File

@ -48,7 +48,7 @@ public class PushdownProjectThroughLimit extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalProject(logicalLimit(any())).thenApply(ctx -> {
return logicalProject(logicalLimit()).thenApply(ctx -> {
LogicalProject<LogicalLimit<Plan>> logicalProject = ctx.root;
LogicalLimit<Plan> logicalLimit = logicalProject.child();
return new LogicalLimit<>(logicalLimit.getLimit(), logicalLimit.getOffset(),

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Aggregate.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.UnaryPlan;
import org.apache.doris.nereids.trees.plans.logical.OutputPrunable;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
@ -32,7 +33,7 @@ import java.util.Set;
/**
* Common interface for logical/physical Aggregate.
*/
public interface Aggregate<CHILD_TYPE extends Plan> extends UnaryPlan<CHILD_TYPE> {
public interface Aggregate<CHILD_TYPE extends Plan> extends UnaryPlan<CHILD_TYPE>, OutputPrunable {
List<Expression> getGroupByExpressions();
@ -43,6 +44,11 @@ public interface Aggregate<CHILD_TYPE extends Plan> extends UnaryPlan<CHILD_TYPE
@Override
Aggregate<Plan> withChildren(List<Plan> children);
@Override
default Aggregate<CHILD_TYPE> pruneOutputs(List<NamedExpression> prunedOutputs) {
return withAggOutput(prunedOutputs);
}
default Set<AggregateFunction> getAggregateFunctions() {
return ExpressionUtils.collect(getOutputExpressions(), AggregateFunction.class::isInstance);
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
View File

@ -52,7 +52,7 @@ import java.util.Optional;
*/
public class LogicalAggregate<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_TYPE>
implements Aggregate<CHILD_TYPE>, OutputSavePoint {
implements Aggregate<CHILD_TYPE> {
private final boolean normalized;
private final List<Expression> groupByExpressions;
@ -225,6 +225,11 @@ public class LogicalAggregate<CHILD_TYPE extends Plan>
sourceRepeat, Optional.empty(), Optional.empty(), child());
}
@Override
public List<NamedExpression> getOutputs() {
return outputExpressions;
}
@Override
public LogicalAggregate<CHILD_TYPE> withAggOutput(List<NamedExpression> newOutput) {
return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved,

20
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java
View File

@ -39,9 +39,9 @@ import java.util.Optional;
* e.g.
* select * from tbl limit 0
*/
public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation {
public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation, OutputPrunable {
private final List<? extends NamedExpression> projects;
private final List<NamedExpression> projects;
public LogicalEmptyRelation(List<? extends NamedExpression> projects) {
this(projects, Optional.empty(), Optional.empty());
@ -59,7 +59,7 @@ public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation {
}
@Override
public List<? extends NamedExpression> getProjects() {
public List<NamedExpression> getProjects() {
return projects;
}
@ -68,6 +68,10 @@ public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation {
return ImmutableList.of();
}
public LogicalEmptyRelation withProjects(List<? extends NamedExpression> projects) {
return new LogicalEmptyRelation(projects, Optional.empty(), Optional.empty());
}
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalEmptyRelation(projects, groupExpression, Optional.of(logicalPropertiesSupplier.get()));
@ -111,4 +115,14 @@ public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation {
public int hashCode() {
return Objects.hash(projects);
}
@Override
public List<NamedExpression> getOutputs() {
return projects;
}
@Override
public Plan pruneOutputs(List<NamedExpression> prunedOutputs) {
return withProjects(prunedOutputs);
}
}

16
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
View File

@ -39,7 +39,7 @@ import java.util.Optional;
* A relation that contains only one row consist of some constant expressions.
* e.g. select 100, 'value'
*/
public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation {
public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation, OutputPrunable {
private final List<NamedExpression> projects;
private final boolean buildUnionNode;
@ -125,7 +125,21 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation
return buildUnionNode;
}
public LogicalOneRowRelation withProjects(List<NamedExpression> namedExpressions) {
return new LogicalOneRowRelation(namedExpressions, buildUnionNode, Optional.empty(), Optional.empty());
}
public Plan withBuildUnionNode(boolean buildUnionNode) {
return new LogicalOneRowRelation(projects, buildUnionNode, Optional.empty(), Optional.empty());
}
@Override
public List<NamedExpression> getOutputs() {
return projects;
}
@Override
public Plan pruneOutputs(List<NamedExpression> prunedOutputs) {
return withProjects(prunedOutputs);
}
}

12
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
View File

@ -41,7 +41,7 @@ import java.util.Optional;
* Logical project plan.
*/
public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE>
implements Project, OutputSavePoint {
implements Project, OutputPrunable {
private final List<NamedExpression> projects;
private final List<NamedExpression> excepts;
@ -199,4 +199,14 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
public boolean isDistinct() {
return isDistinct;
}
@Override
public List<NamedExpression> getOutputs() {
return projects;
}
@Override
public Plan pruneOutputs(List<NamedExpression> prunedOutputs) {
return withProjects(prunedOutputs);
}
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
View File

@ -41,7 +41,7 @@ import java.util.Optional;
* LogicalRepeat.
*/
public class LogicalRepeat<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE>
implements Repeat<CHILD_TYPE>, OutputSavePoint {
implements Repeat<CHILD_TYPE> {
// max num of distinct sets in grouping sets clause
public static final int MAX_GROUPING_SETS_NUM = 64;
@ -84,6 +84,11 @@ public class LogicalRepeat<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
return outputExpressions;
}
@Override
public List<NamedExpression> getOutputs() {
return outputExpressions;
}
@Override
public String toString() {
return Utils.toSqlString("LogicalRepeat",

20
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java
View File

@ -46,14 +46,14 @@ public class LogicalSort<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
private final List<OrderKey> orderKeys;
private final boolean orderKeysPruned;
private final boolean normalized;
public LogicalSort(List<OrderKey> orderKeys, CHILD_TYPE child) {
this(orderKeys, Optional.empty(), Optional.empty(), child);
}
public LogicalSort(List<OrderKey> orderKeys, CHILD_TYPE child, boolean orderKeysPruned) {
this(orderKeys, Optional.empty(), Optional.empty(), child, orderKeysPruned);
public LogicalSort(List<OrderKey> orderKeys, CHILD_TYPE child, boolean normalized) {
this(orderKeys, Optional.empty(), Optional.empty(), child, normalized);
}
/**
@ -65,10 +65,10 @@ public class LogicalSort<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
}
public LogicalSort(List<OrderKey> orderKeys, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, CHILD_TYPE child, boolean orderKeysPruned) {
Optional<LogicalProperties> logicalProperties, CHILD_TYPE child, boolean normalized) {
super(PlanType.LOGICAL_SORT, groupExpression, logicalProperties, child);
this.orderKeys = ImmutableList.copyOf(Objects.requireNonNull(orderKeys, "orderKeys can not be null"));
this.orderKeysPruned = orderKeysPruned;
this.normalized = normalized;
}
@Override
@ -80,8 +80,8 @@ public class LogicalSort<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
return orderKeys;
}
public boolean isOrderKeysPruned() {
return orderKeysPruned;
public boolean isNormalized() {
return normalized;
}
@Override
@ -122,13 +122,13 @@ public class LogicalSort<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
@Override
public LogicalSort<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new LogicalSort<>(orderKeys, children.get(0), orderKeysPruned);
return new LogicalSort<>(orderKeys, children.get(0), normalized);
}
@Override
public LogicalSort<Plan> withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalSort<>(orderKeys, groupExpression, Optional.of(getLogicalProperties()), child(),
orderKeysPruned);
normalized);
}
@Override
@ -141,7 +141,7 @@ public class LogicalSort<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
Optional.of(getLogicalProperties()), child(), false);
}
public LogicalSort<Plan> withOrderKeysPruned(boolean orderKeysPruned) {
public LogicalSort<Plan> withNormalize(boolean orderKeysPruned) {
return new LogicalSort<>(orderKeys, groupExpression, Optional.of(getLogicalProperties()), child(),
orderKeysPruned);
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
View File

@ -32,7 +32,7 @@ import java.util.Optional;
/**
* Logical Union.
*/
public class LogicalUnion extends LogicalSetOperation {
public class LogicalUnion extends LogicalSetOperation implements OutputPrunable {
// When the union is DISTINCT, an additional LogicalAggregation needs to be created,
// so add this flag to judge whether agg has been created to avoid repeated creation
@ -143,4 +143,9 @@ public class LogicalUnion extends LogicalSetOperation {
public LogicalUnion withNewChildren(List<Plan> children) {
return withChildren(children);
}
@Override
public LogicalUnion pruneOutputs(List<NamedExpression> prunedOutputs) {
return withNewOutputs(prunedOutputs);
}
}

30
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputPrunable.java Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import java.util.List;
/** OutputPrunable */
public interface OutputPrunable extends OutputSavePoint {
List<NamedExpression> getOutputs();
Plan pruneOutputs(List<NamedExpression> prunedOutputs);
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
View File

@ -134,6 +134,11 @@ public class PhysicalHashAggregate<CHILD_TYPE extends Plan> extends PhysicalUnar
return outputExpressions;
}
@Override
public List<NamedExpression> getOutputs() {
return outputExpressions;
}
public Optional<List<Expression>> getPartitionExpressions() {
return partitionExpressions;
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java
View File

@ -90,6 +90,11 @@ public class PhysicalRepeat<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD
return outputExpressions;
}
@Override
public List<NamedExpression> getOutputs() {
return outputExpressions;
}
@Override
public String toString() {
return Utils.toSqlString("PhysicalRepeat[" + id.asInt() + "]" + getGroupIdAsString(),

25
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/DefaultPlanRewriter.java
View File

@ -32,16 +32,7 @@ public abstract class DefaultPlanRewriter<C> extends PlanVisitor<Plan, C> {
@Override
public Plan visit(Plan plan, C context) {
List<Plan> newChildren = new ArrayList<>();
boolean hasNewChildren = false;
for (Plan child : plan.children()) {
Plan newChild = child.accept(this, context);
if (newChild != child) {
hasNewChildren = true;
}
newChildren.add(newChild);
}
return hasNewChildren ? plan.withChildren(newChildren) : plan;
return visitChildren(this, plan, context);
}
@Override
@ -52,4 +43,18 @@ public abstract class DefaultPlanRewriter<C> extends PlanVisitor<Plan, C> {
}
return storageLayerAggregate;
}
/** visitChildren */
public static final <P extends Plan, C> P visitChildren(DefaultPlanRewriter<C> rewriter, P plan, C context) {
List<Plan> newChildren = new ArrayList<>();
boolean hasNewChildren = false;
for (Plan child : plan.children()) {
Plan newChild = child.accept(rewriter, context);
if (newChild != child) {
hasNewChildren = true;
}
newChildren.add(newChild);
}
return hasNewChildren ? (P) plan.withChildren(newChildren) : plan;
}
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
@ -203,10 +204,10 @@ public class ExpressionUtils {
/**
* Choose the minimum slot from input parameter.
*/
public static Slot selectMinimumColumn(List<Slot> slots) {
public static <S extends NamedExpression> S selectMinimumColumn(Collection<S> slots) {
Preconditions.checkArgument(!slots.isEmpty());
Slot minSlot = null;
for (Slot slot : slots) {
S minSlot = null;
for (S slot : slots) {
if (minSlot == null) {
minSlot = slot;
} else {

7
fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
View File

@ -53,7 +53,12 @@ public class TableFunctionNode extends PlanNode {
public TableFunctionNode(PlanNodeId id, PlanNode inputNode, TupleId lateralViewTupleId,
ArrayList<Expr> fnCallExprList, List<SlotId> outputSlotIds) {
super(id, "TABLE FUNCTION NODE", StatisticalType.TABLE_FUNCTION_NODE);
tupleIds.addAll(inputNode.getTupleIds());
List<TupleId> childOutputTupleIds = inputNode.getOutputTupleIds();
if (childOutputTupleIds != null && !childOutputTupleIds.isEmpty()) {
tupleIds.addAll(childOutputTupleIds);
} else {
tupleIds.addAll(inputNode.getTupleIds());
}
tupleIds.add(lateralViewTupleId);
this.lateralViewTupleIds = Lists.newArrayList(lateralViewTupleId);
this.fnCallExprList = fnCallExprList;

3
fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java
View File

@ -233,7 +233,8 @@ public class RuntimeFilterTest extends SSBTestBase {
}
private Optional<List<RuntimeFilter>> getRuntimeFilters(String sql) {
PlanChecker checker = PlanChecker.from(connectContext).analyze(sql)
PlanChecker checker = PlanChecker.from(connectContext)
.analyze(sql)
.rewrite()
.implement();
PhysicalPlan plan = checker.getPhysicalPlan();

50
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java
View File

@ -61,7 +61,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
PlanChecker.from(connectContext)
.analyze("select id,name,grade from student left join score on student.id = score.sid"
+ " where score.grade > 60")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalProject(
logicalFilter(
@ -93,7 +93,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
.analyze("select name,sex,cid,grade "
+ "from student left join score on student.id = score.sid "
+ "where score.grade > 60")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalProject(
logicalFilter(
@ -123,7 +123,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
public void testPruneColumns3() {
PlanChecker.from(connectContext)
.analyze("select id,name from student where age > 18")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalProject(
logicalFilter(
@ -145,7 +145,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
+ "on student.id = score.sid left join course "
+ "on score.cid = course.cid "
+ "where score.grade > 60")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalProject(
logicalFilter(
@ -183,7 +183,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
public void pruneCountStarStmt() {
PlanChecker.from(connectContext)
.analyze("SELECT COUNT(*) FROM test.course")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalAggregate(
logicalProject(
@ -198,7 +198,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
public void pruneCountConstantStmt() {
PlanChecker.from(connectContext)
.analyze("SELECT COUNT(1) FROM test.course")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalAggregate(
logicalProject(
@ -213,7 +213,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
public void pruneCountConstantAndSumConstantStmt() {
PlanChecker.from(connectContext)
.analyze("SELECT COUNT(1), SUM(2) FROM test.course")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalAggregate(
logicalProject(
@ -228,7 +228,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
public void pruneCountStarAndSumConstantStmt() {
PlanChecker.from(connectContext)
.analyze("SELECT COUNT(*), SUM(2) FROM test.course")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalAggregate(
logicalProject(
@ -243,7 +243,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
public void pruneCountStarAndSumColumnStmt() {
PlanChecker.from(connectContext)
.analyze("SELECT COUNT(*), SUM(grade) FROM test.score")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalAggregate(
logicalProject(
@ -258,7 +258,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
public void pruneCountStarAndSumColumnAndSumConstantStmt() {
PlanChecker.from(connectContext)
.analyze("SELECT COUNT(*), SUM(grade) + SUM(2) FROM test.score")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalAggregate(
logicalProject(
@ -273,7 +273,7 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
public void pruneColumnForOneSideOnCrossJoin() {
PlanChecker.from(connectContext)
.analyze("select id,name from student cross join score")
.applyTopDown(new ColumnPruning())
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalProject(
logicalJoin(
@ -291,7 +291,33 @@ public class ColumnPruningTest extends TestWithFeService implements MemoPatternM
);
}
@Test
public void pruneAggregateOutput() {
PlanChecker.from(connectContext)
.analyze("select id from (select id, sum(age) from student group by id)a")
.customRewrite(new ColumnPruning())
.matchesFromRoot(
logicalProject(
logicalSubQueryAlias(
logicalAggregate(
logicalProject(
logicalOlapScan()
).when(p -> getOutputQualifiedNames(p).equals(
ImmutableList.of("default_cluster:test.student.id")
))
).when(agg -> getOutputQualifiedNames(agg.getOutputs()).equals(
ImmutableList.of("default_cluster:test.student.id")
))
)
)
);
}
private List<String> getOutputQualifiedNames(LogicalProject<? extends Plan> p) {
return p.getProjects().stream().map(NamedExpression::getQualifiedName).collect(Collectors.toList());
return getOutputQualifiedNames(p.getOutputs());
}
private List<String> getOutputQualifiedNames(List<? extends NamedExpression> output) {
return output.stream().map(NamedExpression::getQualifiedName).collect(Collectors.toList());
}
}

24
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java
View File

@ -83,15 +83,25 @@ class ReorderJoinTest implements MemoPatternMatchSupported {
.join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0))
.joinEmptyOn(scan3, JoinType.CROSS_JOIN)
.filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0)))
.build(),
new LogicalPlanBuilder(scan1)
.joinEmptyOn(scan3, JoinType.CROSS_JOIN)
.join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0))
.filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0)))
.build()
);
check(plans);
LogicalPlan plan2 = new LogicalPlanBuilder(scan1)
.joinEmptyOn(scan3, JoinType.CROSS_JOIN)
.join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0))
.filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0)))
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan2)
.rewrite()
.printlnTree()
.matchesFromRoot(
logicalJoin(
logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()),
logicalProject(logicalOlapScan())
).whenNot(join -> join.getJoinType().isCrossJoin())
);
}
@Test
@ -116,7 +126,7 @@ class ReorderJoinTest implements MemoPatternMatchSupported {
.rewrite()
.matchesFromRoot(
rightSemiLogicalJoin(
leafPlan(),
logicalProject(logicalOlapScan()),
innerLogicalJoin()
)
);

7
fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
View File

@ -30,6 +30,7 @@ import org.apache.doris.nereids.jobs.batch.CascadesOptimizer;
import org.apache.doris.nereids.jobs.batch.NereidsRewriter;
import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob;
import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob;
import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob;
import org.apache.doris.nereids.memo.CopyInResult;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -123,6 +124,12 @@ public class PlanChecker {
return this;
}
public PlanChecker customRewrite(CustomRewriter customRewriter) {
new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE).execute(cascadesContext.getCurrentJobContext());
cascadesContext.toMemo();
return this;
}
public PlanChecker applyTopDown(RuleFactory ruleFactory) {
return applyTopDown(ruleFactory.buildRules());
}

2
regression-test/data/nereids_syntax_p0/set_operation.out
View File

@ -406,9 +406,9 @@ d d 3
3 9.0 3 9
-- !union30 --
0.0001 1E-7
1.0000 2.0000000
1.0100 2.0000000
0.0001 1E-7
-- !union31 --
1 2

11
regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy
View File

@ -17,14 +17,13 @@
package org.apache.doris.regression.action
import groovy.transform.stc.ClosureParams
import groovy.transform.stc.FromString
import groovy.util.logging.Slf4j
import org.apache.doris.regression.suite.SuiteContext
import org.apache.doris.regression.util.JdbcUtils
import groovy.util.logging.Slf4j
import java.sql.ResultSetMetaData
import java.util.stream.Collectors
import java.sql.Connection
import java.sql.ResultSetMetaData
@Slf4j
class CreateMVAction implements SuiteAction {
@ -63,7 +62,7 @@ class CreateMVAction implements SuiteAction {
while (!sqlResult.contains("FINISHED")) {
def tmp = doRun("SHOW ALTER TABLE MATERIALIZED VIEW ORDER BY CreateTime DESC LIMIT 1;")
sqlResult = tmp.result[0]
log.info("result: ${sqlResult}")
log.info("result: ${sqlResult}".toString())
if (tryTimes == 60 || sqlResult.contains("CANCELLED")) {
throw new IllegalStateException("MV create check times over limit");
}

2
regression-test/suites/mtmv_p0/test_create_mtmv.groovy
View File

@ -56,7 +56,7 @@ suite("test_create_mtmv") {
INSERT INTO ${tableNamePv} VALUES("2022-10-26",1,200),("2022-10-28",2,200),("2022-10-28",3,300);
"""
sql """drop materialized view if exists ${mvName}"""
sql """drop materialized view if exists ${mvName}"""
sql """
CREATE MATERIALIZED VIEW ${mvName}

44
regression-test/suites/mv_p0/multi_slot_multi_mv/multi_slot_multi_mv.groovy
View File

@ -15,8 +15,6 @@
// specific language governing permissions and limitations
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
suite ("multi_slot_multi_mv") {
sql """ DROP TABLE IF EXISTS d_table; """
@ -38,24 +36,40 @@ suite ("multi_slot_multi_mv") {
createMV ("create materialized view k1a2p2ap3p as select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table;")
sql "create materialized view k1a2p2ap3ps as select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1;"
while (!result.contains("FINISHED")){
result = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='d_table' ORDER BY CreateTime DESC LIMIT 1;"
result = result.toString()
logger.info("result: ${result}")
if(result.contains("CANCELLED")){
return
}
Thread.sleep(1000)
}
createMV("create materialized view k1a2p2ap3ps as select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1;")
sql "insert into d_table select -4,-4,-4,'d';"
qt_select_star "select * from d_table order by k1;"
explain {
sql("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1")
contains "(k1a2p2ap3p)"
def retry_times = 60
for (def i = 0; i < retry_times; ++i) {
boolean is_k1a2p2ap3p = false
boolean is_k1a2p2ap3ps = false
boolean is_d_table = false
explain {
sql("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1")
check { explainStr, ex, startTime, endTime ->
if (ex != null) {
throw ex;
}
logger.info("explain result: ${explainStr}".toString())
is_k1a2p2ap3p = explainStr.contains"(k1a2p2ap3p)"
is_k1a2p2ap3ps = explainStr.contains("(k1a2p2ap3ps)")
is_d_table = explainStr.contains("(d_table)")
assert is_k1a2p2ap3p || is_k1a2p2ap3ps || is_d_table
}
}
// FIXME: the mv selector maybe select base table forever when exist multi mv,
// so this pr just treat as success if select base table.
// we should remove is_d_table in the future
if (is_d_table || is_k1a2p2ap3p || is_k1a2p2ap3ps) {
break
}
if (i + 1 == retry_times) {
throw new IllegalStateException("retry and failed too much")
}
sleep(1000)
}
qt_select_mv "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1;"

6
regression-test/suites/mv_p0/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy
View File

@ -15,8 +15,6 @@
// specific language governing permissions and limitations
// under the License.
import org.codehaus.groovy.runtime.IOGroovyMethods
suite ("testAggQueryOnAggMV2") {
sql """ DROP TABLE IF EXISTS emps; """
sql """
@ -51,11 +49,11 @@ suite ("testAggQueryOnAggMV2") {
}
qt_select_star "select * from emps order by empid, salary;"
explain {
explain {
sql("select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;")
contains "(emps_mv)"
}
qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;"
}
}

32
regression-test/suites/mysql_ssl_p0/test_mysql_connection.groovy
View File

@ -15,31 +15,21 @@
// specific language governing permissions and limitations
// under the License.
suite("test_mysql_connection") {
suite("test_mysql_connection") { suite ->
// NOTE: this suite need you install mysql client 5.7 + to support --ssl-mode parameter
def executeMySQLCommand = { String command ->
try {
String line;
StringBuilder errMsg = new StringBuilder();
StringBuilder msg = new StringBuilder();
Process p = Runtime.getRuntime().exec(new String[]{"/bin/bash", "-c", command});
def cmds = ["/bin/bash", "-c", command]
logger.info("Execute: ${cmds}".toString())
Process p = cmds.execute()
BufferedReader errInput = new BufferedReader(new InputStreamReader(p.getErrorStream()));
while ((line = errInput.readLine()) != null) {
errMsg.append(line);
}
assert errMsg.length() == 0: "error occurred!" + errMsg.toString();
errInput.close();
def errMsg = new StringBuilder()
def msg = new StringBuilder()
p.waitForProcessOutput(msg, errMsg)
BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));
while ((line = input.readLine()) != null) {
msg.append(line);
}
assert msg.toString().contains("version"): "error occurred!" + errMsg.toString();
input.close();
} catch (IOException e) {
e.printStackTrace();
}
assert errMsg.length() == 0: "error occurred!" + errMsg
assert msg.toString().contains("version"): "error occurred!" + errMsg
assert p.exitValue() == 0
}
String jdbcUrlConfig = context.config.jdbcUrl;

4
regression-test/suites/nereids_p0/aggregate/aggregate_output_null.groovy
View File

@ -74,7 +74,7 @@ suite("aggregate_output_null") {
('z178NhOZ','b');
"""
qt_select """
order_qt_select """
SELECT
t2.a,
t1.c,
@ -89,4 +89,4 @@ suite("aggregate_output_null") {
sql "DROP TABLE t1"
sql "DROP TABLE t2"
}
}

8
regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy
View File

@ -67,12 +67,12 @@ suite("test_aggregate_collect") {
sql "INSERT INTO ${tableName_12} values(1,10,'2022-11-1',6.8754576), (2,8,'2022-11-3',0.576), (2,10,'2022-11-2',1.234) ,(3,10,'2022-11-2',0.576) ,(5,29,'2022-11-2',6.8754576) ,(6,8,'2022-11-1',6.8754576)"
// Nereids does't support array function
// qt_select43 "select topn_array(level,2) from ${tableName_12}"
// order_qt_select43 "select topn_array(level,2) from ${tableName_12}"
// Nereids does't support array function
// qt_select44 "select topn_array(level,2,100) from ${tableName_12}"
// order_qt_select44 "select topn_array(level,2,100) from ${tableName_12}"
// Nereids does't support array function
// qt_select45 "select topn_array(dt,2,100) from ${tableName_12}"
// order_qt_select45 "select topn_array(dt,2,100) from ${tableName_12}"
// Nereids does't support array function
// qt_select46 "select topn_array(num,2,100) from ${tableName_12}"
// order_qt_select46 "select topn_array(num,2,100) from ${tableName_12}"
sql "DROP TABLE IF EXISTS ${tableName_12}"
}

2
regression-test/suites/nereids_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy
View File

@ -43,6 +43,6 @@ suite("test_array_functions_of_array_difference") {
// Nereids does't support array function
// qt_select "SELECT *, array_difference(k2) FROM ${tableName}"
// qt_select "SELECT *, array_difference(k2) FROM ${tableName} order by k1"
}

2
regression-test/suites/nereids_p0/sql_functions/json_function/test_query_json_array.groovy
View File

@ -43,6 +43,6 @@ suite("test_query_json_array", "query") {
sql "insert into ${tableName} values(4,null,null,'test','2022-01-01 11:11:11');"
sql "insert into ${tableName} values(5,1,true,'test','2022-01-01 11:11:11');"
// Nereids does't support array function
// qt_sql2 "select json_array('k0',k0,'k1',k1,'k2',k2,'k3',k3,'k4',k4,'k5', null,'k6','k6') from ${tableName};"
// order_qt_sql2 "select json_array('k0',k0,'k1',k1,'k2',k2,'k3',k3,'k4',k4,'k5', null,'k6','k6') from ${tableName};"
sql "DROP TABLE ${tableName};"
}

4
regression-test/suites/nereids_syntax_p0/lateral_view.groovy
View File

@ -45,6 +45,7 @@ suite("nereids_lateral_view") {
LATERAL VIEW explode_json_array_string(c2) lv2 AS clv2
LATERAL VIEW explode_json_array_int(c3) lv3 AS clv3
LATERAL VIEW explode_json_array_double(c4) lv4 AS clv4
order by c1, c2, c3, c4, clv1, clv2, clv3, clv4
"""
order_qt_all_function_outer """
@ -53,6 +54,7 @@ suite("nereids_lateral_view") {
LATERAL VIEW explode_json_array_string_outer(c2) lv2 AS clv2
LATERAL VIEW explode_json_array_int_outer(c3) lv3 AS clv3
LATERAL VIEW explode_json_array_double_outer(c4) lv4 AS clv4
order by c1, c2, c3, c4, clv1, clv2, clv3, clv4
"""
order_qt_column_prune """
@ -61,6 +63,7 @@ suite("nereids_lateral_view") {
LATERAL VIEW explode_json_array_string_outer(c2) lv2 AS clv2
LATERAL VIEW explode_json_array_int(c3) lv3 AS clv3
LATERAL VIEW explode_json_array_double_outer(c4) lv4 AS clv4
order by c1, c2, c3, c4, clv1, clv2, clv3, clv4
"""
order_qt_alias_query """
@ -69,6 +72,7 @@ suite("nereids_lateral_view") {
LATERAL VIEW explode_json_array_string_outer(c2) lv2 AS clv2
LATERAL VIEW explode_json_array_int(c3) lv3 AS clv3
LATERAL VIEW explode_json_array_double_outer(c4) lv4 AS clv4
order by c1, c2, c3, c4, clv1, clv2, clv3, clv4
"""
order_qt_function_nested """

18
regression-test/suites/nereids_syntax_p0/set_operation.groovy
View File

@ -193,14 +193,14 @@ suite("test_nereids_set_operation") {
"""
// test_union_basic
qt_union30 """select 1, 2 union select 1.01, 2.0 union (select 0.0001, 0.0000001)"""
qt_union31 """select 1, 2 union (select "hell0", "")"""
qt_union32 """select 1, 2 union select 1.0, 2.0 union (select 1.00000000, 2.00000)"""
qt_union33 """select 1, 2 union all select 1.0, 2.0 union (select 1.00000000, 2.00000) """
qt_union34 """select 1, 2 union all select 1.0, 2.0 union all (select 1.00000000, 2.00000) """
qt_union35 """select 1, 2 union select 1.0, 2.0 union all (select 1.00000000, 2.00000) """
qt_union36 """select 1, 2 union distinct select 1.0, 2.0 union distinct (select 1.00000000, 2.00000) """
qt_union38 """select "2016-07-01" union (select "2016-07-02")"""
qt_union30 """select 1 c1, 2 union select 1.01, 2.0 union (select 0.0001, 0.0000001) order by c1"""
qt_union31 """select 1 c1, 2 union (select "hell0", "") order by c1"""
qt_union32 """select 1 c1, 2 union select 1.0, 2.0 union (select 1.00000000, 2.00000) order by c1"""
qt_union33 """select 1 c1, 2 union all select 1.0, 2.0 union (select 1.00000000, 2.00000) order by c1"""
qt_union34 """select 1 c1, 2 union all select 1.0, 2.0 union all (select 1.00000000, 2.00000) order by c1"""
qt_union35 """select 1 c1, 2 union select 1.0, 2.0 union all (select 1.00000000, 2.00000) order by c1"""
qt_union36 """select 1 c1, 2 union distinct select 1.0, 2.0 union distinct (select 1.00000000, 2.00000) order by c1"""
qt_union38 """select "2016-07-01" c1 union (select "2016-07-02") order by c1"""
// test_union_bug
// PALO-3617
@ -275,7 +275,7 @@ suite("test_nereids_set_operation") {
(select k1, k5 from setOperationTable)
"""
qt_union43 """select '2020-05-25' day from test_table union all select day from test_table;"""
order_qt_union43 """select '2020-05-25' day from test_table union all select day from test_table;"""
qt_union44 """
select * from

4
regression-test/suites/query_p0/aggregate/aggregate_output_null.groovy
View File

@ -72,7 +72,7 @@ suite("aggregate_output_null") {
('z178NhOZ','b');
"""
qt_select """
order_qt_select """
SELECT
t2.a,
t1.c,
@ -87,4 +87,4 @@ suite("aggregate_output_null") {
sql "DROP TABLE t1"
sql "DROP TABLE t2"
}
}

16
regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_collect.groovy
View File

@ -177,7 +177,7 @@ suite("test_aggregate_collect") {
${tableName}
"""
qt_select """
order_qt_select """
SELECT
size(collect_list(c_bool,1)),
size(collect_list(c_tinyint,1)),
@ -449,7 +449,7 @@ suite("test_aggregate_collect") {
${tableName_11}
"""
qt_select """
order_qt_select """
SELECT
size(group_uniq_array(c_bool,1)),
size(group_uniq_array(c_tinyint,1)),
@ -472,7 +472,7 @@ suite("test_aggregate_collect") {
${tableName_11}
"""
qt_select """
order_qt_select """
SELECT
size(group_array(c_bool,1)),
size(group_array(c_tinyint,1)),
@ -613,7 +613,7 @@ suite("test_aggregate_collect") {
sql """
CREATE TABLE IF NOT EXISTS ${tableName_12} (
id int,
level int,
level int,
dt datev2,
num decimal(27,9)
)
@ -624,9 +624,9 @@ suite("test_aggregate_collect") {
"""
sql "INSERT INTO ${tableName_12} values(1,10,'2022-11-1',6.8754576), (2,8,'2022-11-3',0.576), (2,10,'2022-11-2',1.234) ,(3,10,'2022-11-2',0.576) ,(5,29,'2022-11-2',6.8754576) ,(6,8,'2022-11-1',6.8754576)"
qt_select43 "select topn_array(level,2) from ${tableName_12}"
qt_select44 "select topn_array(level,2,100) from ${tableName_12}"
qt_select45 "select topn_array(dt,2,100) from ${tableName_12}"
qt_select46 "select topn_array(num,2,100) from ${tableName_12}"
order_qt_select43 "select topn_array(level,2) from ${tableName_12}"
order_qt_select44 "select topn_array(level,2,100) from ${tableName_12}"
order_qt_select45 "select topn_array(dt,2,100) from ${tableName_12}"
order_qt_select46 "select topn_array(num,2,100) from ${tableName_12}"
sql "DROP TABLE IF EXISTS ${tableName_12}"
}