database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids) Matiarilzed view query rewrite util implementation (#27568)

Browse Source 
The basic util implementatation which is used by materialized view rewrite
This commit is contained in:
seawinde
2023-12-05 11:48:04 +08:00
committed by GitHub
parent 26d642d5e9
commit da40e1c767
18 changed files with 1116 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;

9
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -20,8 +20,10 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.rules.exploration.mv.Mapping.ExpressionIndexMapping;
import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionIndexMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
@ -141,8 +143,9 @@ public abstract class AbstractMaterializedViewRule {
targetTopExpressions, targetStructInfo.getOriginalPlan(), Sets.newHashSet(), Sets.newHashSet());
SlotMapping sourceToTargetSlotMapping = SlotMapping.generate(sourceToTargetMapping);
// mv sql plan expressions transform to query based
List<? extends Expression> queryBasedExpressions = ExpressionUtils.permute(shuttledTargetExpressions,
sourceToTargetSlotMapping.inverse());
List<? extends Expression> queryBasedExpressions = ExpressionUtils.replace(
shuttledTargetExpressions.stream().map(Expression.class::cast).collect(Collectors.toList()),
sourceToTargetSlotMapping.inverse().getSlotMap());
// mv sql query based expression and index mapping
ExpressionIndexMapping.generate(queryBasedExpressions);
// TODO visit source expression and replace the expression with expressionIndexMapping

54
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View File

@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors.PredicatesSpliter;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
@ -55,8 +54,7 @@ public class Predicates {
* Split the expression to equal, range and residual predicate.
* */
public static SplitPredicate splitPredicates(Expression expression) {
PredicatesSpliter predicatesSplit = new PredicatesSpliter(expression);
expression.accept(predicatesSplit, null);
PredicatesSplitter predicatesSplit = new PredicatesSplitter(expression);
return predicatesSplit.getSplitPredicate();
}
@ -64,26 +62,26 @@ public class Predicates {
* The split different representation for predicate expression, such as equal, range and residual predicate.
* */
public static final class SplitPredicate {
private final Expression equalPredicates;
private final Expression rangePredicates;
private final Expression residualPredicates;
private final Expression equalPredicate;
private final Expression rangePredicate;
private final Expression residualPredicate;
public SplitPredicate(Expression equalPredicates, Expression rangePredicates, Expression residualPredicates) {
this.equalPredicates = equalPredicates;
this.rangePredicates = rangePredicates;
this.residualPredicates = residualPredicates;
public SplitPredicate(Expression equalPredicate, Expression rangePredicate, Expression residualPredicate) {
this.equalPredicate = equalPredicate;
this.rangePredicate = rangePredicate;
this.residualPredicate = residualPredicate;
}
public Expression getEqualPredicates() {
return equalPredicates;
public Expression getEqualPredicate() {
return equalPredicate;
}
public Expression getRangePredicates() {
return rangePredicates;
public Expression getRangePredicate() {
return rangePredicate;
}
public Expression getResidualPredicates() {
return residualPredicates;
public Expression getResidualPredicate() {
return residualPredicate;
}
public static SplitPredicate empty() {
@ -103,29 +101,25 @@ public class Predicates {
* isEmpty
* */
public boolean isEmpty() {
return equalPredicates == null
&& rangePredicates == null
&& residualPredicates == null;
}
public Expression composedExpression() {
return ExpressionUtils.and(equalPredicates, rangePredicates, residualPredicates);
return equalPredicate == null
&& rangePredicate == null
&& residualPredicate == null;
}
public List<Expression> toList() {
return ImmutableList.of(equalPredicates, rangePredicates, residualPredicates);
return ImmutableList.of(equalPredicate, rangePredicate, residualPredicate);
}
/**
* Check the predicates in SplitPredicate is whether all true or not
*/
public boolean isAlwaysTrue() {
return equalPredicates instanceof BooleanLiteral
&& rangePredicates instanceof BooleanLiteral
&& residualPredicates instanceof BooleanLiteral
&& ((BooleanLiteral) equalPredicates).getValue()
&& ((BooleanLiteral) rangePredicates).getValue()
&& ((BooleanLiteral) residualPredicates).getValue();
return equalPredicate instanceof BooleanLiteral
&& rangePredicate instanceof BooleanLiteral
&& residualPredicate instanceof BooleanLiteral
&& ((BooleanLiteral) equalPredicate).getValue()
&& ((BooleanLiteral) rangePredicate).getValue()
&& ((BooleanLiteral) residualPredicate).getValue();
}
}
}

112
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java Normal file
View File

@ -0,0 +1,112 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import java.util.ArrayList;
import java.util.List;
/**
* Split the expression to equal, range and residual predicate.
* Should instance when used.
* TODO support complex predicate split
*/
public class PredicatesSplitter {
private final List<Expression> equalPredicates = new ArrayList<>();
private final List<Expression> rangePredicates = new ArrayList<>();
private final List<Expression> residualPredicates = new ArrayList<>();
private final List<Expression> conjunctExpressions;
private final PredicateExtract instance = new PredicateExtract();
public PredicatesSplitter(Expression target) {
this.conjunctExpressions = ExpressionUtils.extractConjunction(target);
for (Expression expression : conjunctExpressions) {
expression.accept(instance, expression);
}
}
/**
* PredicateExtract
*/
public class PredicateExtract extends DefaultExpressionVisitor<Void, Expression> {
@Override
public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Expression sourceExpression) {
Expression leftArg = comparisonPredicate.getArgument(0);
Expression rightArg = comparisonPredicate.getArgument(1);
boolean leftArgOnlyContainsColumnRef = containOnlyColumnRef(leftArg, true);
boolean rightArgOnlyContainsColumnRef = containOnlyColumnRef(rightArg, true);
if (comparisonPredicate instanceof EqualPredicate) {
if (leftArgOnlyContainsColumnRef && rightArgOnlyContainsColumnRef) {
equalPredicates.add(comparisonPredicate);
return null;
} else {
residualPredicates.add(comparisonPredicate);
}
} else if ((leftArgOnlyContainsColumnRef && rightArg instanceof Literal)
|| (rightArgOnlyContainsColumnRef && leftArg instanceof Literal)) {
rangePredicates.add(comparisonPredicate);
} else {
residualPredicates.add(comparisonPredicate);
}
return null;
}
@Override
public Void visitCompoundPredicate(CompoundPredicate compoundPredicate, Expression context) {
if (compoundPredicate instanceof Or) {
residualPredicates.add(compoundPredicate);
return null;
}
return super.visit(compoundPredicate, context);
}
}
public Predicates.SplitPredicate getSplitPredicate() {
return Predicates.SplitPredicate.of(
equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates),
rangePredicates.isEmpty() ? null : ExpressionUtils.and(rangePredicates),
residualPredicates.isEmpty() ? null : ExpressionUtils.and(residualPredicates));
}
private static boolean containOnlyColumnRef(Expression expression, boolean allowCast) {
if (expression instanceof SlotReference && ((SlotReference) expression).isColumnFromTable()) {
return true;
}
if (allowCast && expression instanceof Cast) {
return containOnlyColumnRef(((Cast) expression).child(), true);
}
if (expression instanceof Alias) {
return containOnlyColumnRef(((Alias) expression).child(), true);
}
return false;
}
}

63
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java
View File

@ -1,63 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
import java.util.List;
/**
* Relation mapping
* such as query pattern is a1 left join a2 left join b
* view pattern is a1 left join a2 left join b. the mapping will be
* [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}]
*/
public class RelationMapping extends Mapping {
private final BiMap<MappedRelation, MappedRelation> mappedRelationMap;
public RelationMapping(BiMap<MappedRelation, MappedRelation> mappedRelationMap) {
this.mappedRelationMap = mappedRelationMap;
}
public BiMap<MappedRelation, MappedRelation> getMappedRelationMap() {
return mappedRelationMap;
}
/**
* Generate mapping according to source and target relation
*/
public static List<RelationMapping> generate(List<CatalogRelation> source, List<CatalogRelation> target) {
Multimap<TableIf, CatalogRelation> queryTableRelationIdMap = ArrayListMultimap.create();
for (CatalogRelation relation : source) {
queryTableRelationIdMap.put(relation.getTable(), relation);
}
Multimap<TableIf, CatalogRelation> viewTableRelationIdMap = ArrayListMultimap.create();
for (CatalogRelation relation : target) {
viewTableRelationIdMap.put(relation.getTable(), relation);
}
// todo generate relation map
return ImmutableList.of();
}
}

49
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java
View File

@ -1,49 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
import com.google.common.collect.BiMap;
/**
* SlotMapping, this is open generated from relationMapping
*/
public class SlotMapping extends Mapping {
private final BiMap<MappedSlot, MappedSlot> relationSlotMap;
public SlotMapping(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
this.relationSlotMap = relationSlotMap;
}
public BiMap<MappedSlot, MappedSlot> getRelationSlotMap() {
return relationSlotMap;
}
public SlotMapping inverse() {
return SlotMapping.of(relationSlotMap.inverse());
}
public static SlotMapping of(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
return new SlotMapping(relationSlotMap);
}
public static SlotMapping generate(RelationMapping relationMapping) {
// TODO implement
return SlotMapping.of(null);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -52,7 +52,7 @@ public class StructInfo {
// construct equivalenceClass according to equals predicates
this.equivalenceClass = new EquivalenceClass();
SplitPredicate splitPredicate = Predicates.splitPredicates(predicates.composedExpression());
for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicates())) {
for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicate())) {
EqualTo equalTo = (EqualTo) expression;
equivalenceClass.addEquivalenceClass(
(SlotReference) equalTo.getArguments().get(0),

48
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionIndexMapping.java Normal file
View File

@ -0,0 +1,48 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import java.util.List;
/**
* Expression and it's index mapping
*/
public class ExpressionIndexMapping extends Mapping {
private final Multimap<Expression, Integer> expressionIndexMapping;
public ExpressionIndexMapping(Multimap<Expression, Integer> expressionIndexMapping) {
this.expressionIndexMapping = expressionIndexMapping;
}
public Multimap<Expression, Integer> getExpressionIndexMapping() {
return expressionIndexMapping;
}
public static ExpressionIndexMapping generate(List<? extends Expression> expressions) {
Multimap<Expression, Integer> expressionIndexMapping = ArrayListMultimap.create();
for (int i = 0; i < expressions.size(); i++) {
expressionIndexMapping.put(expressions.get(i), i);
}
return new ExpressionIndexMapping(expressionIndexMapping);
}
}

61
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
View File

@ -15,18 +15,15 @@
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import java.util.List;
import java.util.Objects;
import javax.annotation.Nullable;
/**
* Mapping slot from query to view or inversely,
@ -38,6 +35,7 @@ public abstract class Mapping {
* The relation for mapping
*/
public static final class MappedRelation {
public final RelationId relationId;
public final CatalogRelation belongedRelation;
@ -46,7 +44,7 @@ public abstract class Mapping {
this.belongedRelation = belongedRelation;
}
public MappedRelation of(RelationId relationId, CatalogRelation belongedRelation) {
public static MappedRelation of(RelationId relationId, CatalogRelation belongedRelation) {
return new MappedRelation(relationId, belongedRelation);
}
@ -82,15 +80,31 @@ public abstract class Mapping {
public static final class MappedSlot {
public final ExprId exprId;
public final Slot slot;
@Nullable
public final CatalogRelation belongedRelation;
public MappedSlot(ExprId exprId, CatalogRelation belongedRelation) {
public MappedSlot(ExprId exprId,
Slot slot,
CatalogRelation belongedRelation) {
this.exprId = exprId;
this.slot = slot;
this.belongedRelation = belongedRelation;
}
public MappedSlot of(ExprId exprId, CatalogRelation belongedRelation) {
return new MappedSlot(exprId, belongedRelation);
public static MappedSlot of(ExprId exprId,
Slot slot,
CatalogRelation belongedRelation) {
return new MappedSlot(exprId, slot, belongedRelation);
}
public static MappedSlot of(Slot slot,
CatalogRelation belongedRelation) {
return new MappedSlot(slot.getExprId(), slot, belongedRelation);
}
public static MappedSlot of(Slot slot) {
return new MappedSlot(slot.getExprId(), slot, null);
}
public ExprId getExprId() {
@ -101,6 +115,10 @@ public abstract class Mapping {
return belongedRelation;
}
public Slot getSlot() {
return slot;
}
@Override
public boolean equals(Object o) {
if (this == o) {
@ -118,27 +136,4 @@ public abstract class Mapping {
return Objects.hash(exprId);
}
}
/**
* Expression and it's index mapping
*/
public static class ExpressionIndexMapping extends Mapping {
private final Multimap<Expression, Integer> expressionIndexMapping;
public ExpressionIndexMapping(Multimap<Expression, Integer> expressionIndexMapping) {
this.expressionIndexMapping = expressionIndexMapping;
}
public Multimap<Expression, Integer> getExpressionIndexMapping() {
return expressionIndexMapping;
}
public static ExpressionIndexMapping generate(List<? extends Expression> expressions) {
Multimap<Expression, Integer> expressionIndexMapping = ArrayListMultimap.create();
for (int i = 0; i < expressions.size(); i++) {
expressionIndexMapping.put(expressions.get(i), i);
}
return new ExpressionIndexMapping(expressionIndexMapping);
}
}
}

113
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java Normal file
View File

@ -0,0 +1,113 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableBiMap.Builder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* Relation mapping
* such as query pattern is a1 left join a2 left join b
* view pattern is a1 left join a2 left join b. the mapping will be
* [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}]
*/
public class RelationMapping extends Mapping {
private final ImmutableBiMap<MappedRelation, MappedRelation> mappedRelationMap;
public RelationMapping(ImmutableBiMap<MappedRelation, MappedRelation> mappedRelationMap) {
this.mappedRelationMap = mappedRelationMap;
}
public BiMap<MappedRelation, MappedRelation> getMappedRelationMap() {
return mappedRelationMap;
}
public static RelationMapping of(ImmutableBiMap<MappedRelation, MappedRelation> mappedRelationMap) {
return new RelationMapping(mappedRelationMap);
}
/**
* Generate mapping according to source and target relation
*/
public static List<RelationMapping> generate(List<CatalogRelation> sources, List<CatalogRelation> targets) {
// Construct tmp map, key is the table qualifier, value is the corresponding catalog relations
LinkedListMultimap<Long, MappedRelation> sourceTableRelationIdMap = LinkedListMultimap.create();
for (CatalogRelation relation : sources) {
sourceTableRelationIdMap.put(getTableQualifier(relation.getTable()),
MappedRelation.of(relation.getRelationId(), relation));
}
LinkedListMultimap<Long, MappedRelation> targetTableRelationIdMap = LinkedListMultimap.create();
for (CatalogRelation relation : targets) {
targetTableRelationIdMap.put(getTableQualifier(relation.getTable()),
MappedRelation.of(relation.getRelationId(), relation));
}
Set<Long> sourceTableKeySet = sourceTableRelationIdMap.keySet();
List<List<Pair<MappedRelation, MappedRelation>>> mappedRelations = new ArrayList<>();
for (Long sourceTableQualifier : sourceTableKeySet) {
List<MappedRelation> sourceMappedRelations = sourceTableRelationIdMap.get(sourceTableQualifier);
List<MappedRelation> targetMappedRelations = targetTableRelationIdMap.get(sourceTableQualifier);
if (targetMappedRelations.isEmpty()) {
continue;
}
// if source and target relation appear once, just map them
if (targetMappedRelations.size() == 1 && sourceMappedRelations.size() == 1) {
mappedRelations.add(ImmutableList.of(Pair.of(sourceMappedRelations.get(0),
targetMappedRelations.get(0))));
continue;
}
// relation appear more than once, should cartesian them
ImmutableList<Pair<MappedRelation, MappedRelation>> relationMapping = Lists.cartesianProduct(
sourceTableRelationIdMap.get(sourceTableQualifier), targetMappedRelations)
.stream()
.map(listPair -> Pair.of(listPair.get(0), listPair.get(1)))
.collect(ImmutableList.toImmutableList());
mappedRelations.add(relationMapping);
}
int mappedRelationCount = mappedRelations.size();
return Lists.cartesianProduct(mappedRelations).stream()
.map(mappedRelationList -> {
Builder<MappedRelation, MappedRelation> mapBuilder = ImmutableBiMap.builder();
for (int relationIndex = 0; relationIndex < mappedRelationCount; relationIndex++) {
mapBuilder.put(mappedRelationList.get(relationIndex).key(),
mappedRelationList.get(relationIndex).value());
}
return RelationMapping.of(mapBuilder.build());
})
.collect(ImmutableList.toImmutableList());
}
private static Long getTableQualifier(TableIf tableIf) {
return tableIf.getId();
}
}

84
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java Normal file
View File

@ -0,0 +1,84 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
* SlotMapping, this is open generated from relationMapping
*/
public class SlotMapping extends Mapping {
private final BiMap<MappedSlot, MappedSlot> slotMapping;
public SlotMapping(BiMap<MappedSlot, MappedSlot> slotMapping) {
this.slotMapping = slotMapping;
}
public BiMap<MappedSlot, MappedSlot> getSlotBiMap() {
return slotMapping;
}
public SlotMapping inverse() {
return slotMapping == null
? SlotMapping.of(HashBiMap.create()) : SlotMapping.of(slotMapping.inverse());
}
public static SlotMapping of(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
return new SlotMapping(relationSlotMap);
}
/**
* SlotMapping, this is open generated from relationMapping
*/
@Nullable
public static SlotMapping generate(RelationMapping relationMapping) {
BiMap<MappedSlot, MappedSlot> relationSlotMap = HashBiMap.create();
BiMap<MappedRelation, MappedRelation> mappedRelationMap = relationMapping.getMappedRelationMap();
for (Map.Entry<MappedRelation, MappedRelation> mappedRelationEntry : mappedRelationMap.entrySet()) {
Map<String, Slot> targetNameSlotMap =
mappedRelationEntry.getValue().getBelongedRelation().getOutput().stream()
.collect(Collectors.toMap(Slot::getName, slot -> slot));
for (Slot sourceSlot : mappedRelationEntry.getKey().getBelongedRelation().getOutput()) {
Slot targetSlot = targetNameSlotMap.get(sourceSlot.getName());
// source slot can not map from target, bail out
if (targetSlot == null) {
return null;
}
relationSlotMap.put(MappedSlot.of(sourceSlot, mappedRelationEntry.getKey().getBelongedRelation()),
MappedSlot.of(targetSlot, mappedRelationEntry.getValue().getBelongedRelation()));
}
}
return SlotMapping.of(relationSlotMap);
}
/**
* SlotMapping, getSlotMap
*/
public Map<? extends Expression, ? extends Expression> getSlotMap() {
return (Map) this.getSlotBiMap();
}
}

49
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
View File

@ -17,16 +17,9 @@
package org.apache.doris.nereids.trees.expressions.visitor;
import org.apache.doris.nereids.rules.exploration.mv.Predicates;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.util.ExpressionUtils;
import java.util.ArrayList;
import java.util.List;
/**
* This is the factory for all ExpressionVisitor instance.
@ -61,46 +54,4 @@ public class ExpressionVisitors {
return true;
}
}
/**
* Split the expression to equal, range and residual predicate.
* Should instance when used.
*/
public static class PredicatesSpliter extends DefaultExpressionVisitor<Void, Void> {
private List<Expression> equalPredicates = new ArrayList<>();
private List<Expression> rangePredicates = new ArrayList<>();
private List<Expression> residualPredicates = new ArrayList<>();
private final Expression target;
public PredicatesSpliter(Expression target) {
this.target = target;
}
@Override
public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Void context) {
// TODO Smallest implement, complete later
if (comparisonPredicate instanceof EqualTo) {
Expression leftArgument = comparisonPredicate.getArgument(0);
Expression rightArgument = comparisonPredicate.getArgument(1);
if (leftArgument.isSlot() && rightArgument.isSlot()) {
equalPredicates.add(comparisonPredicate);
} else {
rangePredicates.add(comparisonPredicate);
}
}
return super.visit(comparisonPredicate, context);
}
public Expression getTarget() {
return target;
}
public Predicates.SplitPredicate getSplitPredicate() {
return Predicates.SplitPredicate.of(
equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates),
rangePredicates.isEmpty() ? null : ExpressionUtils.and(rangePredicates),
residualPredicates.isEmpty() ? null : ExpressionUtils.and(residualPredicates));
}
}
}

160
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java Normal file
View File

@ -0,0 +1,160 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.visitor;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer.ExpressionReplaceContext;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* ExpressionLineageReplacer
* Get from rewrite plan and can also get from plan struct info, if from plan struct info it depends on
* the nodes from graph.
*/
public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, ExpressionReplaceContext> {
public static final ExpressionLineageReplacer INSTANCE = new ExpressionLineageReplacer();
@Override
public Expression visit(Plan plan, ExpressionReplaceContext context) {
List<? extends Expression> expressions = plan.getExpressions();
Map<ExprId, Expression> targetExpressionMap = context.getExprIdExpressionMap();
// Filter the namedExpression used by target and collect the namedExpression
expressions.stream()
.filter(expression -> expression instanceof NamedExpression
&& targetExpressionMap.containsKey(((NamedExpression) expression).getExprId()))
.forEach(expression -> expression.accept(NamedExpressionCollector.INSTANCE, context));
return super.visit(plan, context);
}
/**
* Replace the expression with lineage according the exprIdExpressionMap
*/
public static class ExpressionReplacer extends DefaultExpressionRewriter<Map<ExprId, Expression>> {
public static final ExpressionReplacer INSTANCE = new ExpressionReplacer();
@Override
public Expression visitNamedExpression(NamedExpression namedExpression,
Map<ExprId, Expression> exprIdExpressionMap) {
if (exprIdExpressionMap.containsKey(namedExpression.getExprId())) {
return super.visit(exprIdExpressionMap.get(namedExpression.getExprId()), exprIdExpressionMap);
}
return super.visitNamedExpression(namedExpression, exprIdExpressionMap);
}
}
/**
* The Collector for target named expressions
* TODO Collect named expression by targetTypes, tableIdentifiers
*/
public static class NamedExpressionCollector
extends DefaultExpressionVisitor<Void, ExpressionReplaceContext> {
public static final NamedExpressionCollector INSTANCE = new NamedExpressionCollector();
@Override
public Void visitSlotReference(SlotReference slotReference, ExpressionReplaceContext context) {
context.getExprIdExpressionMap().put(slotReference.getExprId(), slotReference);
return super.visitSlotReference(slotReference, context);
}
@Override
public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot, ExpressionReplaceContext context) {
context.getExprIdExpressionMap().put(arrayItemSlot.getExprId(), arrayItemSlot);
return super.visitArrayItemSlot(arrayItemSlot, context);
}
@Override
public Void visitAlias(Alias alias, ExpressionReplaceContext context) {
// remove the alias
if (context.getExprIdExpressionMap().containsKey(alias.getExprId())) {
context.getExprIdExpressionMap().put(alias.getExprId(), alias.child());
}
return super.visitAlias(alias, context);
}
}
/**
* The context for replacing the expression with lineage
*/
public static class ExpressionReplaceContext {
private final List<Expression> targetExpressions;
private final Set<TableType> targetTypes;
private final Set<String> tableIdentifiers;
private Map<ExprId, Expression> exprIdExpressionMap;
private List<Expression> replacedExpressions;
/**ExpressionReplaceContext*/
public ExpressionReplaceContext(List<Expression> targetExpressions,
Set<TableType> targetTypes,
Set<String> tableIdentifiers) {
this.targetExpressions = targetExpressions;
this.targetTypes = targetTypes;
this.tableIdentifiers = tableIdentifiers;
// collect only named expressions and replace them with linage identifier later
this.exprIdExpressionMap = targetExpressions.stream()
.map(each -> each.collectToList(NamedExpression.class::isInstance))
.flatMap(Collection::stream)
.map(NamedExpression.class::cast)
.collect(Collectors.toMap(NamedExpression::getExprId, expr -> expr));
}
public List<Expression> getTargetExpressions() {
return targetExpressions;
}
public Set<TableType> getTargetTypes() {
return targetTypes;
}
public Set<String> getTableIdentifiers() {
return tableIdentifiers;
}
public Map<ExprId, Expression> getExprIdExpressionMap() {
return exprIdExpressionMap;
}
/**
* getReplacedExpressions
*/
public List<Expression> getReplacedExpressions() {
if (this.replacedExpressions == null) {
this.replacedExpressions = targetExpressions.stream()
.map(original -> original.accept(ExpressionReplacer.INSTANCE, getExprIdExpressionMap()))
.collect(Collectors.toList());
}
return this.replacedExpressions;
}
}
}

30
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -19,7 +19,6 @@ package org.apache.doris.nereids.util;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.exploration.mv.SlotMapping;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
import org.apache.doris.nereids.trees.TreeNode;
@ -48,6 +47,7 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
@ -206,28 +206,28 @@ public class ExpressionUtils {
}
/**
* Replace the slot in expression with the lineage identifier from specified
* baseTable sets or target table types.
* <p>
* For example as following:
* Replace the slot in expressions with the lineage identifier from specifiedbaseTable sets or target table types
* example as following:
* select a + 10 as a1, d from (
* select b - 5 as a, d from table
* );
* after shuttle a1, d in select will be b - 5 + 10, d
* op expression before is: a + 10 as a1, d. after is: b - 5 + 10, d
* todo to get from plan struct info
*/
public static List<? extends Expression> shuttleExpressionWithLineage(List<? extends Expression> expression,
public static List<? extends Expression> shuttleExpressionWithLineage(List<? extends Expression> expressions,
Plan plan,
Set<TableType> targetTypes,
Set<String> tableIdentifiers) {
return ImmutableList.of();
}
/**
* Replace the slot in expressions according to the slotMapping
* if any slot cannot be mapped then return null
*/
public static List<? extends Expression> permute(List<? extends Expression> expressions, SlotMapping slotMapping) {
return ImmutableList.of();
ExpressionLineageReplacer.ExpressionReplaceContext replaceContext =
new ExpressionLineageReplacer.ExpressionReplaceContext(
expressions.stream().map(NamedExpression.class::cast).collect(Collectors.toList()),
targetTypes,
tableIdentifiers);
plan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
// Replace expressions by expression map
return replaceContext.getReplacedExpressions();
}
/**