database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](mtmv) Fix result wrong when query rewrite by mv if query contains null_unsafe equals expression (#39629) (#40041)

Browse Source 
## Proposed changes

commitId: 5d4ad028
pr: https://github.com/apache/doris/pull/39629
This commit is contained in:
seawinde
2024-08-29 00:31:36 +08:00
committed by GitHub
parent c3c6a4fc4c
commit c5d9e178be
5 changed files with 142 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java
View File

@ -21,12 +21,9 @@ import org.apache.doris.nereids.trees.expressions.SlotReference;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* EquivalenceClass, this is used for equality propagation when predicate compensation
@ -40,14 +37,19 @@ public class EquivalenceClass {
* a: [a, b],
* b: [a, b]
* }
* or column a = a,
* this would be
* {
* a: [a, a]
* }
*/
private Map<SlotReference, Set<SlotReference>> equivalenceSlotMap = new LinkedHashMap<>();
private List<Set<SlotReference>> equivalenceSlotList;
private Map<SlotReference, List<SlotReference>> equivalenceSlotMap = new LinkedHashMap<>();
private List<List<SlotReference>> equivalenceSlotList;
public EquivalenceClass() {
}
public EquivalenceClass(Map<SlotReference, Set<SlotReference>> equivalenceSlotMap) {
public EquivalenceClass(Map<SlotReference, List<SlotReference>> equivalenceSlotMap) {
this.equivalenceSlotMap = equivalenceSlotMap;
}
@ -56,13 +58,13 @@ public class EquivalenceClass {
*/
public void addEquivalenceClass(SlotReference leftSlot, SlotReference rightSlot) {
Set<SlotReference> leftSlotSet = equivalenceSlotMap.get(leftSlot);
Set<SlotReference> rightSlotSet = equivalenceSlotMap.get(rightSlot);
List<SlotReference> leftSlotSet = equivalenceSlotMap.get(leftSlot);
List<SlotReference> rightSlotSet = equivalenceSlotMap.get(rightSlot);
if (leftSlotSet != null && rightSlotSet != null) {
// Both present, we need to merge
if (leftSlotSet.size() < rightSlotSet.size()) {
// We swap them to merge
Set<SlotReference> tmp = rightSlotSet;
List<SlotReference> tmp = rightSlotSet;
rightSlotSet = leftSlotSet;
leftSlotSet = tmp;
}
@ -80,7 +82,7 @@ public class EquivalenceClass {
equivalenceSlotMap.put(leftSlot, rightSlotSet);
} else {
// None are present, add to same equivalence class
Set<SlotReference> equivalenceClass = new LinkedHashSet<>();
List<SlotReference> equivalenceClass = new ArrayList<>();
equivalenceClass.add(leftSlot);
equivalenceClass.add(rightSlot);
equivalenceSlotMap.put(leftSlot, equivalenceClass);
@ -88,7 +90,7 @@ public class EquivalenceClass {
}
}
public Map<SlotReference, Set<SlotReference>> getEquivalenceSlotMap() {
public Map<SlotReference, List<SlotReference>> getEquivalenceSlotMap() {
return equivalenceSlotMap;
}
@ -101,15 +103,15 @@ public class EquivalenceClass {
*/
public EquivalenceClass permute(Map<SlotReference, SlotReference> mapping) {
Map<SlotReference, Set<SlotReference>> permutedEquivalenceSlotMap = new HashMap<>();
for (Map.Entry<SlotReference, Set<SlotReference>> slotReferenceSetEntry : equivalenceSlotMap.entrySet()) {
Map<SlotReference, List<SlotReference>> permutedEquivalenceSlotMap = new HashMap<>();
for (Map.Entry<SlotReference, List<SlotReference>> slotReferenceSetEntry : equivalenceSlotMap.entrySet()) {
SlotReference mappedSlotReferenceKey = mapping.get(slotReferenceSetEntry.getKey());
if (mappedSlotReferenceKey == null) {
// can not permute then need to return null
return null;
}
Set<SlotReference> equivalenceValueSet = slotReferenceSetEntry.getValue();
final Set<SlotReference> mappedSlotReferenceSet = new HashSet<>();
List<SlotReference> equivalenceValueSet = slotReferenceSetEntry.getValue();
final List<SlotReference> mappedSlotReferenceSet = new ArrayList<>();
for (SlotReference target : equivalenceValueSet) {
SlotReference mappedSlotReferenceValue = mapping.get(target);
if (mappedSlotReferenceValue == null) {
@ -123,15 +125,14 @@ public class EquivalenceClass {
}
/**
* Return the list of equivalence set, remove duplicate
* Return the list of equivalence list, remove duplicate
*/
public List<Set<SlotReference>> getEquivalenceSetList() {
public List<List<SlotReference>> getEquivalenceSetList() {
if (equivalenceSlotList != null) {
return equivalenceSlotList;
}
List<Set<SlotReference>> equivalenceSets = new ArrayList<>();
Set<Set<SlotReference>> visited = new HashSet<>();
List<List<SlotReference>> equivalenceSets = new ArrayList<>();
List<List<SlotReference>> visited = new ArrayList<>();
equivalenceSlotMap.values().forEach(slotSet -> {
if (!visited.contains(slotSet)) {
equivalenceSets.add(slotSet);

13
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View File

@ -18,7 +18,7 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.rules.expression.ExpressionNormalization;
import org.apache.doris.nereids.rules.expression.ExpressionOptimization;
@ -33,6 +33,7 @@ import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
@ -98,15 +99,15 @@ public class Predicates {
if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) {
return null;
}
EquivalenceClassSetMapping queryToViewEquivalenceMapping =
EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased);
EquivalenceClassMapping queryToViewEquivalenceMapping =
EquivalenceClassMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased);
// can not map all target equivalence class, can not compensate
if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size()
< viewEquivalenceClass.getEquivalenceSetList().size()) {
return null;
}
// do equal compensate
Set<Set<SlotReference>> mappedQueryEquivalenceSet =
Set<List<SlotReference>> mappedQueryEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet();
queryEquivalenceClass.getEquivalenceSetList().forEach(
queryEquivalenceSet -> {
@ -120,9 +121,9 @@ public class Predicates {
}
} else {
// compensate the equivalence both in query and view, but query has more equivalence
Set<SlotReference> viewEquivalenceSet =
List<SlotReference> viewEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet);
Set<SlotReference> copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet);
List<SlotReference> copiedQueryEquivalenceSet = new ArrayList<>(queryEquivalenceSet);
copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet);
SlotReference first = viewEquivalenceSet.iterator().next();
for (SlotReference slotReference : copiedQueryEquivalenceSet) {

33
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassMapping.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.exploration.mv.EquivalenceClass;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -30,39 +31,41 @@ import java.util.Set;
* This will extract the equivalence class set in EquivalenceClass and mapping set in
* two different EquivalenceClass.
*/
public class EquivalenceClassSetMapping extends Mapping {
public class EquivalenceClassMapping extends Mapping {
private final Map<Set<SlotReference>, Set<SlotReference>> equivalenceClassSetMap;
private final Map<List<SlotReference>, List<SlotReference>> equivalenceClassSetMap;
public EquivalenceClassSetMapping(Map<Set<SlotReference>,
Set<SlotReference>> equivalenceClassSetMap) {
public EquivalenceClassMapping(Map<List<SlotReference>,
List<SlotReference>> equivalenceClassSetMap) {
this.equivalenceClassSetMap = equivalenceClassSetMap;
}
public static EquivalenceClassSetMapping of(Map<Set<SlotReference>, Set<SlotReference>> equivalenceClassSetMap) {
return new EquivalenceClassSetMapping(equivalenceClassSetMap);
public static EquivalenceClassMapping of(Map<List<SlotReference>, List<SlotReference>> equivalenceClassSetMap) {
return new EquivalenceClassMapping(equivalenceClassSetMap);
}
/**
* Generate source equivalence set map to target equivalence set
*/
public static EquivalenceClassSetMapping generate(EquivalenceClass source, EquivalenceClass target) {
public static EquivalenceClassMapping generate(EquivalenceClass source, EquivalenceClass target) {
Map<Set<SlotReference>, Set<SlotReference>> equivalenceClassSetMap = new HashMap<>();
List<Set<SlotReference>> sourceSets = source.getEquivalenceSetList();
List<Set<SlotReference>> targetSets = target.getEquivalenceSetList();
Map<List<SlotReference>, List<SlotReference>> equivalenceClassSetMap = new HashMap<>();
List<List<SlotReference>> sourceSets = source.getEquivalenceSetList();
List<List<SlotReference>> targetSets = target.getEquivalenceSetList();
for (Set<SlotReference> sourceSet : sourceSets) {
for (Set<SlotReference> targetSet : targetSets) {
for (List<SlotReference> sourceList : sourceSets) {
Set<SlotReference> sourceSet = new HashSet<>(sourceList);
for (List<SlotReference> targetList : targetSets) {
Set<SlotReference> targetSet = new HashSet<>(targetList);
if (sourceSet.containsAll(targetSet)) {
equivalenceClassSetMap.put(sourceSet, targetSet);
equivalenceClassSetMap.put(sourceList, targetList);
}
}
}
return EquivalenceClassSetMapping.of(equivalenceClassSetMap);
return EquivalenceClassMapping.of(equivalenceClassSetMap);
}
public Map<Set<SlotReference>, Set<SlotReference>> getEquivalenceClassSetMap() {
public Map<List<SlotReference>, List<SlotReference>> getEquivalenceClassSetMap() {
return equivalenceClassSetMap;
}
}