[fix](nereids) fix some planner bugs (#21533)
1. allow cast boolean as date like type in nereids, the result is null 2. PruneOlapScanTablet rule can prune tablet even if a mv index is selected. 3. constant conjunct should not be pushed through agg node in old planner
This commit is contained in:
starocean999
@ -40,6 +40,7 @@ import org.apache.doris.common.ErrorReport;
|
||||
import org.apache.doris.common.IdGenerator;
|
||||
import org.apache.doris.common.Pair;
|
||||
import org.apache.doris.common.util.TimeUtils;
|
||||
import org.apache.doris.planner.AggregationNode;
|
||||
import org.apache.doris.planner.PlanNode;
|
||||
import org.apache.doris.planner.RuntimeFilter;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
@ -2416,10 +2417,11 @@ public class Analyzer {
|
||||
* Wrapper around getUnassignedConjuncts(List<TupleId> tupleIds).
|
||||
*/
|
||||
public List<Expr> getUnassignedConjuncts(PlanNode node) {
|
||||
// constant conjuncts should be push down to all leaf node.
|
||||
// constant conjuncts should be push down to all leaf node except agg node.
|
||||
// (see getPredicatesBoundedByGroupbysSourceExpr method)
|
||||
// so we need remove constant conjuncts when expr is not a leaf node.
|
||||
List<Expr> unassigned = getUnassignedConjuncts(node.getTblRefIds());
|
||||
if (!node.getChildren().isEmpty()) {
|
||||
if (!node.getChildren().isEmpty() && !(node instanceof AggregationNode)) {
|
||||
unassigned = unassigned.stream()
|
||||
.filter(e -> !e.isConstant()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@ -59,11 +59,10 @@ public class CheckCast extends AbstractExpressionRewriteRule {
|
||||
|
||||
/**
|
||||
* forbid this original and target type
|
||||
* 1. boolean to date, datev2, datetime, datetimev2
|
||||
* 2. original type is object type
|
||||
* 3. target type is object type
|
||||
* 4. original type is same with target type
|
||||
* 5. target type is null type
|
||||
* 1. original type is object type
|
||||
* 2. target type is object type
|
||||
* 3. original type is same with target type
|
||||
* 4. target type is null type
|
||||
*/
|
||||
private boolean checkPrimitiveType(DataType originalType, DataType targetType) {
|
||||
if (!originalType.isPrimitive() || !targetType.isPrimitive()) {
|
||||
@ -72,9 +71,6 @@ public class CheckCast extends AbstractExpressionRewriteRule {
|
||||
if (originalType.equals(targetType)) {
|
||||
return false;
|
||||
}
|
||||
if (originalType.isBooleanType() && targetType.isDateLikeType()) {
|
||||
return false;
|
||||
}
|
||||
if (originalType.isNullType()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -58,8 +58,11 @@ public class PruneOlapScanTablet extends OneRewriteRuleFactory {
|
||||
for (Long id : olapScan.getSelectedPartitionIds()) {
|
||||
Partition partition = table.getPartition(id);
|
||||
MaterializedIndex index = partition.getIndex(olapScan.getSelectedIndexId());
|
||||
selectedTabletIdsBuilder.addAll(getSelectedTabletIds(filter.getConjuncts(),
|
||||
index, partition.getDistributionInfo()));
|
||||
selectedTabletIdsBuilder
|
||||
.addAll(getSelectedTabletIds(filter.getConjuncts(), index,
|
||||
olapScan.getSelectedIndexId() == olapScan.getTable()
|
||||
.getBaseIndexId(),
|
||||
partition.getDistributionInfo()));
|
||||
}
|
||||
List<Long> selectedTabletIds = selectedTabletIdsBuilder.build();
|
||||
if (new HashSet(selectedTabletIds).equals(new HashSet(olapScan.getSelectedTabletIds()))) {
|
||||
@ -70,7 +73,7 @@ public class PruneOlapScanTablet extends OneRewriteRuleFactory {
|
||||
}
|
||||
|
||||
private Collection<Long> getSelectedTabletIds(Set<Expression> expressions,
|
||||
MaterializedIndex index, DistributionInfo info) {
|
||||
MaterializedIndex index, boolean isBaseIndexSelected, DistributionInfo info) {
|
||||
if (info.getType() != DistributionInfoType.HASH) {
|
||||
return index.getTabletIdsInOrder();
|
||||
}
|
||||
@ -81,7 +84,8 @@ public class PruneOlapScanTablet extends OneRewriteRuleFactory {
|
||||
return new HashDistributionPruner(index.getTabletIdsInOrder(),
|
||||
hashInfo.getDistributionColumns(),
|
||||
filterMap,
|
||||
hashInfo.getBucketNum()
|
||||
hashInfo.getBucketNum(),
|
||||
isBaseIndexSelected
|
||||
).prune();
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
|
||||
package org.apache.doris.planner;
|
||||
|
||||
import org.apache.doris.analysis.CreateMaterializedViewStmt;
|
||||
import org.apache.doris.analysis.InPredicate;
|
||||
import org.apache.doris.analysis.LiteralExpr;
|
||||
import org.apache.doris.analysis.SlotRef;
|
||||
@ -58,12 +59,15 @@ public class HashDistributionPruner implements DistributionPruner {
|
||||
private Map<String, PartitionColumnFilter> distributionColumnFilters;
|
||||
private int hashMod;
|
||||
|
||||
private boolean isBaseIndexSelected;
|
||||
|
||||
public HashDistributionPruner(List<Long> bucketsList, List<Column> columns,
|
||||
Map<String, PartitionColumnFilter> filters, int hashMod) {
|
||||
Map<String, PartitionColumnFilter> filters, int hashMod, boolean isBaseIndexSelected) {
|
||||
this.bucketsList = bucketsList;
|
||||
this.distributionColumns = columns;
|
||||
this.distributionColumnFilters = filters;
|
||||
this.hashMod = hashMod;
|
||||
this.isBaseIndexSelected = isBaseIndexSelected;
|
||||
}
|
||||
|
||||
// columnId: which column to compute
|
||||
@ -75,7 +79,11 @@ public class HashDistributionPruner implements DistributionPruner {
|
||||
return Lists.newArrayList(bucketsList.get((int) ((hashValue & 0xffffffff) % hashMod)));
|
||||
}
|
||||
Column keyColumn = distributionColumns.get(columnId);
|
||||
PartitionColumnFilter filter = distributionColumnFilters.get(keyColumn.getName());
|
||||
String columnName = isBaseIndexSelected ? keyColumn.getName()
|
||||
: org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule
|
||||
.normalizeName(
|
||||
CreateMaterializedViewStmt.mvColumnBuilder(keyColumn.getName()));
|
||||
PartitionColumnFilter filter = distributionColumnFilters.get(columnName);
|
||||
if (null == filter) {
|
||||
// no filter in this column, no partition Key
|
||||
// return all subPartition
|
||||
|
||||
@ -692,7 +692,8 @@ public class OlapScanNode extends ScanNode {
|
||||
distributionPruner = new HashDistributionPruner(table.getTabletIdsInOrder(),
|
||||
info.getDistributionColumns(),
|
||||
columnFilters,
|
||||
info.getBucketNum());
|
||||
info.getBucketNum(),
|
||||
getSelectedIndexId() == olapTable.getBaseIndexId());
|
||||
return distributionPruner.prune();
|
||||
}
|
||||
case RANDOM: {
|
||||
|
||||
@ -91,7 +91,7 @@ public class HashDistributionPrunerTest {
|
||||
filters.put("channel", channelFilter);
|
||||
filters.put("shop_type", shopTypeFilter);
|
||||
|
||||
HashDistributionPruner pruner = new HashDistributionPruner(tabletIds, columns, filters, tabletIds.size());
|
||||
HashDistributionPruner pruner = new HashDistributionPruner(tabletIds, columns, filters, tabletIds.size(), true);
|
||||
|
||||
Collection<Long> results = pruner.prune();
|
||||
// 20 = 1 * 5 * 2 * 2 * 1 (element num of each filter)
|
||||
|
||||
@ -69,7 +69,8 @@ public class OlapScanNodeTest {
|
||||
partitions,
|
||||
columns,
|
||||
filterMap,
|
||||
3);
|
||||
3,
|
||||
true);
|
||||
|
||||
Collection<Long> ids = partitionPruner.prune();
|
||||
Assert.assertEquals(ids.size(), 1);
|
||||
@ -112,7 +113,8 @@ public class OlapScanNodeTest {
|
||||
partitions,
|
||||
columns,
|
||||
filterMap,
|
||||
3);
|
||||
3,
|
||||
true);
|
||||
|
||||
Collection<Long> ids = partitionPruner.prune();
|
||||
Assert.assertEquals(ids.size(), 3);
|
||||
|
||||
Reference in New Issue
Block a user