database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feat](nereids) support nullSafeEqual estimation (#31616)

Browse Source
This commit is contained in:
minghong
2024-03-01 19:04:06 +08:00
committed by yiguolei
parent 38cb17567a
commit 874f4c693b
3 changed files with 52 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
View File

@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.Like;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@ -239,7 +240,8 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
Expression left = cp.left();
Expression right = cp.right();
if (cp instanceof EqualPredicate) {
return estimateColumnEqualToColumn(left, statsForLeft, right, statsForRight, context);
return estimateColumnEqualToColumn(left, statsForLeft, right, statsForRight,
cp instanceof NullSafeEqual, context);
}
if (cp instanceof GreaterThan || cp instanceof GreaterThanEqual) {
return estimateColumnLessThanColumn(right, statsForRight, left, statsForLeft, context);
@ -488,7 +490,7 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
}
private Statistics estimateColumnEqualToColumn(Expression leftExpr, ColumnStatistic leftStats,
Expression rightExpr, ColumnStatistic rightStats, EstimationContext context) {
Expression rightExpr, ColumnStatistic rightStats, boolean keepNull, EstimationContext context) {
StatisticRange leftRange = StatisticRange.from(leftStats, leftExpr.getDataType());
StatisticRange rightRange = StatisticRange.from(rightStats, rightExpr.getDataType());
StatisticRange leftIntersectRight = leftRange.intersect(rightRange);
@ -497,11 +499,16 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
intersectBuilder.setNdv(intersect.getDistinctValues());
intersectBuilder.setMinValue(intersect.getLow());
intersectBuilder.setMaxValue(intersect.getHigh());
intersectBuilder.setNumNulls(0);
double numNull = 0;
if (keepNull) {
numNull = Math.min(leftStats.numNulls, rightStats.numNulls);
}
intersectBuilder.setNumNulls(numNull);
double sel = 1 / StatsMathUtil.nonZeroDivisor(Math.max(leftStats.ndv, rightStats.ndv));
Statistics updatedStatistics = context.statistics.withSel(sel);
Statistics updatedStatistics = context.statistics.withSel(sel, numNull);
updatedStatistics.addColumnStats(leftExpr, intersectBuilder.build());
updatedStatistics.addColumnStats(rightExpr, intersectBuilder.build());
context.addKeyIfSlot(leftExpr);
context.addKeyIfSlot(rightExpr);
return updatedStatistics;

6
fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
View File

@ -95,11 +95,15 @@ public class Statistics {
}
public Statistics withSel(double sel) {
return withSel(sel, 0);
}
public Statistics withSel(double sel, double numNull) {
sel = StatsMathUtil.minNonNaN(sel, 1);
if (Double.isNaN(rowCount)) {
return this;
}
double newCount = rowCount * sel;
double newCount = rowCount * sel + numNull;
return new Statistics(newCount, widthInJoinCluster, new HashMap<>(expressionToColumnStats));
}