database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids) fix stats error when using dateTime type filter (#27571)

Browse Source 
Currently doris doesn't support datetime type filter stats estimation, but only for date type.
It will cause the filter using datetime type column with the same date and different time computing out a inaccurate selectivity and estimate a wrong row count, such as :

where o.book_time >= '2020-03-01 00:00:00.0' and o.book_time <= '2020-03-01 23:59:59.0';

This pr adds the datetime type(only support hh:mm:ss scale) filter estimation and improve the row count estimation for the above case.
This commit is contained in:
xzj7019
2023-11-25 11:04:51 +08:00
committed by GitHub
parent 1b8d7da078
commit d4f2db74f9
6 changed files with 147 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java
View File

@ -21,6 +21,10 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import java.time.DateTimeException;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
/**
* Datetime type in Nereids.
*/
@ -55,4 +59,21 @@ public class DateTimeType extends DateLikeType {
public int width() {
return WIDTH;
}
@Override
public double rangeLength(double high, double low) {
if (high == low) {
return 0;
}
if (Double.isInfinite(high) || Double.isInfinite(low)) {
return Double.POSITIVE_INFINITY;
}
try {
LocalDateTime to = toLocalDateTime(high);
LocalDateTime from = toLocalDateTime(low);
return ChronoUnit.SECONDS.between(from, to);
} catch (DateTimeException e) {
return Double.POSITIVE_INFINITY;
}
}
}

20
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
View File

@ -26,6 +26,9 @@ import org.apache.doris.nereids.types.coercion.IntegralType;
import com.google.common.base.Preconditions;
import java.time.DateTimeException;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Objects;
/**
@ -126,4 +129,21 @@ public class DateTimeV2Type extends DateLikeType {
public int getScale() {
return scale;
}
@Override
public double rangeLength(double high, double low) {
if (high == low) {
return 0;
}
if (Double.isInfinite(high) || Double.isInfinite(low)) {
return Double.POSITIVE_INFINITY;
}
try {
LocalDateTime to = toLocalDateTime(high);
LocalDateTime from = toLocalDateTime(low);
return ChronoUnit.SECONDS.between(from, to);
} catch (DateTimeException e) {
return Double.POSITIVE_INFINITY;
}
}
}

21
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java
View File

@ -21,6 +21,10 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import java.time.DateTimeException;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
/**
* Date type in Nereids.
*/
@ -50,5 +54,22 @@ public class DateType extends DateLikeType {
public int width() {
return WIDTH;
}
@Override
public double rangeLength(double high, double low) {
if (high == low) {
return 0;
}
if (Double.isInfinite(high) || Double.isInfinite(low)) {
return Double.POSITIVE_INFINITY;
}
try {
LocalDate to = toLocalDate(high);
LocalDate from = toLocalDate(low);
return ChronoUnit.DAYS.between(from, to);
} catch (DateTimeException e) {
return Double.POSITIVE_INFINITY;
}
}
}

21
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java
View File

@ -20,6 +20,10 @@ package org.apache.doris.nereids.types;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import java.time.DateTimeException;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
/**
* Date type in Nereids.
*/
@ -41,5 +45,22 @@ public class DateV2Type extends DateLikeType {
public int width() {
return WIDTH;
}
@Override
public double rangeLength(double high, double low) {
if (high == low) {
return 0;
}
if (Double.isInfinite(high) || Double.isInfinite(low)) {
return Double.POSITIVE_INFINITY;
}
try {
LocalDate to = toLocalDate(high);
LocalDate from = toLocalDate(low);
return ChronoUnit.DAYS.between(from, to);
} catch (DateTimeException e) {
return Double.POSITIVE_INFINITY;
}
}
}

33
fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java
View File

@ -27,15 +27,15 @@ import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.DateV2Type;
import java.time.DateTimeException;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
import java.time.LocalDateTime;
/**
* date like type.
*/
public abstract class DateLikeType extends PrimitiveType {
private LocalDate toLocalDate(double d) {
protected LocalDate toLocalDate(double d) {
// d = (year * 10000 + month * 100 + day) * 1000000L;
int date = (int) (d / 1000000);
int day = date % 100;
@ -44,21 +44,18 @@ public abstract class DateLikeType extends PrimitiveType {
return LocalDate.of(year, month, day);
}
@Override
public double rangeLength(double high, double low) {
if (high == low) {
return 0;
}
if (Double.isInfinite(high) || Double.isInfinite(low)) {
return Double.POSITIVE_INFINITY;
}
try {
LocalDate to = toLocalDate(high);
LocalDate from = toLocalDate(low);
return ChronoUnit.DAYS.between(from, to);
} catch (DateTimeException e) {
return Double.POSITIVE_INFINITY;
}
protected LocalDateTime toLocalDateTime(double d) {
// d = (year * 10000 + month * 100 + day) * 1000000L + time
// time = (hour * 10000 + minute * 100 + second);
int date = (int) (d / 1000000);
int day = date % 100;
int month = (date / 100) % 100;
int year = date / 10000;
int time = (int) (d % 1000000);
int second = time % 100;
int minute = (time / 100) % 100;
int hour = time / 10000;
return LocalDateTime.of(year, month, day, hour, minute, second);
}
/**