database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature] Support pre-aggregation for quantile type (#8234)

Browse Source 
Add a new column-type to speed up the approximation of quantiles.
1. The  new column-type is named `quantile_state` with fixed aggregation function `quantile_union`, which stores the intermediate results of pre-aggregated approximation calculations for quantiles.
2. support pre-aggregation of new column-type and quantile_state related functions.
This commit is contained in:
spaces-x
2022-03-24 09:11:34 +08:00
committed by GitHub
parent 36c85d2f06
commit bea9a7ba4f
67 changed files with 1498 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -236,7 +236,7 @@ parser code {:
// Total keywords of doris
terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL, KW_ALTER, KW_AND, KW_ANTI, KW_APPEND, KW_AS, KW_ASC, KW_AUTHORS, KW_ARRAY,
KW_BACKEND, KW_BACKUP, KW_BETWEEN, KW_BEGIN, KW_BIGINT, KW_BINLOG, KW_BITMAP, KW_BITMAP_UNION, KW_BLOB, KW_BOOLEAN, KW_BROKER, KW_BACKENDS, KW_BY, KW_BUILTIN,
KW_BACKEND, KW_BACKUP, KW_BETWEEN, KW_BEGIN, KW_BIGINT, KW_BINLOG, KW_BITMAP, KW_BITMAP_UNION, KW_QUANTILE_STATE, KW_QUANTILE_UNION, KW_BLOB, KW_BOOLEAN, KW_BROKER, KW_BACKENDS, KW_BY, KW_BUILTIN,
KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK, KW_CLUSTER, KW_CLUSTERS, KW_CLEAN,
KW_COLLATE, KW_COLLATION, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMMIT, KW_COMMITTED, KW_COMPACT,
KW_CONFIG, KW_CONNECTION, KW_CONNECTION_ID, KW_CONSISTENT, KW_CONVERT, KW_COUNT, KW_CREATE, KW_CREATION, KW_CROSS, KW_CUBE, KW_CURRENT, KW_CURRENT_USER,
@ -2100,6 +2100,12 @@ opt_agg_type ::=
{:
RESULT = AggregateType.BITMAP_UNION;
:}
| KW_QUANTILE_UNION
{:
RESULT = AggregateType.QUANTILE_UNION;
:}
;
opt_partition ::=
@ -4566,6 +4572,8 @@ type ::=
{: RESULT = Type.TIME; :}
| KW_BITMAP
{: RESULT = Type.BITMAP; :}
| KW_QUANTILE_STATE
{: RESULT = Type.QUANTILE_STATE; :}
| KW_STRING
{: RESULT = ScalarType.createStringType(); :}
| KW_TEXT
@ -5418,8 +5426,12 @@ keyword ::=
{: RESULT = id; :}
| KW_BITMAP:id
{: RESULT = id; :}
| KW_QUANTILE_STATE:id
{: RESULT = id; :}
| KW_BITMAP_UNION:id
{: RESULT = id; :}
| KW_QUANTILE_UNION:id
{: RESULT = id; :}
| KW_BLOB:id
{: RESULT = id; :}
| KW_BOOLEAN:id

21
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
View File

@ -364,8 +364,8 @@ public class CreateTableStmt extends DdlStmt {
&& keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
columnDefs.add(ColumnDef.newDeleteSignColumnDef(AggregateType.REPLACE));
}
boolean hasHll = false;
boolean hasBitmap = false;
boolean hasObjectStored = false;
String objectStoredColumn = "";
Set<String> columnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
for (ColumnDef columnDef : columnDefs) {
columnDef.analyze(engineName.equals("olap"));
@ -380,12 +380,9 @@ public class CreateTableStmt extends DdlStmt {
}
}
if (columnDef.getType().isHllType()) {
hasHll = true;
}
if (columnDef.getAggregateType() == AggregateType.BITMAP_UNION) {
hasBitmap = columnDef.getType().isBitmapType();
if (columnDef.getType().isObjectStored()) {
hasObjectStored = true;
objectStoredColumn = columnDef.getName();
}
if (!columnSet.add(columnDef.getName())) {
@ -393,12 +390,8 @@ public class CreateTableStmt extends DdlStmt {
}
}
if (hasHll && keysDesc.getKeysType() != KeysType.AGG_KEYS) {
throw new AnalysisException("HLL must be used in AGG_KEYS");
}
if (hasBitmap && keysDesc.getKeysType() != KeysType.AGG_KEYS) {
throw new AnalysisException("BITMAP_UNION must be used in AGG_KEYS");
if (hasObjectStored && keysDesc.getKeysType() != KeysType.AGG_KEYS) {
throw new AnalysisException("column:" + objectStoredColumn + " must be used in AGG_KEYS.");
}
if (engineName.equals("olap")) {

5
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -1261,6 +1261,11 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
if (targetType.getPrimitiveType() == PrimitiveType.BITMAP) {
throw new AnalysisException("bitmap column require the function return type is BITMAP");
}
// TODO(weixiang): why bitmap is so strict but hll is not strict, may be bitmap can be same to hll
// here `quantile_state` is also strict now. may be can be same to hll too.
if (targetType.getPrimitiveType() == PrimitiveType.QUANTILE_STATE) {
throw new AnalysisException("quantile_state column require the function return type is QUANTILE_STATE");
}
// TargetTable's hll column must be hll_hash's result
if (targetType.getPrimitiveType() == PrimitiveType.HLL) {
checkHllCompatibility();

19
fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View File

@ -532,6 +532,25 @@ public class FunctionCallExpr extends Expr {
return;
}
if (fnName.getFunction().equalsIgnoreCase(FunctionSet.QUANTILE_UNION)) {
if (children.size() != 1) {
throw new AnalysisException(fnName + "function could only have one child");
}
Type inputType = getChild(0).getType();
if (!inputType.isQuantileStateType()) {
throw new AnalysisException(fnName + " function's argument should be of QUANTILE_STATE type, but was" + inputType);
}
}
if (fnName.getFunction().equalsIgnoreCase(FunctionSet.TO_QUANTILE_STATE)) {
if (children.size() != 2) {
throw new AnalysisException(fnName + "function must have two children");
}
if (!getChild(1).isConstant()) {
throw new AnalysisException(fnName + "function's second argument should be constant");
}
}
if ((fnName.getFunction().equalsIgnoreCase("HLL_UNION_AGG")
|| fnName.getFunction().equalsIgnoreCase("HLL_CARDINALITY")
|| fnName.getFunction().equalsIgnoreCase("HLL_RAW_AGG"))

7
fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
View File

@ -419,11 +419,8 @@ public class InsertStmt extends DdlStmt {
}
// hll column mush in mentionedColumns
for (Column col : targetTable.getBaseSchema()) {
if (col.getType().isHllType() && !mentionedColumns.contains(col.getName())) {
throw new AnalysisException (" hll column " + col.getName() + " mush in insert into columns");
}
if (col.getType().isBitmapType() && !mentionedColumns.contains(col.getName())) {
throw new AnalysisException (" object column " + col.getName() + " mush in insert into columns");
if (col.getType().isObjectStored() && !mentionedColumns.contains(col.getName())) {
throw new AnalysisException (" object-stored column " + col.getName() + " mush in insert into columns");
}
}
}

8
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View File

@ -1132,12 +1132,8 @@ public class SelectStmt extends QueryStmt {
"GROUP BY clause?): " + orderByElements.get(i).getExpr().toSql());
}
if (sortInfo.getOrderingExprs().get(i).type.isHllType()) {
throw new AnalysisException("ORDER BY expression could not contain hll column.");
}
if (sortInfo.getOrderingExprs().get(i).type.isBitmapType()) {
throw new AnalysisException("ORDER BY expression could not contain bitmap column.");
if (sortInfo.getOrderingExprs().get(i).type.isObjectStored()) {
throw new AnalysisException("ORDER BY expression could not contain object-stored columnx.");
}
}
}

25
fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java
View File

@ -33,7 +33,9 @@ public enum AggregateType {
REPLACE_IF_NOT_NULL("REPLACE_IF_NOT_NULL"),
HLL_UNION("HLL_UNION"),
NONE("NONE"),
BITMAP_UNION("BITMAP_UNION");
BITMAP_UNION("BITMAP_UNION"),
QUANTILE_UNION("QUANTILE_UNION");
private static EnumMap<AggregateType, EnumSet<PrimitiveType>> compatibilityMap;
@ -84,13 +86,14 @@ public enum AggregateType {
compatibilityMap.put(MAX, EnumSet.copyOf(primitiveTypeList));
primitiveTypeList.clear();
// all types except bitmap and hll.
EnumSet<PrimitiveType> exc_bitmap_hll = EnumSet.allOf(PrimitiveType.class);
exc_bitmap_hll.remove(PrimitiveType.BITMAP);
exc_bitmap_hll.remove(PrimitiveType.HLL);
compatibilityMap.put(REPLACE, EnumSet.copyOf(exc_bitmap_hll));
// all types except object stored column type, such as bitmap hll quantile_state.
EnumSet<PrimitiveType> exc_object_stored = EnumSet.allOf(PrimitiveType.class);
exc_object_stored.remove(PrimitiveType.BITMAP);
exc_object_stored.remove(PrimitiveType.HLL);
exc_object_stored.remove(PrimitiveType.QUANTILE_STATE);
compatibilityMap.put(REPLACE, EnumSet.copyOf(exc_object_stored));
compatibilityMap.put(REPLACE_IF_NOT_NULL, EnumSet.copyOf(exc_bitmap_hll));
compatibilityMap.put(REPLACE_IF_NOT_NULL, EnumSet.copyOf(exc_object_stored));
primitiveTypeList.clear();
primitiveTypeList.add(PrimitiveType.HLL);
@ -100,7 +103,11 @@ public enum AggregateType {
primitiveTypeList.add(PrimitiveType.BITMAP);
compatibilityMap.put(BITMAP_UNION, EnumSet.copyOf(primitiveTypeList));
compatibilityMap.put(NONE, EnumSet.copyOf(exc_bitmap_hll));
primitiveTypeList.clear();
primitiveTypeList.add(PrimitiveType.QUANTILE_STATE);
compatibilityMap.put(QUANTILE_UNION, EnumSet.copyOf(primitiveTypeList));
compatibilityMap.put(NONE, EnumSet.copyOf(exc_object_stored));
}
private final String sqlName;
@ -153,6 +160,8 @@ public enum AggregateType {
return TAggregationType.HLL_UNION;
case BITMAP_UNION:
return TAggregationType.BITMAP_UNION;
case QUANTILE_UNION:
return TAggregationType.QUANTILE_UNION;
default:
return null;
}

2
fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
View File

@ -492,6 +492,7 @@ public class Function implements Writable {
case CHAR:
case HLL:
case BITMAP:
case QUANTILE_STATE:
case STRING:
return "string_val";
case DATE:
@ -530,6 +531,7 @@ public class Function implements Writable {
case CHAR:
case HLL:
case BITMAP:
case QUANTILE_STATE:
case STRING:
return "StringVal";
case DATE:

28
fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
View File

@ -760,6 +760,11 @@ public class FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
public static final String ORTHOGONAL_BITMAP_INTERSECT_COUNT = "orthogonal_bitmap_intersect_count";
public static final String ORTHOGONAL_BITMAP_UNION_COUNT = "orthogonal_bitmap_union_count";
public static final String QUANTILE_UNION = "quantile_union";
//TODO(weixiang): is quantile_percent can be replaced by approx_percentile?
public static final String QUANTILE_PERCENT = "quantile_percent";
public static final String TO_QUANTILE_STATE = "to_quantile_state";
private static final Map<Type, String> ORTHOGONAL_BITMAP_INTERSECT_INIT_SYMBOL =
ImmutableMap.<Type, String>builder()
.put(Type.TINYINT,
@ -1966,8 +1971,27 @@ public class FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
"",
"",
true, false, true, true));
//Percentile
//quantile_state
addBuiltin(AggregateFunction.createBuiltin(QUANTILE_UNION, Lists.newArrayList(Type.QUANTILE_STATE),
Type.QUANTILE_STATE,
Type.QUANTILE_STATE,
"_ZN5doris22QuantileStateFunctions19quantile_state_initEPN9doris_udf15FunctionContextEPNS1_9StringValE",
"_ZN5doris22QuantileStateFunctions14quantile_unionEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_",
"_ZN5doris22QuantileStateFunctions14quantile_unionEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_",
"_ZN5doris22QuantileStateFunctions24quantile_state_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE",
"_ZN5doris22QuantileStateFunctions24quantile_state_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE",
true, false, true));
addBuiltin(AggregateFunction.createBuiltin(QUANTILE_UNION, Lists.newArrayList(Type.QUANTILE_STATE),
Type.QUANTILE_STATE,
Type.QUANTILE_STATE,
"",
"",
"",
"",
"",
true, false, true, true));
//Percentile
addBuiltin(AggregateFunction.createBuiltin("percentile",
Lists.newArrayList(Type.BIGINT, Type.DOUBLE), Type.DOUBLE, Type.VARCHAR,
prefix + "15percentile_initEPN9doris_udf15FunctionContextEPNS1_9StringValE",

24
fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
View File

@ -49,11 +49,12 @@ public enum PrimitiveType {
VARCHAR("VARCHAR", 16, TPrimitiveType.VARCHAR),
DECIMALV2("DECIMALV2", 16, TPrimitiveType.DECIMALV2),
HLL("HLL", 16, TPrimitiveType.HLL),
TIME("TIME", 8, TPrimitiveType.TIME),
// we use OBJECT type represent BITMAP type in Backend
// these following types are stored as object binary in BE.
HLL("HLL", 16, TPrimitiveType.HLL),
BITMAP("BITMAP", 16, TPrimitiveType.OBJECT),
QUANTILE_STATE("QUANTILE_STATE", 16, TPrimitiveType.QUANTILE_STATE),
ARRAY("ARRAY", 24, TPrimitiveType.ARRAY),
MAP("MAP", 24, TPrimitiveType.MAP),
STRUCT("MAP", 24, TPrimitiveType.STRUCT),
@ -87,7 +88,7 @@ public enum PrimitiveType {
builder.put(NULL_TYPE, CHAR);
builder.put(NULL_TYPE, VARCHAR);
builder.put(NULL_TYPE, STRING);
builder.put(NULL_TYPE, BITMAP);
builder.put(NULL_TYPE, BITMAP); //TODO(weixiang):why null type can cast to bitmap?
builder.put(NULL_TYPE, TIME);
// Boolean
builder.put(BOOLEAN, BOOLEAN);
@ -249,6 +250,7 @@ public enum PrimitiveType {
builder.put(VARCHAR, STRING);
builder.put(VARCHAR, HLL);
builder.put(VARCHAR, BITMAP);
builder.put(VARCHAR, QUANTILE_STATE);
// Varchar
builder.put(STRING, BOOLEAN);
@ -266,6 +268,7 @@ public enum PrimitiveType {
builder.put(STRING, STRING);
builder.put(STRING, HLL);
builder.put(STRING, BITMAP);
builder.put(STRING, QUANTILE_STATE);
// DecimalV2
builder.put(DECIMALV2, BOOLEAN);
@ -289,6 +292,11 @@ public enum PrimitiveType {
builder.put(BITMAP, BITMAP);
builder.put(BITMAP, VARCHAR);
builder.put(BITMAP, STRING);
// QUANTILE_STATE
builder.put(QUANTILE_STATE, QUANTILE_STATE);
builder.put(QUANTILE_STATE, VARCHAR);
builder.put(QUANTILE_STATE, STRING);
//TIME
builder.put(TIME, TIME);
@ -340,6 +348,7 @@ public enum PrimitiveType {
supportedTypes.add(BITMAP);
supportedTypes.add(ARRAY);
supportedTypes.add(MAP);
supportedTypes.add(QUANTILE_STATE);
}
public static ArrayList<PrimitiveType> getIntegerTypes() {
@ -392,7 +401,8 @@ public enum PrimitiveType {
compatibilityMatrix[NULL_TYPE.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[NULL_TYPE.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[NULL_TYPE.ordinal()][TIME.ordinal()] = TIME;
compatibilityMatrix[NULL_TYPE.ordinal()][BITMAP.ordinal()] = BITMAP;
compatibilityMatrix[NULL_TYPE.ordinal()][BITMAP.ordinal()] = BITMAP; //TODO(weixiang): bitmap can be null?
compatibilityMatrix[NULL_TYPE.ordinal()][QUANTILE_STATE.ordinal()] = QUANTILE_STATE; //TODO(weixiang): QUANTILE_STATE can be null?
compatibilityMatrix[BOOLEAN.ordinal()][BOOLEAN.ordinal()] = BOOLEAN;
compatibilityMatrix[BOOLEAN.ordinal()][TINYINT.ordinal()] = TINYINT;
@ -533,6 +543,8 @@ public enum PrimitiveType {
compatibilityMatrix[BITMAP.ordinal()][BITMAP.ordinal()] = BITMAP;
compatibilityMatrix[TIME.ordinal()][TIME.ordinal()] = TIME;
compatibilityMatrix[QUANTILE_STATE.ordinal()][QUANTILE_STATE.ordinal()] = QUANTILE_STATE;
}
static {
@ -606,6 +618,8 @@ public enum PrimitiveType {
return HLL;
case OBJECT:
return BITMAP;
case QUANTILE_STATE:
return QUANTILE_STATE;
case ARRAY:
return ARRAY;
case MAP:

1
fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java
View File

@ -162,6 +162,7 @@ public class ScalarFunction extends Function {
case HLL:
case BITMAP:
case STRING:
case QUANTILE_STATE:
beFn += "_string_val";
break;
case DATE:

7
fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java
View File

@ -151,6 +151,8 @@ public class ScalarType extends Type {
return createHllType();
case BITMAP:
return BITMAP;
case QUANTILE_STATE:
return QUANTILE_STATE;
case DATE:
return DATE;
case DATETIME:
@ -202,6 +204,8 @@ public class ScalarType extends Type {
return createHllType();
case "BITMAP":
return BITMAP;
case "QUANTILE_STATE":
return QUANTILE_STATE;
case "DATE":
return DATE;
case "DATETIME":
@ -382,6 +386,7 @@ public class ScalarType extends Type {
case DATETIME:
case HLL:
case BITMAP:
case QUANTILE_STATE:
stringBuilder.append(type.toString().toLowerCase());
break;
case STRING:
@ -756,6 +761,8 @@ public class ScalarType extends Type {
return 16385;
case BITMAP:
return 1024; // this is a estimated value
case QUANTILE_STATE:
return 1024; // TODO(weixiang): no used in FE, figure out whether can delete this funcion?
case STRING:
return 1024;
default:

38
fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
View File

@ -78,6 +78,7 @@ public abstract class Type {
public static final ScalarType HLL = ScalarType.createHllType();
public static final ScalarType CHAR = (ScalarType) ScalarType.createCharType(-1);
public static final ScalarType BITMAP = new ScalarType(PrimitiveType.BITMAP);
public static final ScalarType QUANTILE_STATE = new ScalarType(PrimitiveType.QUANTILE_STATE);
// Only used for alias function, to represent any type in function args
public static final ScalarType ALL = new ScalarType(PrimitiveType.ALL);
public static final MapType Map = new MapType();
@ -118,6 +119,7 @@ public abstract class Type {
supportedTypes.add(VARCHAR);
supportedTypes.add(HLL);
supportedTypes.add(BITMAP);
supportedTypes.add(QUANTILE_STATE);
supportedTypes.add(CHAR);
supportedTypes.add(DATE);
supportedTypes.add(DATETIME);
@ -195,7 +197,9 @@ public abstract class Type {
// 3. don't support group by
// 4. don't support index
public boolean isOnlyMetricType() {
return isScalarType(PrimitiveType.HLL) || isScalarType(PrimitiveType.BITMAP);
// now only_metric_type is the same to object_stored_type
// but actually they are not same in semantics.
return isObjectStored();
}
public static final String OnlyMetricTypeErrorMsg =
@ -210,6 +214,12 @@ public abstract class Type {
return isScalarType(PrimitiveType.BITMAP);
}
public boolean isQuantileStateType() { return isScalarType(PrimitiveType.QUANTILE_STATE); }
public boolean isObjectStored() {
return isHllType() || isBitmapType() || isQuantileStateType();
}
public boolean isScalarType() {
return this instanceof ScalarType;
}
@ -528,6 +538,8 @@ public abstract class Type {
return new StructType();
case BITMAP:
return Type.BITMAP;
case QUANTILE_STATE:
return Type.QUANTILE_STATE;
default:
return null;
}
@ -824,6 +836,7 @@ public abstract class Type {
compatibilityMatrix[BOOLEAN.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[BOOLEAN.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// TINYINT
@ -843,6 +856,7 @@ public abstract class Type {
compatibilityMatrix[TINYINT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[TINYINT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// SMALLINT
compatibilityMatrix[SMALLINT.ordinal()][INT.ordinal()] = PrimitiveType.INT;
@ -860,6 +874,7 @@ public abstract class Type {
compatibilityMatrix[SMALLINT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[SMALLINT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// INT
compatibilityMatrix[INT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
@ -880,6 +895,8 @@ public abstract class Type {
compatibilityMatrix[INT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[INT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// BIGINT
// 64 bit integer does not fit in mantissa of double or float.
@ -901,6 +918,7 @@ public abstract class Type {
compatibilityMatrix[BIGINT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[BIGINT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// LARGEINT
compatibilityMatrix[LARGEINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE;
@ -914,6 +932,7 @@ public abstract class Type {
compatibilityMatrix[LARGEINT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[LARGEINT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// FLOAT
compatibilityMatrix[FLOAT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
@ -926,6 +945,7 @@ public abstract class Type {
compatibilityMatrix[FLOAT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[FLOAT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// DOUBLE
compatibilityMatrix[DOUBLE.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
@ -937,6 +957,7 @@ public abstract class Type {
compatibilityMatrix[DOUBLE.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[DOUBLE.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// DATE
compatibilityMatrix[DATE.ordinal()][DATETIME.ordinal()] = PrimitiveType.DATETIME;
@ -947,6 +968,7 @@ public abstract class Type {
compatibilityMatrix[DATE.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// DATETIME
compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
@ -956,6 +978,7 @@ public abstract class Type {
compatibilityMatrix[DATETIME.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// We can convert some but not all string values to timestamps.
// CHAR
@ -965,6 +988,7 @@ public abstract class Type {
compatibilityMatrix[CHAR.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING;
compatibilityMatrix[CHAR.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// VARCHAR
compatibilityMatrix[VARCHAR.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE;
@ -972,11 +996,13 @@ public abstract class Type {
compatibilityMatrix[VARCHAR.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING;
compatibilityMatrix[VARCHAR.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
//String
compatibilityMatrix[STRING.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// DECIMALV2
@ -984,19 +1010,26 @@ public abstract class Type {
compatibilityMatrix[DECIMALV2.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// HLL
compatibilityMatrix[HLL.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// BITMAP
compatibilityMatrix[BITMAP.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BITMAP.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BITMAP.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE;
// TIME
//QUANTILE_STATE
compatibilityMatrix[QUANTILE_STATE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// TIME why here not???
compatibilityMatrix[TIME.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
// Check all of the necessary entries that should be filled.
@ -1040,6 +1073,7 @@ public abstract class Type {
case VARCHAR:
case HLL:
case BITMAP:
case QUANTILE_STATE:
return VARCHAR;
case DECIMALV2:
return DECIMALV2;

1
fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
View File

@ -73,6 +73,7 @@ public class Util {
TYPE_STRING_MAP.put(PrimitiveType.HLL, "varchar(%d)");
TYPE_STRING_MAP.put(PrimitiveType.BOOLEAN, "bool");
TYPE_STRING_MAP.put(PrimitiveType.BITMAP, "bitmap");
TYPE_STRING_MAP.put(PrimitiveType.QUANTILE_STATE,"quantile_state");
TYPE_STRING_MAP.put(PrimitiveType.ARRAY, "Array<%s>");
TYPE_STRING_MAP.put(PrimitiveType.NULL_TYPE, "null");
}

1
fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
View File

@ -544,6 +544,7 @@ public class BrokerScanNode extends LoadScanNode {
return rangeDesc;
}
//TODO(wx):support quantile state column or forbidden it.
@Override
public void finalize(Analyzer analyzer) throws UserException {
locationsList = Lists.newArrayList();

14
fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
View File

@ -116,6 +116,16 @@ public abstract class LoadScanNode extends ScanNode {
}
}
protected void checkQuantileStateCompatibility(Analyzer analyzer, SlotDescriptor slotDesc, Expr expr) throws AnalysisException {
if (slotDesc.getColumn().getAggregationType() == AggregateType.QUANTILE_UNION) {
expr.analyze(analyzer);
if (!expr.getType().isQuantileStateType()) {
String errorMsg = String.format("quantile_state column %s require the function return type is QUANTILE_STATE");
throw new AnalysisException(errorMsg);
}
}
}
protected void finalizeParams(Map<String, SlotDescriptor> slotDescByName,
Map<String, Expr> exprMap,
TBrokerScanRangeParams params,
@ -173,6 +183,10 @@ public abstract class LoadScanNode extends ScanNode {
checkBitmapCompatibility(analyzer, destSlotDesc, expr);
checkQuantileStateCompatibility(analyzer, destSlotDesc, expr);
// check quantile_state
if (negative && destSlotDesc.getColumn().getAggregationType() == AggregateType.SUM) {
expr = new ArithmeticExpr(ArithmeticExpr.Operator.MULTIPLY, expr, new IntLiteral(-1));
expr.analyze(analyzer);

10
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -598,7 +598,15 @@ public class SingleNodePlanner {
returnColumnValidate = false;
break;
}
} else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("multi_distinct_count")) {
} else if (aggExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.QUANTILE_UNION)) {
if (col.getAggregationType() != AggregateType.QUANTILE_UNION) {
turnOffReason =
"Aggregate Operator not match: QUANTILE_UNION <---> " + col.getAggregationType();
returnColumnValidate = false;
break;
}
}
else if (aggExpr.getFnName().getFunction().equalsIgnoreCase("multi_distinct_count")) {
// count(distinct k1), count(distinct k2) / count(distinct k1,k2) can turn on pre aggregation
if ((!col.isKey())) {
turnOffReason = "Multi count or sum distinct with non-key column: " + col.getName();

4
fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java
View File

@ -544,6 +544,10 @@ public class HadoopLoadPendingTask extends LoadPendingTask {
case BITMAP:
columnType = "BITMAP";
break;
// TODO(weixiang): not support in broker load.
case QUANTILE_STATE:
columnType = "QUANTILE_STATE";
break;
case DECIMALV2:
columnType = "DECIMAL";
break;

2
fe/fe-core/src/main/jflex/sql_scanner.flex
View File

@ -111,6 +111,8 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("binlog", new Integer(SqlParserSymbols.KW_BINLOG));
keywordMap.put("bitmap", new Integer(SqlParserSymbols.KW_BITMAP));
keywordMap.put("bitmap_union", new Integer(SqlParserSymbols.KW_BITMAP_UNION));
keywordMap.put("quantile_state", new Integer(SqlParserSymbols.KW_QUANTILE_STATE));
keywordMap.put("quantile_union", new Integer(SqlParserSymbols.KW_QUANTILE_UNION));
keywordMap.put("blob", new Integer(SqlParserSymbols.KW_BLOB));
keywordMap.put("boolean", new Integer(SqlParserSymbols.KW_BOOLEAN));
keywordMap.put("broker", new Integer(SqlParserSymbols.KW_BROKER));