database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids) support agg state type in create table (#32171)

Browse Source 
this PR introduce a behavior change, syntax of create table with agg_state type is changed.
This commit is contained in:
morrySnow
2024-03-14 14:57:32 +08:00
committed by yiguolei
parent 62023d705d
commit ea2fbfaffa
24 changed files with 121 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
View File

@ -332,12 +332,12 @@ public class ColumnDef {
}
// check if aggregate type is valid
if (aggregateType != AggregateType.GENERIC_AGGREGATION
if (aggregateType != AggregateType.GENERIC
&& !aggregateType.checkCompatibility(type.getPrimitiveType())) {
throw new AnalysisException(String.format("Aggregate type %s is not compatible with primitive type %s",
toString(), type.toSql()));
}
if (aggregateType == AggregateType.GENERIC_AGGREGATION) {
if (aggregateType == AggregateType.GENERIC) {
if (!SessionVariable.enableAggState()) {
throw new AnalysisException("agg state not enable, need set enable_agg_state=true");
}

2
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View File

@ -549,7 +549,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
type = Type.BIGINT;
break;
default:
mvAggregateType = AggregateType.GENERIC_AGGREGATION;
mvAggregateType = AggregateType.GENERIC;
if (functionCallExpr.getParams().isDistinct() || functionCallExpr.getParams().isStar()) {
throw new AnalysisException(
"The Materialized-View's generic aggregation not support star or distinct");

2
fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java
View File

@ -37,7 +37,7 @@ public enum AggregateType {
NONE("NONE"),
BITMAP_UNION("BITMAP_UNION"),
QUANTILE_UNION("QUANTILE_UNION"),
GENERIC_AGGREGATION("GENERIC_AGGREGATION");
GENERIC("GENERIC");
private static EnumMap<AggregateType, EnumSet<PrimitiveType>> compatibilityMap;

35
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View File

@ -128,9 +128,6 @@ public class Column implements Writable, GsonPostProcessable {
@SerializedName(value = "uniqueId")
private int uniqueId;
@SerializedName(value = "genericAggregationName")
private String genericAggregationName;
@SerializedName(value = "clusterKeyId")
private int clusterKeyId = -1;
@ -244,8 +241,8 @@ public class Column implements Writable, GsonPostProcessable {
c.setIsAllowNull(aggState.getSubTypeNullables().get(i));
addChildrenColumn(c);
}
this.genericAggregationName = aggState.getFunctionName();
this.aggregationType = AggregateType.GENERIC_AGGREGATION;
this.isAllowNull = false;
this.aggregationType = AggregateType.GENERIC;
}
}
@ -449,11 +446,7 @@ public class Column implements Writable, GsonPostProcessable {
}
public String getAggregationString() {
if (getAggregationType() == AggregateType.GENERIC_AGGREGATION) {
return getGenericAggregationString();
} else {
return getAggregationType().name();
}
return getAggregationType().name();
}
public boolean isAggregated() {
@ -764,22 +757,6 @@ public class Column implements Writable, GsonPostProcessable {
return toSql(isUniqueTable, false);
}
public String getGenericAggregationString() {
StringBuilder sb = new StringBuilder();
sb.append(genericAggregationName).append("(");
for (int i = 0; i < children.size(); i++) {
if (i != 0) {
sb.append(", ");
}
sb.append(children.get(i).getType().toSql());
if (children.get(i).isAllowNull()) {
sb.append(" NULL");
}
}
sb.append(")");
return sb.toString();
}
public String toSql(boolean isUniqueTable, boolean isCompatible) {
StringBuilder sb = new StringBuilder();
sb.append("`").append(name).append("` ");
@ -791,11 +768,9 @@ public class Column implements Writable, GsonPostProcessable {
} else {
sb.append(typeStr);
}
if (aggregationType == AggregateType.GENERIC_AGGREGATION) {
sb.append(" ").append(getGenericAggregationString());
} else if (aggregationType != null && aggregationType != AggregateType.NONE && !isUniqueTable
if (aggregationType != null && aggregationType != AggregateType.NONE && !isUniqueTable
&& !isAggregationTypeImplicit) {
sb.append(" ").append(aggregationType.name());
sb.append(" ").append(aggregationType.toSql());
}
if (isAllowNull) {
sb.append(" NULL");

34
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.BuiltinAggregateFunctions;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.ScalarType;
@ -42,6 +43,7 @@ import org.apache.doris.mtmv.MTMVRefreshTriggerInfo;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.DorisParser.AddConstraintContext;
import org.apache.doris.nereids.DorisParser.AggClauseContext;
import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext;
import org.apache.doris.nereids.DorisParser.AliasQueryContext;
import org.apache.doris.nereids.DorisParser.AliasedQueryContext;
import org.apache.doris.nereids.DorisParser.AlterMTMVContext;
@ -75,6 +77,7 @@ import org.apache.doris.nereids.DorisParser.CreateProcedureContext;
import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext;
import org.apache.doris.nereids.DorisParser.CreateTableContext;
import org.apache.doris.nereids.DorisParser.CteContext;
import org.apache.doris.nereids.DorisParser.DataTypeWithNullableContext;
import org.apache.doris.nereids.DorisParser.DateCeilContext;
import org.apache.doris.nereids.DorisParser.DateFloorContext;
import org.apache.doris.nereids.DorisParser.Date_addContext;
@ -422,6 +425,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.logical.UsingJoin;
import org.apache.doris.nereids.types.AggStateType;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.MapType;
@ -2519,7 +2523,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
String colName = ctx.colName.getText();
DataType colType = ctx.type instanceof PrimitiveDataTypeContext
? visitPrimitiveDataType(((PrimitiveDataTypeContext) ctx.type))
: visitComplexDataType(((ComplexDataTypeContext) ctx.type));
: ctx.type instanceof ComplexDataTypeContext
? visitComplexDataType((ComplexDataTypeContext) ctx.type)
: visitAggStateDataType((AggStateDataTypeContext) ctx.type);
colType = colType.conversion();
boolean isKey = ctx.KEY() != null;
boolean isNotNull = ctx.NOT() != null;
@ -3248,6 +3254,32 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return ExplainLevel.ALL_PLAN;
}
@Override
public Pair<DataType, Boolean> visitDataTypeWithNullable(DataTypeWithNullableContext ctx) {
return ParserUtils.withOrigin(ctx, () -> Pair.of(typedVisit(ctx.dataType()), ctx.NOT() == null));
}
@Override
public DataType visitAggStateDataType(AggStateDataTypeContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {
List<Pair<DataType, Boolean>> dataTypeWithNullables = ctx.dataTypes.stream()
.map(this::visitDataTypeWithNullable)
.collect(Collectors.toList());
List<DataType> dataTypes = dataTypeWithNullables.stream()
.map(dt -> dt.first)
.collect(ImmutableList.toImmutableList());
List<Boolean> nullables = dataTypeWithNullables.stream()
.map(dt -> dt.second)
.collect(ImmutableList.toImmutableList());
String functionName = ctx.functionNameIdentifier().getText();
if (!BuiltinAggregateFunctions.INSTANCE.aggFuncNames.contains(functionName)) {
// TODO use function binder to check function exists
throw new ParseException("Can not found function '" + functionName + "'", ctx);
}
return new AggStateType(functionName, dataTypes, nullables);
});
}
@Override
public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
View File

@ -1487,7 +1487,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
@Override
public Expression visitAggregateFunction(AggregateFunction aggregateFunction, RewriteContext context) {
String aggStateName = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(
AggregateType.GENERIC_AGGREGATION, StateCombinator.create(aggregateFunction).toSql()));
AggregateType.GENERIC, StateCombinator.create(aggregateFunction).toSql()));
Column mvColumn = context.checkContext.getColumn(aggStateName);
if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {

64
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
View File

@ -246,6 +246,20 @@ public class ColumnDefinition {
}
}
if (aggType != null) {
// check if aggregate type is valid
if (aggType != AggregateType.GENERIC
&& !aggType.checkCompatibility(type.toCatalogDataType().getPrimitiveType())) {
throw new AnalysisException(String.format("Aggregate type %s is not compatible with primitive type %s",
aggType, type.toSql()));
}
if (aggType == AggregateType.GENERIC) {
if (!SessionVariable.enableAggState()) {
throw new AnalysisException("agg state not enable, need set enable_agg_state=true");
}
}
}
if (isOlap) {
if (!isKey && keysType.equals(KeysType.UNIQUE_KEYS)) {
aggTypeImplicit = true;
@ -334,14 +348,28 @@ public class ColumnDefinition {
// from old planner CreateTableStmt's analyze method, after call columnDef.analyze(engineName.equals("olap"));
if (isOlap && type.isComplexType()) {
if (aggType != null && aggType != AggregateType.NONE
&& aggType != AggregateType.REPLACE) {
throw new AnalysisException(type.toCatalogDataType().getPrimitiveType()
+ " column can't support aggregation " + aggType);
}
if (isKey) {
throw new AnalysisException(type.toCatalogDataType().getPrimitiveType()
+ " can only be used in the non-key column of the duplicate table at present.");
+ " can only be used in the non-key column at present.");
}
if (type.isAggStateType()) {
if (aggType == null) {
throw new AnalysisException(type.toCatalogDataType().getPrimitiveType()
+ " column must have aggregation type");
} else {
if (aggType != AggregateType.GENERIC
&& aggType != AggregateType.NONE
&& aggType != AggregateType.REPLACE) {
throw new AnalysisException(type.toCatalogDataType().getPrimitiveType()
+ " column can't support aggregation " + aggType);
}
}
isNullable = false;
} else {
if (aggType != null && aggType != AggregateType.NONE && aggType != AggregateType.REPLACE) {
throw new AnalysisException(type.toCatalogDataType().getPrimitiveType()
+ " column can't support aggregation " + aggType);
}
}
}
@ -350,30 +378,6 @@ public class ColumnDefinition {
}
}
/**
* check if is nested complex type.
*/
private boolean isNestedComplexType(DataType dataType) {
if (!dataType.isComplexType()) {
return false;
}
if (dataType instanceof ArrayType) {
if (((ArrayType) dataType).getItemType() instanceof ArrayType) {
return isNestedComplexType(((ArrayType) dataType).getItemType());
} else {
return ((ArrayType) dataType).getItemType().isComplexType();
}
}
if (dataType instanceof MapType) {
return ((MapType) dataType).getKeyType().isComplexType()
|| ((MapType) dataType).getValueType().isComplexType();
}
if (dataType instanceof StructType) {
return ((StructType) dataType).getFields().stream().anyMatch(f -> f.getDataType().isComplexType());
}
return false;
}
// from TypeDef.java analyze()
private void validateDataType(Type catalogType) {
if (catalogType.exceedsMaxNestingDepth()) {

7
fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java
View File

@ -35,8 +35,6 @@ import java.util.stream.Collectors;
*/
public class AggStateType extends DataType {
public static final AggStateType SYSTEM_DEFAULT = new AggStateType(null, ImmutableList.of(), ImmutableList.of());
public static final int WIDTH = 16;
private final List<DataType> subTypes;
@ -94,11 +92,6 @@ public class AggStateType extends DataType {
return "agg_state";
}
@Override
public DataType defaultConcreteType() {
return SYSTEM_DEFAULT;
}
@Override
public boolean equals(Object o) {
if (!(o instanceof AggStateType)) {