database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](materialized-view) support create mv contain aggstate column (#20812)

Browse Source 
support create mv contain aggstate column
This commit is contained in:
Pxl
2023-06-21 13:06:52 +08:00
committed by GitHub
parent fcd778fb4f
commit 5f0bb49d46
15 changed files with 393 additions and 1085 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
View File

@ -518,8 +518,7 @@ public class ColumnDef {
}
return new Column(name, type, isKey, aggregateType, isAllowNull, isAutoInc, defaultValue.value, comment,
visible, defaultValue.defaultValueExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue.getValue(),
genericAggregationName, typeList, nullableList);
visible, defaultValue.defaultValueExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue.getValue());
}
@Override

31
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View File

@ -19,6 +19,8 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Function;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndexMeta;
@ -224,20 +226,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
if (selectListItemExpr instanceof FunctionCallExpr
&& ((FunctionCallExpr) selectListItemExpr).isAggregateFunction()) {
FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItemExpr;
String functionName = functionCallExpr.getFnName().getFunction();
// current version not support count(distinct) function in creating materialized
// view
if (!isReplay) {
MVColumnPattern mvColumnPattern = FN_NAME_TO_PATTERN.get(functionName.toLowerCase());
if (mvColumnPattern == null) {
throw new AnalysisException(
"Materialized view does not support this function:" + functionCallExpr.toSqlImpl());
}
if (!mvColumnPattern.match(functionCallExpr)) {
throw new AnalysisException(
"The function " + functionName + " must match pattern:" + mvColumnPattern.toString());
}
}
if (beginIndexOfAggregation == -1) {
beginIndexOfAggregation = i;
@ -415,7 +403,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
throws AnalysisException {
String functionName = functionCallExpr.getFnName().getFunction();
List<Expr> childs = functionCallExpr.getChildren();
Preconditions.checkArgument(childs.size() == 1);
Expr defineExpr = childs.get(0);
Type baseType = defineExpr.getType();
AggregateType mvAggregateType = null;
@ -465,7 +452,9 @@ public class CreateMaterializedViewStmt extends DdlStmt {
type = Type.BIGINT;
break;
default:
throw new AnalysisException("Unsupported function:" + functionName);
mvAggregateType = AggregateType.GENERIC_AGGREGATION;
defineExpr = Function.convertToStateCombinator(functionCallExpr);
type = defineExpr.type;
}
if (mvAggregateType == null) {
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
@ -478,8 +467,8 @@ public class CreateMaterializedViewStmt extends DdlStmt {
} else if (defineExpr instanceof CastExpr && defineExpr.getChild(0) instanceof SlotRef) {
slot = (SlotRef) defineExpr.getChild(0);
} else {
throw new AnalysisException("Aggregate function require single slot argument, invalid argument is: "
+ defineExpr.toSql());
throw new AnalysisException(
"Aggregate function require single slot argument, invalid argument is: " + defineExpr.toSql());
}
AggregateType input = slot.getColumn().getAggregationType();
@ -512,6 +501,12 @@ public class CreateMaterializedViewStmt extends DdlStmt {
name = item.getName();
break;
default:
if (Env.getCurrentEnv()
.isAggFunctionName(functionCallExpr.getFnName().getFunction().toLowerCase())) {
MVColumnItem genericItem = buildMVColumnItem(analyzer, functionCallExpr);
expr = genericItem.getDefineExpr();
name = genericItem.getName();
}
break;
}
}

31
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View File

@ -69,8 +69,8 @@ public class Column implements Writable, GsonPostProcessable {
private static final String COLUMN_MAP_KEY = "key";
private static final String COLUMN_MAP_VALUE = "value";
public static final Column UNSUPPORTED_COLUMN = new Column("unknown",
Type.UNSUPPORTED, true, null, true, false, null, "invalid", true, null, -1, null, null, null, null);
public static final Column UNSUPPORTED_COLUMN = new Column("unknown", Type.UNSUPPORTED, true, null, true, false,
null, "invalid", true, null, -1, null);
@SerializedName(value = "name")
private String name;
@ -170,26 +170,24 @@ public class Column implements Writable, GsonPostProcessable {
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
String defaultValue, String comment) {
this(name, type, isKey, aggregateType, isAllowNull, false, defaultValue, comment, true, null,
COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue, null, null, null);
COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue);
}
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
String comment, boolean visible, int colUniqueId) {
this(name, type, isKey, aggregateType, isAllowNull, false, null, comment, visible, null,
colUniqueId, null, null, null, null);
this(name, type, isKey, aggregateType, isAllowNull, false, null, comment, visible, null, colUniqueId, null);
}
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef,
int colUniqueId, String realDefaultValue) {
this(name, type, isKey, aggregateType, isAllowNull, false, defaultValue, comment, visible, defaultValueExprDef,
colUniqueId, realDefaultValue, null, null, null);
colUniqueId, realDefaultValue);
}
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
boolean isAutoInc, String defaultValue, String comment, boolean visible,
DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue,
String genericAggregationName, List<Type> genericAggregationArguments, List<Boolean> nullableList) {
DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue) {
this.name = name;
if (this.name == null) {
this.name = "";
@ -215,13 +213,15 @@ public class Column implements Writable, GsonPostProcessable {
createChildrenColumn(this.type, this);
this.uniqueId = colUniqueId;
this.genericAggregationName = genericAggregationName;
if (genericAggregationArguments != null) {
for (int i = 0; i < genericAggregationArguments.size(); i++) {
Column c = new Column(COLUMN_AGG_ARGUMENT_CHILDREN, genericAggregationArguments.get(i));
c.setIsAllowNull(nullableList.get(i));
if (type.isAggStateType()) {
AggStateType aggState = (AggStateType) type;
for (int i = 0; i < aggState.getSubTypes().size(); i++) {
Column c = new Column(COLUMN_AGG_ARGUMENT_CHILDREN, aggState.getSubTypes().get(i));
c.setIsAllowNull(aggState.getSubTypeNullables().get(i));
addChildrenColumn(c);
}
this.genericAggregationName = aggState.getFunctionName();
this.aggregationType = AggregateType.GENERIC_AGGREGATION;
}
}
@ -516,8 +516,9 @@ public class Column implements Writable, GsonPostProcessable {
tColumn.setColUniqueId(uniqueId);
if (type.isAggStateType()) {
tColumn.setAggregation(genericAggregationName);
tColumn.setResultIsNullable(((AggStateType) type).getResultIsNullable());
AggStateType aggState = (AggStateType) type;
tColumn.setAggregation(aggState.getFunctionName());
tColumn.setResultIsNullable(aggState.getResultIsNullable());
for (Column column : children) {
tColumn.addToChildrenColumn(column.toThrift());
}

4
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -4743,6 +4743,10 @@ public class Env {
return functionSet.isNullResultWithOneNullParamFunctions(funcName);
}
public boolean isAggFunctionName(String name) {
return functionSet.isAggFunctionName(name);
}
@Deprecated
public long loadCluster(DataInputStream dis, long checksum) throws IOException, DdlException {
return getInternalCatalog().loadCluster(dis, checksum);

1305
fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
View File

File diff suppressed because it is too large Load Diff

2
fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
View File

@ -216,7 +216,7 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
}
if (matchedColumn != null) {
LOG.debug("trans old MV, MV: {}, DefineExpr:{}, DefineName:{}",
LOG.info("trans old MV: {}, DefineExpr:{}, DefineName:{}",
matchedColumn.getName(), entry.getValue().toSqlWithoutTbl(), entry.getKey());
matchedColumn.setDefineExpr(entry.getValue());
matchedColumn.setDefineName(entry.getKey());

5
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java vendored
View File

@ -411,9 +411,8 @@ public class HMSExternalTable extends ExternalTable {
tmpSchema.add(new Column(field.getName(),
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType(),
IcebergExternalTable.ICEBERG_DATETIME_SCALE_MS),
true, null,
true, false, null, field.getComment(), true, null,
schema.caseInsensitiveFindField(field.getName()).fieldId(), null, null, null, null));
true, null, true, false, null, field.getComment(), true, null,
schema.caseInsensitiveFindField(field.getName()).fieldId(), null));
}
return tmpSchema;
}

3
fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcMySQLClient.java vendored
View File

@ -196,8 +196,7 @@ public class JdbcMySQLClient extends JdbcClient {
dorisTableSchema.add(new Column(field.getColumnName(),
jdbcTypeToDoris(field), field.isKey(), null,
field.isAllowNull(), field.isAutoincrement(), field.getDefaultValue(), field.getRemarks(),
true, null, -1, null,
null, null, null));
true, null, -1, null));
}
return dorisTableSchema;
}