database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bugfix](vec) Fix all create mv using to_bitmap() on negative value columns when enable_vectorized_alter_table is true (#13448)

Browse Source 
* [Bugfix] add negtive value check when create mv using vec
This commit is contained in:
Zhengguo Yang
2022-10-19 15:40:04 +08:00
committed by GitHub
parent 5423de68dd
commit 0b368fbbfa
11 changed files with 117 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View File

@ -64,11 +64,11 @@ public class CreateMaterializedViewStmt extends DdlStmt {
static {
FN_NAME_TO_PATTERN = Maps.newHashMap();
FN_NAME_TO_PATTERN.put(AggregateType.SUM.name().toLowerCase(),
new MVColumnOneChildPattern(AggregateType.SUM.name().toLowerCase()));
new MVColumnOneChildPattern(AggregateType.SUM.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(AggregateType.MIN.name().toLowerCase(),
new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase()));
new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(AggregateType.MAX.name().toLowerCase(),
new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase()));
new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(FunctionSet.COUNT, new MVColumnOneChildPattern(FunctionSet.COUNT));
FN_NAME_TO_PATTERN.put(FunctionSet.BITMAP_UNION, new MVColumnBitmapUnionPattern());
FN_NAME_TO_PATTERN.put(FunctionSet.HLL_UNION, new MVColumnHLLUnionPattern());
@ -147,16 +147,16 @@ public class CreateMaterializedViewStmt extends DdlStmt {
analyzeFromClause();
if (selectStmt.getWhereClause() != null) {
throw new AnalysisException("The where clause is not supported in add materialized view clause, expr:"
+ selectStmt.getWhereClause().toSql());
+ selectStmt.getWhereClause().toSql());
}
if (selectStmt.getHavingPred() != null) {
throw new AnalysisException("The having clause is not supported in add materialized view clause, expr:"
+ selectStmt.getHavingPred().toSql());
+ selectStmt.getHavingPred().toSql());
}
analyzeOrderByClause();
if (selectStmt.getLimit() != -1) {
throw new AnalysisException("The limit clause is not supported in add materialized view clause, expr:"
+ " limit " + selectStmt.getLimit());
+ " limit " + selectStmt.getLimit());
}
}
@ -182,7 +182,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
Expr selectListItemExpr = selectListItem.getExpr();
if (!(selectListItemExpr instanceof SlotRef) && !(selectListItemExpr instanceof FunctionCallExpr)) {
throw new AnalysisException("The materialized view only support the single column or function expr. "
+ "Error column: " + selectListItemExpr.toSql());
+ "Error column: " + selectListItemExpr.toSql());
}
if (selectListItemExpr instanceof SlotRef) {
if (meetAggregate) {
@ -211,6 +211,25 @@ public class CreateMaterializedViewStmt extends DdlStmt {
throw new AnalysisException(
"The function " + functionName + " must match pattern:" + mvColumnPattern.toString());
}
// for bitmap_union(to_bitmap(column)) function, we should check value is not negative
// in vectorized schema_change mode, so we should rewrite the function to
// bitmap_union(to_bitmap_with_check(column))
if (functionName.equalsIgnoreCase("bitmap_union")) {
if (functionCallExpr.getChildren().size() == 1
&& functionCallExpr.getChild(0) instanceof FunctionCallExpr) {
Expr child = functionCallExpr.getChild(0);
FunctionCallExpr childFunctionCallExpr = (FunctionCallExpr) child;
if (childFunctionCallExpr.getFnName().getFunction().equalsIgnoreCase("to_bitmap")) {
childFunctionCallExpr.setFnName(
new FunctionName(childFunctionCallExpr.getFnName().getDb(),
"to_bitmap_with_check"));
childFunctionCallExpr.getFn().setName(
new FunctionName(childFunctionCallExpr.getFn().getFunctionName().getDb(),
"to_bitmap_with_check"));
}
}
}
}
// check duplicate column
List<SlotRef> slots = new ArrayList<>();
@ -256,7 +275,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
List<OrderByElement> orderByElements = selectStmt.getOrderByElements();
if (orderByElements.size() > mvColumnItemList.size()) {
throw new AnalysisException("The number of columns in order clause must be less than " + "the number of "
+ "columns in select clause");
+ "columns in select clause");
}
if (beginIndexOfAggregation != -1 && (orderByElements.size() != (beginIndexOfAggregation))) {
throw new AnalysisException("The key of columns in mv must be all of group by columns");
@ -265,13 +284,13 @@ public class CreateMaterializedViewStmt extends DdlStmt {
Expr orderByElement = orderByElements.get(i).getExpr();
if (!(orderByElement instanceof SlotRef)) {
throw new AnalysisException("The column in order clause must be original column without calculation. "
+ "Error column: " + orderByElement.toSql());
+ "Error column: " + orderByElement.toSql());
}
MVColumnItem mvColumnItem = mvColumnItemList.get(i);
SlotRef slotRef = (SlotRef) orderByElement;
if (!mvColumnItem.getName().equalsIgnoreCase(slotRef.getColumnName())) {
throw new AnalysisException("The order of columns in order by clause must be same as "
+ "the order of columns in select list");
+ "the order of columns in select list");
}
Preconditions.checkState(mvColumnItem.getAggregationType() == null);
mvColumnItem.setIsKey(true);
@ -451,7 +470,8 @@ public class CreateMaterializedViewStmt extends DdlStmt {
CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef);
List<Expr> params = Lists.newArrayList();
params.add(castExpr);
FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.TO_BITMAP, params);
FunctionCallExpr defineExpr =
new FunctionCallExpr(FunctionSet.TO_BITMAP_WITH_CHECK, params);
result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr);
} else {
result.put(baseColumnName, null);
@ -471,7 +491,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
case FunctionSet.COUNT:
Expr defineExpr = new CaseExpr(null, Lists.newArrayList(
new CaseWhenClause(new IsNullPredicate(slots.get(0), false),
new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT));
new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT));
result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr);
break;
default:

4
fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View File

@ -128,6 +128,10 @@ public class FunctionCallExpr extends Expr {
isTableFnCall = tableFnCall;
}
public void setFnName(FunctionName fnName) {
this.fnName = fnName;
}
public Function getFn() {
return fn;
}

3
fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java
View File

@ -44,7 +44,8 @@ public class MVColumnBitmapUnionPattern implements MVColumnPattern {
}
} else if (fnExpr.getChild(0) instanceof FunctionCallExpr) {
FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0);
if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.TO_BITMAP)) {
if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.TO_BITMAP)
&& !child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.TO_BITMAP_WITH_CHECK)) {
return false;
}
SlotRef slotRef = child0FnExpr.getChild(0).unwrapSlotRef();

4
fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
View File

@ -201,6 +201,10 @@ public class Function implements Writable {
location = loc;
}
public void setName(FunctionName name) {
this.name = name;
}
public TFunctionBinaryType getBinaryType() {
return binaryType;
}

1
fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
View File

@ -911,6 +911,7 @@ public class FunctionSet<T> {
.build();
public static final String TO_BITMAP = "to_bitmap";
public static final String TO_BITMAP_WITH_CHECK = "to_bitmap_with_check";
public static final String BITMAP_UNION = "bitmap_union";
public static final String BITMAP_UNION_COUNT = "bitmap_union_count";
public static final String BITMAP_UNION_INT = "bitmap_union_int";

1
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java
View File

@ -41,6 +41,7 @@ public class FunctionCallEqualRule implements MVExprEqualRule {
builder.put(FunctionSet.HLL_UNION, "hll_union");
builder.put(FunctionSet.HLL_UNION, "hll_raw_agg");
builder.put(FunctionSet.TO_BITMAP, FunctionSet.TO_BITMAP);
builder.put(FunctionSet.TO_BITMAP_WITH_CHECK, FunctionSet.TO_BITMAP_WITH_CHECK);
builder.put(FunctionSet.HLL_HASH, FunctionSet.HLL_HASH);
columnAggTypeMatchFnName = builder.build();
}

3
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java
View File

@ -65,7 +65,8 @@ public class ToBitmapToSlotRefRule implements ExprRewriteRule {
return expr;
}
FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0);
if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("to_bitmap")) {
if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("to_bitmap")
&& !child0FnExpr.getFnName().getFunction().equalsIgnoreCase("to_bitmap_with_check")) {
return expr;
}
if (child0FnExpr.getChild(0) instanceof SlotRef) {