database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature] (Nereids) add check to disable unsupported type (#17196)

Browse Source 
1. disable decimalv3
2. disable json
3. disable complex type: array, map, struct
4. disable switch: group_by_and_having_use_alias
This commit is contained in:
mch_ucchi
2023-03-03 17:57:48 +08:00
committed by GitHub
parent 17164cf7a8
commit 9f97cd029f
11 changed files with 255 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java
View File

@ -19,13 +19,23 @@ package org.apache.doris.nereids.processor.post;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.types.JsonType;
import org.apache.doris.nereids.types.MapType;
import org.apache.doris.nereids.types.StructType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import java.util.Collection;
import java.util.Set;
@ -47,7 +57,7 @@ public class Validator extends PlanPostProcessor {
// Set<Slot> childOutputSet = child.getOutputSet();
child.accept(this, context);
return project;
return visit(project, context);
}
@Override
@ -72,6 +82,38 @@ public class Validator extends PlanPostProcessor {
}
child.accept(this, context);
return filter;
return visit(filter, context);
}
@Override
public Plan visit(Plan plan, CascadesContext context) {
plan.getExpressions().forEach(ExpressionChecker.INSTANCE::check);
plan.children().forEach(child -> child.accept(this, context));
return plan;
}
private static class ExpressionChecker extends DefaultExpressionVisitor<Expression, Void> {
public static final ExpressionChecker INSTANCE = new ExpressionChecker();
public void check(Expression expression) {
expression.accept(this, null);
}
public Expression visit(Expression expr, Void unused) {
try {
checkTypes(expr.getDataType());
} catch (UnboundException ignored) {
return expr;
}
expr.children().forEach(child -> child.accept(this, null));
return expr;
}
private void checkTypes(DataType dataType) {
if (ImmutableSet.of(MapType.class, StructType.class, JsonType.class,
ArrayType.class, DecimalV3Type.class).contains(dataType.getClass())) {
throw new AnalysisException(String.format("type %s is unsupported for Nereids", dataType));
}
}
}
}

38
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
View File

@ -66,6 +66,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
import org.apache.doris.nereids.trees.plans.logical.UsingJoin;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@ -331,6 +332,7 @@ public class BindExpression implements AnalysisRuleFactory {
groupBy = groupBy.stream()
.map(expr -> bindFunction(expr, ctx.cascadesContext))
.collect(ImmutableList.toImmutableList());
checkIfOutputAliasNameDuplicatedForGroupBy(groupBy, output);
return agg.withGroupByAndOutput(groupBy, output);
})
),
@ -376,6 +378,7 @@ public class BindExpression implements AnalysisRuleFactory {
.collect(ImmutableList.toImmutableList()))
.collect(ImmutableList.toImmutableList());
List<NamedExpression> newOutput = adjustNullableForRepeat(groupingSets, output);
groupingSets.forEach(list -> checkIfOutputAliasNameDuplicatedForGroupBy(list, newOutput));
return repeat.withGroupSetsAndOutput(groupingSets, newOutput);
})
),
@ -421,9 +424,9 @@ public class BindExpression implements AnalysisRuleFactory {
})
),
RuleType.BINDING_HAVING_SLOT.build(
logicalHaving(aggregate()).thenApply(ctx -> {
logicalHaving(aggregate()).when(Plan::canBind).thenApply(ctx -> {
LogicalHaving<Aggregate<Plan>> having = ctx.root;
Plan childPlan = having.child();
Aggregate<Plan> childPlan = having.child();
Set<Expression> boundConjuncts = having.getConjuncts().stream()
.map(expr -> {
expr = bindSlot(expr, childPlan.children(), ctx.cascadesContext, false);
@ -431,6 +434,8 @@ public class BindExpression implements AnalysisRuleFactory {
})
.map(expr -> bindFunction(expr, ctx.cascadesContext))
.collect(Collectors.toSet());
checkIfOutputAliasNameDuplicatedForGroupBy(ImmutableList.copyOf(boundConjuncts),
childPlan.getOutputExpressions());
return new LogicalHaving<>(boundConjuncts, having.child());
})
),
@ -445,6 +450,9 @@ public class BindExpression implements AnalysisRuleFactory {
})
.map(expr -> bindFunction(expr, ctx.cascadesContext))
.collect(Collectors.toSet());
checkIfOutputAliasNameDuplicatedForGroupBy(ImmutableList.copyOf(boundConjuncts),
childPlan.getOutput().stream().map(NamedExpression.class::cast)
.collect(Collectors.toList()));
return new LogicalHaving<>(boundConjuncts, having.child());
})
),
@ -668,4 +676,30 @@ public class BindExpression implements AnalysisRuleFactory {
public boolean canBind(Plan plan) {
return !plan.hasUnboundExpression() || plan.canBind();
}
private void checkIfOutputAliasNameDuplicatedForGroupBy(List<Expression> expressions,
List<NamedExpression> output) {
// if group_by_and_having_use_alias_first=true, we should fall back to original planner until we
// support the session variable.
if (output.stream().noneMatch(Alias.class::isInstance)) {
return;
}
List<Alias> aliasList = output.stream().filter(Alias.class::isInstance)
.map(Alias.class::cast).collect(Collectors.toList());
List<NamedExpression> exprAliasList = expressions.stream()
.map(expr -> (Set<NamedExpression>) expr.collect(NamedExpression.class::isInstance))
.flatMap(Collection::stream)
.collect(Collectors.toList());
boolean isGroupByContainAlias = exprAliasList.stream().anyMatch(ne ->
aliasList.stream().anyMatch(alias -> !alias.getExprId().equals(ne.getExprId())
&& alias.getName().equals(ne.getName())));
if (isGroupByContainAlias
&& ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().isGroupByAndHavingUseAliasFirst()) {
throw new AnalysisException("group_by_and_having_use_alias=true is unsupported for Nereids");
}
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/types/JsonType.java
View File

@ -36,7 +36,7 @@ public class JsonType extends DataType {
@Override
public Type toCatalogDataType() {
return Type.QUANTILE_STATE;
return Type.JSONB;
}
@Override

1
fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java
View File

@ -33,7 +33,6 @@ import java.util.Optional;
* Judgment expression type.
*/
public class TypeUtils {
public static boolean isAddOrSubtract(Expression expr) {
return isAdd(expr) || isSubtract(expr);
}

114
fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java Normal file
View File

@ -0,0 +1,114 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids;
import org.apache.doris.common.Config;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class UnsupportedTypeTest extends TestWithFeService {
@Override
protected void runBeforeAll() throws Exception {
Config.enable_map_type = true;
Config.enable_struct_type = true;
createDatabase("test");
connectContext.setDatabase("default_cluster:test");
createTables(
"create table type_tb (\n"
+ " id int NOT NULL, \n"
+ " kjsonb jsonb,\n"
+ " kdcml decimalv3(15, 2),\n"
+ " karr array<int>,\n"
+ " `date` bigint(20) NOT NULL\n"
+ " )\n"
+ " DUPLICATE KEY(id) \n"
+ " distributed by hash(id) buckets 2\n"
+ " properties (\n"
+ " \"replication_num\"=\"1\"\n"
+ " )",
"create table type_tb1 (\n"
+ " id int NOT NULL, \n"
+ " kmap map<string, string>,\n"
+ " kstruct struct<a: int, b: int>\n"
+ " )\n"
+ " DUPLICATE KEY(id) \n"
+ " distributed by hash(id) buckets 2\n"
+ " properties (\n"
+ " \"replication_num\"=\"1\"\n"
+ " )");
}
@Test
public void testUnsupportedTypeThrowException() {
String[] sqls = {
"select id from type_tb",
"select jsonb_parse('{\"k1\":\"v31\",\"k2\":300}')",
"select karr from type_tb",
"select array_range(10)",
"select kdcml from type_tb",
"select cast(0.3 as decimalv3(12, 2))",
"select kmap from type_tb1",
};
Class[] exceptions = {
null,
AnalysisException.class,
AnalysisException.class,
AnalysisException.class,
AnalysisException.class,
AnalysisException.class,
AnalysisException.class
};
runPlanner(sqls[0]);
for (int i = 1; i < sqls.length; ++i) {
int iCopy = i;
Assertions.assertThrows(exceptions[i], () -> runPlanner(sqls[iCopy]));
}
}
@Test
public void testGroupByAndHavingUseAliasFirstThrowException() {
String[] sqls = {"SELECT\n"
+ " date_format(date, '%x%v') AS `date`,\n"
+ " count(date) AS `diff_days`\n"
+ " FROM type_tb\n"
+ " GROUP BY date\n"
+ " HAVING date = 20221111\n"
+ " ORDER BY date;",
"SELECT\n"
+ " date_format(date, '%x%v') AS `date`,\n"
+ " count(date) AS `diff_days`\n"
+ " FROM type_tb\n"
+ " GROUP BY date\n"
+ " HAVING date = 20221111\n"
+ " ORDER BY date;"
};
runPlanner(sqls[0]);
connectContext.getSessionVariable().groupByAndHavingUseAliasFirst = true;
Assertions.assertThrows(AnalysisException.class, () -> runPlanner(sqls[1]));
}
private void runPlanner(String sql) {
new NereidsPlanner(MemoTestUtils.createStatementContext(connectContext, sql)).plan(new NereidsParser().parseSingle(sql), PhysicalProperties.ANY);
}
}