database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](Nereids) process DELETE_SIGN_COLUMN of OlapTable(#16030)

Browse Source 
1. add DELETE_SIGN_COLUMN in non-visible-columns in LogicalOlapScan
2. when the table has a delete sign, add a filter `delete_sign_coumn = 0`
3. use output slots and non-visible slots to bind slot
This commit is contained in:
谢健
2023-01-20 11:27:35 +08:00
committed by GitHub
parent 6485221ffb
commit 73621bdb18
6 changed files with 112 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
@ -25,6 +26,7 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.View;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.analyzer.UnboundRelation;
@ -33,8 +35,13 @@ import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
@ -42,8 +49,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.RelationUtil;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
import java.util.Collections;
@ -148,19 +157,40 @@ public class BindRelation extends OneAnalysisRuleFactory {
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
}
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> tableQualifier) {
LogicalOlapScan scan;
List<Long> partIds = getPartitionIds(table, unboundRelation);
if (!CollectionUtils.isEmpty(partIds)) {
scan = new LogicalOlapScan(RelationUtil.newRelationId(),
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds);
} else {
scan = new LogicalOlapScan(RelationUtil.newRelationId(),
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)));
}
if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign()
&& !ConnectContext.get().getSessionVariable()
.skipDeleteSign()) {
// table qualifier is catalog.db.table, we make db.table.column
Slot deleteSlot = null;
for (Slot slot : scan.getOutput()) {
if (slot.getName().equals(Column.DELETE_SIGN)) {
deleteSlot = slot;
break;
}
}
Preconditions.checkArgument(deleteSlot != null);
Expression conjunct = new EqualTo(new TinyIntLiteral((byte) 0), deleteSlot);
return new LogicalFilter(Sets.newHashSet(conjunct), scan);
}
return scan;
}
private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, List<String> tableQualifier,
CascadesContext cascadesContext) {
String dbName = tableQualifier.get(1); //[catalogName, dbName, tableName]
switch (table.getType()) {
case OLAP:
List<Long> partIds = getPartitionIds(table, unboundRelation);
if (!CollectionUtils.isEmpty(partIds)) {
return new LogicalOlapScan(RelationUtil.newRelationId(),
(OlapTable) table, ImmutableList.of(dbName), partIds);
} else {
return new LogicalOlapScan(RelationUtil.newRelationId(),
(OlapTable) table, ImmutableList.of(dbName));
}
return makeOlapScan(table, unboundRelation, tableQualifier);
case VIEW:
Plan viewPlan = parseAndAnalyzeView(((View) table).getDdlSql(), cascadesContext);
return new LogicalSubQueryAlias<>(tableQualifier, viewPlan);

12
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java
View File

@ -692,22 +692,26 @@ public class BindSlotReference implements AnalysisRuleFactory {
@Override
public Expression visitUnboundStar(UnboundStar unboundStar, PlannerContext context) {
List<String> qualifier = unboundStar.getQualifier();
List<Slot> slots = getScope().getSlots()
.stream()
.filter(slot -> !(slot instanceof SlotReference) || ((SlotReference) slot).isVisible())
.collect(Collectors.toList());
switch (qualifier.size()) {
case 0: // select *
return new BoundStar(getScope().getSlots());
return new BoundStar(slots);
case 1: // select table.*
case 2: // select db.table.*
return bindQualifiedStar(qualifier);
return bindQualifiedStar(qualifier, slots);
default:
throw new AnalysisException("Not supported qualifier: "
+ StringUtils.join(qualifier, "."));
}
}
private BoundStar bindQualifiedStar(List<String> qualifierStar) {
private BoundStar bindQualifiedStar(List<String> qualifierStar, List<Slot> boundSlots) {
// FIXME: compatible with previous behavior:
// https://github.com/apache/doris/pull/10415/files/3fe9cb0c3f805ab3a9678033b281b16ad93ec60a#r910239452
List<Slot> slots = getScope().getSlots().stream().filter(boundSlot -> {
List<Slot> slots = boundSlots.stream().filter(boundSlot -> {
switch (qualifierStar.size()) {
// table.*
case 1:

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java
View File

@ -41,6 +41,9 @@ public class CheckPolicy implements AnalysisRuleFactory {
RuleType.CHECK_ROW_POLICY.build(
logicalCheckPolicy(logicalSubQueryAlias()).then(checkPolicy -> checkPolicy.child())
),
RuleType.CHECK_ROW_POLICY.build(
logicalCheckPolicy(logicalFilter()).then(checkPolicy -> checkPolicy.child())
),
RuleType.CHECK_ROW_POLICY.build(
logicalCheckPolicy(logicalRelation()).thenApply(ctx -> {
LogicalCheckPolicy<LogicalRelation> checkPolicy = ctx.root;

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
View File

@ -182,6 +182,10 @@ public class SlotReference extends Slot {
return new SlotReference(exprId, name, dataType, nullable, qualifiers, column);
}
public boolean isVisible() {
return column == null || column.isVisible();
}
@Override
public Slot withName(String name) {
return new SlotReference(exprId, name, dataType, nullable, qualifier, column);

17
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.util.Util;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
@ -34,17 +35,20 @@ import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.trees.plans.algebra.OlapScan;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Logical OlapScan.
@ -262,6 +266,19 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
: Optional.empty();
}
@Override
public List<Slot> computeOutput() {
List<Column> otherColumns = new ArrayList<>();
if (!Util.showHiddenColumns() && getTable().hasDeleteSign()
&& !ConnectContext.get().getSessionVariable()
.skipDeleteSign()) {
otherColumns.add(getTable().getDeleteSignColumn());
}
return Stream.concat(table.getBaseSchema().stream(), otherColumns.stream())
.map(col -> SlotReference.fromColumn(col, qualified()))
.collect(ImmutableList.toImmutableList());
}
@Override
public List<Slot> computeNonUserVisibleOutput() {
Set<String> baseSchemaColNames = table.getBaseSchema().stream()

42
regression-test/suites/nereids_syntax_p0/non_user_visiable_output.groovy Normal file
View File

@ -0,0 +1,42 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("non_user_visiable_output") {
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql 'set enable_vectorized_engine=true'
sql """
drop table if exists t_del;
"""
sql """
create table t_del (
id int not null
)
UNIQUE KEY (id)
distributed by hash(id)
properties(
'replication_num' = '1'
);
"""
sql """insert into t_del values (1),(2),(3);"""
sql "delete from t_del where id = 2;"
test {
sql "select id from t_del order by id;"
result([[1],[3]])
}
}