database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](hive)opt select count(*) stmt push down agg on parquet in hive . (#22115)

Browse Source 
Optimization "select count(*) from table" stmtement , push down "count" type to BE.
support file type : parquet ，orc in hive .

1. 4kfiles , 60kwline num 
    before:  1 min 37.70 sec 
    after:   50.18 sec

2. 50files , 60kwline num
    before: 1.12 sec
    after: 0.82 sec
This commit is contained in:
daidai
2023-07-29 00:31:01 +08:00
committed by GitHub
parent 53d255f482
commit ae8a26335c
21 changed files with 473 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
View File

@ -55,8 +55,11 @@ import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Project;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate;
@ -115,6 +118,20 @@ public class AggregateStrategies implements ImplementationRuleFactory {
return storageLayerAggregate(agg, project, olapScan, ctx.cascadesContext);
})
),
RuleType.STORAGE_LAYER_AGGREGATE_WITH_PROJECT.build(
logicalAggregate(
logicalProject(
logicalFileScan()
)
)
.when(agg -> agg.isNormalized() && enablePushDownNoGroupAgg())
.thenApply(ctx -> {
LogicalAggregate<LogicalProject<LogicalFileScan>> agg = ctx.root;
LogicalProject<LogicalFileScan> project = agg.child();
LogicalFileScan fileScan = project.child();
return storageLayerAggregate(agg, project, fileScan, ctx.cascadesContext);
})
),
RuleType.ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT.build(
basePattern
.when(agg -> agg.getDistinctArguments().size() == 0)
@ -190,14 +207,19 @@ public class AggregateStrategies implements ImplementationRuleFactory {
private LogicalAggregate<? extends Plan> storageLayerAggregate(
LogicalAggregate<? extends Plan> aggregate,
@Nullable LogicalProject<? extends Plan> project,
LogicalOlapScan olapScan, CascadesContext cascadesContext) {
LogicalRelation logicalScan, CascadesContext cascadesContext) {
final LogicalAggregate<? extends Plan> canNotPush = aggregate;
KeysType keysType = olapScan.getTable().getKeysType();
if (keysType != KeysType.AGG_KEYS && keysType != KeysType.DUP_KEYS) {
if (!(logicalScan instanceof LogicalOlapScan) && !(logicalScan instanceof LogicalFileScan)) {
return canNotPush;
}
if (logicalScan instanceof LogicalOlapScan) {
KeysType keysType = ((LogicalOlapScan) logicalScan).getTable().getKeysType();
if (keysType != KeysType.AGG_KEYS && keysType != KeysType.DUP_KEYS) {
return canNotPush;
}
}
List<Expression> groupByExpressions = aggregate.getGroupByExpressions();
if (!groupByExpressions.isEmpty() || !aggregate.getDistinctArguments().isEmpty()) {
return canNotPush;
@ -213,8 +235,11 @@ public class AggregateStrategies implements ImplementationRuleFactory {
if (!supportedAgg.keySet().containsAll(functionClasses)) {
return canNotPush;
}
if (functionClasses.contains(Count.class) && keysType != KeysType.DUP_KEYS) {
return canNotPush;
if (logicalScan instanceof LogicalOlapScan) {
KeysType keysType = ((LogicalOlapScan) logicalScan).getTable().getKeysType();
if (functionClasses.contains(Count.class) && keysType != KeysType.DUP_KEYS) {
return canNotPush;
}
}
if (aggregateFunctions.stream().anyMatch(fun -> fun.arity() > 1)) {
return canNotPush;
@ -281,12 +306,15 @@ public class AggregateStrategies implements ImplementationRuleFactory {
ExpressionUtils.collect(argumentsOfAggregateFunction, SlotReference.class::isInstance);
List<SlotReference> usedSlotInTable = (List<SlotReference>) (List) Project.findProject(aggUsedSlots,
(List<NamedExpression>) (List) olapScan.getOutput());
(List<NamedExpression>) (List) logicalScan.getOutput());
for (SlotReference slot : usedSlotInTable) {
Column column = slot.getColumn().get();
if (keysType == KeysType.AGG_KEYS && !column.isKey()) {
return canNotPush;
if (logicalScan instanceof LogicalOlapScan) {
KeysType keysType = ((LogicalOlapScan) logicalScan).getTable().getKeysType();
if (keysType == KeysType.AGG_KEYS && !column.isKey()) {
return canNotPush;
}
}
// The zone map max length of CharFamily is 512, do not
// over the length: https://github.com/apache/doris/pull/6293
@ -310,19 +338,41 @@ public class AggregateStrategies implements ImplementationRuleFactory {
}
}
PhysicalOlapScan physicalOlapScan = (PhysicalOlapScan) new LogicalOlapScanToPhysicalOlapScan()
.build()
.transform(olapScan, cascadesContext)
.get(0);
if (project != null) {
return aggregate.withChildren(ImmutableList.of(
if (logicalScan instanceof LogicalOlapScan) {
PhysicalOlapScan physicalScan = (PhysicalOlapScan) new LogicalOlapScanToPhysicalOlapScan()
.build()
.transform((LogicalOlapScan) logicalScan, cascadesContext)
.get(0);
if (project != null) {
return aggregate.withChildren(ImmutableList.of(
project.withChildren(
ImmutableList.of(new PhysicalStorageLayerAggregate(physicalOlapScan, mergeOp)))
));
ImmutableList.of(new PhysicalStorageLayerAggregate(physicalScan, mergeOp)))
));
} else {
return aggregate.withChildren(ImmutableList.of(
new PhysicalStorageLayerAggregate(physicalScan, mergeOp)
));
}
} else if (logicalScan instanceof LogicalFileScan) {
PhysicalFileScan physicalScan = (PhysicalFileScan) new LogicalFileScanToPhysicalFileScan()
.build()
.transform((LogicalFileScan) logicalScan, cascadesContext)
.get(0);
if (project != null) {
return aggregate.withChildren(ImmutableList.of(
project.withChildren(
ImmutableList.of(new PhysicalStorageLayerAggregate(physicalScan, mergeOp)))
));
} else {
return aggregate.withChildren(ImmutableList.of(
new PhysicalStorageLayerAggregate(physicalScan, mergeOp)
));
}
} else {
return aggregate.withChildren(ImmutableList.of(
new PhysicalStorageLayerAggregate(physicalOlapScan, mergeOp)
));
return canNotPush;
}
}

41
fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.DescriptorTable;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.PartitionNames;
@ -77,7 +78,6 @@ import org.apache.doris.thrift.TPaloScanRange;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.TPushAggOp;
import org.apache.doris.thrift.TScanRange;
import org.apache.doris.thrift.TScanRangeLocation;
import org.apache.doris.thrift.TScanRangeLocations;
@ -170,7 +170,6 @@ public class OlapScanNode extends ScanNode {
private boolean useTopnOpt = false;
private TPushAggOp pushDownAggNoGroupingOp = null;
// List of tablets will be scanned by current olap_scan_node
private ArrayList<Long> scanTabletIds = Lists.newArrayList();
@ -223,9 +222,6 @@ public class OlapScanNode extends ScanNode {
this.reasonOfPreAggregation + " " + reason;
}
public void setPushDownAggNoGrouping(TPushAggOp pushDownAggNoGroupingOp) {
this.pushDownAggNoGroupingOp = pushDownAggNoGroupingOp;
}
public boolean isPreAggregation() {
return isPreAggregation;
@ -1363,9 +1359,10 @@ public class OlapScanNode extends ScanNode {
msg.olap_scan_node.setTableName(olapTable.getName());
msg.olap_scan_node.setEnableUniqueKeyMergeOnWrite(olapTable.getEnableUniqueKeyMergeOnWrite());
if (pushDownAggNoGroupingOp != null) {
msg.olap_scan_node.setPushDownAggTypeOpt(pushDownAggNoGroupingOp);
}
msg.setPushDownAggTypeOpt(pushDownAggNoGroupingOp);
msg.olap_scan_node.setPushDownAggTypeOpt(pushDownAggNoGroupingOp);
// In TOlapScanNode , pushDownAggNoGroupingOp field is deprecated.
if (outputColumnUniqueIds != null) {
msg.olap_scan_node.setOutputColumnUniqueIds(outputColumnUniqueIds);
@ -1580,4 +1577,32 @@ public class OlapScanNode extends ScanNode {
olapTable.getId(), selectedIndexId == -1 ? olapTable.getBaseIndexId() : selectedIndexId,
scanReplicaIds);
}
@Override
public boolean pushDownAggNoGrouping(FunctionCallExpr aggExpr) {
KeysType type = getOlapTable().getKeysType();
if (type == KeysType.UNIQUE_KEYS || type == KeysType.PRIMARY_KEYS) {
return false;
}
String aggFunctionName = aggExpr.getFnName().getFunction();
if (aggFunctionName.equalsIgnoreCase("COUNT") && type != KeysType.DUP_KEYS) {
return false;
}
return true;
}
@Override
public boolean pushDownAggNoGroupingCheckCol(FunctionCallExpr aggExpr, Column col) {
KeysType type = getOlapTable().getKeysType();
// The value column of the agg does not support zone_map index.
if (type == KeysType.AGG_KEYS && !col.isKey()) {
return false;
}
return true;
}
}

17
fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
View File

@ -27,11 +27,13 @@ import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprId;
import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.FunctionName;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Function;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Type;
@ -46,6 +48,7 @@ import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TFunctionBinaryType;
import org.apache.doris.thrift.TPlan;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPushAggOp;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@ -1175,4 +1178,18 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
public void setCardinalityAfterFilter(long cardinalityAfterFilter) {
this.cardinalityAfterFilter = cardinalityAfterFilter;
}
protected TPushAggOp pushDownAggNoGroupingOp = TPushAggOp.NONE;
public void setPushDownAggNoGrouping(TPushAggOp pushDownAggNoGroupingOp) {
this.pushDownAggNoGroupingOp = pushDownAggNoGroupingOp;
}
public boolean pushDownAggNoGrouping(FunctionCallExpr aggExpr) {
return false;
}
public boolean pushDownAggNoGroupingCheckCol(FunctionCallExpr aggExpr, Column col) {
return false;
}
}

23
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -56,7 +56,6 @@ import org.apache.doris.catalog.AggregateFunction;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.PrimitiveType;
@ -423,16 +422,6 @@ public class SingleNodePlanner {
private void pushDownAggNoGrouping(AggregateInfo aggInfo, SelectStmt selectStmt, Analyzer analyzer, PlanNode root) {
do {
// TODO: Support other scan node in the future
if (!(root instanceof OlapScanNode)) {
break;
}
KeysType type = ((OlapScanNode) root).getOlapTable().getKeysType();
if (type == KeysType.UNIQUE_KEYS || type == KeysType.PRIMARY_KEYS) {
break;
}
if (CollectionUtils.isNotEmpty(root.getConjuncts())) {
break;
}
@ -457,7 +446,6 @@ public class SingleNodePlanner {
boolean aggExprValidate = true;
TPushAggOp aggOp = null;
for (FunctionCallExpr aggExpr : aggExprs) {
// Only support `min`, `max`, `count` and `count` only effective in dup table
String functionName = aggExpr.getFnName().getFunction();
if (!functionName.equalsIgnoreCase("MAX")
&& !functionName.equalsIgnoreCase("MIN")
@ -466,8 +454,7 @@ public class SingleNodePlanner {
break;
}
if (functionName.equalsIgnoreCase("COUNT")
&& type != KeysType.DUP_KEYS) {
if (!root.pushDownAggNoGrouping(aggExpr)) {
aggExprValidate = false;
break;
}
@ -512,8 +499,7 @@ public class SingleNodePlanner {
continue;
}
// The value column of the agg does not support zone_map index.
if (type == KeysType.AGG_KEYS && !col.isKey()) {
if (!root.pushDownAggNoGroupingCheckCol(aggExpr, col)) {
returnColumnValidate = false;
break;
}
@ -556,8 +542,7 @@ public class SingleNodePlanner {
break;
}
OlapScanNode olapNode = (OlapScanNode) root;
olapNode.setPushDownAggNoGrouping(aggOp);
root.setPushDownAggNoGrouping(aggOp);
} while (false);
}
@ -2210,7 +2195,7 @@ public class SingleNodePlanner {
/**
* Create a tree of PlanNodes for the given tblRef, which can be a BaseTableRef,
* TableValuedFunctionRef, CollectionTableRef or an InlineViewRef.
* CollectionTableRef or an InlineViewRef.
* <p>
* 'fastPartitionKeyScans' indicates whether to try to produce the slots with
* metadata instead of table scans. Only applicable to BaseTableRef which is also

2
fe/fe-core/src/main/java/org/apache/doris/planner/external/FileScanNode.java vendored
View File

@ -76,6 +76,8 @@ public abstract class FileScanNode extends ExternalScanNode {
@Override
protected void toThrift(TPlanNode planNode) {
planNode.setPushDownAggTypeOpt(pushDownAggNoGroupingOp);
planNode.setNodeType(TPlanNodeType.FILE_SCAN_NODE);
TFileScanNode fileScanNode = new TFileScanNode();
fileScanNode.setTupleId(desc.getId().asInt());

25
fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java vendored
View File

@ -17,6 +17,7 @@
package org.apache.doris.planner.external;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
@ -67,13 +68,13 @@ public class HiveScanNode extends FileQueryScanNode {
public static final String PROP_FIELD_DELIMITER = "field.delim";
public static final String DEFAULT_FIELD_DELIMITER = "\1"; // "\x01"
public static final String PROP_LINE_DELIMITER = "line.delim";
public static final String DEFAULT_LINE_DELIMITER = "\n";
public static final String PROP_ARRAY_DELIMITER_HIVE2 = "colelction.delim";
public static final String PROP_ARRAY_DELIMITER_HIVE3 = "collection.delim";
public static final String DEFAULT_ARRAY_DELIMITER = "\2";
protected final HMSExternalTable hmsTable;
private HiveTransaction hiveTransaction = null;
@ -105,12 +106,11 @@ public class HiveScanNode extends FileQueryScanNode {
for (SlotDescriptor slot : desc.getSlots()) {
if (slot.getType().isMapType() || slot.getType().isStructType()) {
throw new UserException("For column `" + slot.getColumn().getName()
+ "`, The column types MAP/STRUCT are not supported yet"
+ " for text input format of Hive. ");
+ "`, The column types MAP/STRUCT are not supported yet"
+ " for text input format of Hive. ");
}
}
}
if (hmsTable.isHiveTransactionalTable()) {
this.hiveTransaction = new HiveTransaction(DebugUtil.printId(ConnectContext.get().queryId()),
ConnectContext.get().getQualifiedUser(), hmsTable, hmsTable.isFullAcidTable());
@ -151,7 +151,7 @@ public class HiveScanNode extends FileQueryScanNode {
partitionValuesList.add(listPartitionItem.getItems().get(0).getPartitionValuesAsStringList());
}
List<HivePartition> allPartitions =
cache.getAllPartitions(hmsTable.getDbName(), hmsTable.getName(), partitionValuesList);
cache.getAllPartitions(hmsTable.getDbName(), hmsTable.getName(), partitionValuesList);
if (ConnectContext.get().getExecutor() != null) {
ConnectContext.get().getExecutor().getSummaryProfile().setGetPartitionsFinishTime();
}
@ -310,4 +310,19 @@ public class HiveScanNode extends FileQueryScanNode {
}
params.setSlotNameToSchemaPos(columnNameToPosition);
}
@Override
public boolean pushDownAggNoGrouping(FunctionCallExpr aggExpr) {
String aggFunctionName = aggExpr.getFnName().getFunction();
if (aggFunctionName.equalsIgnoreCase("COUNT")) {
return true;
}
return false;
}
@Override
public boolean pushDownAggNoGroupingCheckCol(FunctionCallExpr aggExpr, Column col) {
return !col.isAllowNull();
}
}