From a4e793752fccef8e49d98d6992df3ed352eb0942 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Thu, 1 Aug 2024 18:56:19 +0800 Subject: [PATCH] [bugfix](iceberg)revert count(*) directly returned by fe for 2.1 (#38566) (#38655) bp: #38566 --- .../apache/doris/datasource/FileScanNode.java | 1 - .../iceberg/source/IcebergScanNode.java | 13 +++++----- .../apache/doris/nereids/NereidsPlanner.java | 24 +------------------ .../apache/doris/planner/OriginalPlanner.java | 16 ------------- 4 files changed, 8 insertions(+), 46 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index d202e01f52..f08fb0a503 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -71,7 +71,6 @@ public abstract class FileScanNode extends ExternalScanNode { protected long totalPartitionNum = 0; protected long readPartitionNum = 0; protected long fileSplitSize; - public long rowCount = 0; public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType, boolean needCheckColumnPriv) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index bb200fdf02..38d9c8e821 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -75,6 +75,7 @@ import org.apache.iceberg.util.TableScanUtil; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -209,12 +210,6 @@ public class IcebergScanNode extends FileQueryScanNode { HashSet partitionPathSet = new HashSet<>(); boolean isPartitionedTable = icebergTable.spec().isPartitioned(); - long rowCount = getCountFromSnapshot(); - if (getPushDownAggNoGroupingOp().equals(TPushAggOp.COUNT) && rowCount >= 0) { - this.rowCount = rowCount; - return new ArrayList<>(); - } - CloseableIterable fileScanTasks = TableScanUtil.splitFiles(scan.planFiles(), splitSize); try (CloseableIterable combinedScanTasks = TableScanUtil.planTasks(fileScanTasks, splitSize, 1, 0)) { @@ -268,6 +263,12 @@ public class IcebergScanNode extends FileQueryScanNode { throw new UserException(e.getMessage(), e.getCause()); } + TPushAggOp aggOp = getPushDownAggNoGroupingOp(); + if (aggOp.equals(TPushAggOp.COUNT) && getCountFromSnapshot() >= 0) { + // we can create a special empty split and skip the plan process + return splits.isEmpty() ? splits : Collections.singletonList(splits.get(0)); + } + readPartitionNum = partitionPathSet.size(); return splits; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 12bb8c3623..67f4bb73b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -26,7 +26,6 @@ import org.apache.doris.common.FormatOptions; import org.apache.doris.common.NereidsException; import org.apache.doris.common.Pair; import org.apache.doris.common.profile.SummaryProfile; -import org.apache.doris.datasource.iceberg.source.IcebergScanNode; import org.apache.doris.mysql.FieldInfo; import org.apache.doris.nereids.CascadesContext.Lock; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -54,18 +53,14 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; -import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalSqlCache; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.RuntimeFilter; import org.apache.doris.planner.ScanNode; -import org.apache.doris.qe.CommonResultSet; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ResultSet; -import org.apache.doris.qe.ResultSetMetaData; import org.apache.doris.qe.SessionVariable; import com.google.common.annotations.VisibleForTesting; @@ -75,7 +70,6 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -572,23 +566,7 @@ public class NereidsPlanner extends Planner { } } - if (physicalPlan instanceof PhysicalResultSink - && physicalPlan.child(0) instanceof PhysicalHashAggregate && !getScanNodes().isEmpty() - && getScanNodes().get(0) instanceof IcebergScanNode) { - List columns = Lists.newArrayList(); - NamedExpression output = physicalPlan.getOutput().get(0); - columns.add(new Column(output.getName(), output.getDataType().toCatalogDataType())); - if (((IcebergScanNode) getScanNodes().get(0)).rowCount > 0) { - ResultSetMetaData metadata = new CommonResultSet.CommonResultSetMetaData(columns); - ResultSet resultSet = new CommonResultSet(metadata, Collections.singletonList( - Lists.newArrayList(String.valueOf(((IcebergScanNode) getScanNodes().get(0)).rowCount)))); - // only support one iceberg scan node and one count, e.g. select count(*) from icetbl; - return Optional.of(resultSet); - } - return Optional.empty(); - } else { - return Optional.empty(); - } + return Optional.empty(); } private void setFormatOptions() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 70e442546c..2657232db8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -41,7 +41,6 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; import org.apache.doris.common.FormatOptions; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.iceberg.source.IcebergScanNode; import org.apache.doris.nereids.PlannerHook; import org.apache.doris.qe.CommonResultSet; import org.apache.doris.qe.ConnectContext; @@ -642,21 +641,6 @@ public class OriginalPlanner extends Planner { List columns = new ArrayList<>(selectItems.size()); List columnLabels = parsedSelectStmt.getColLabels(); List data = new ArrayList<>(); - if ((singleNodePlanner.getScanNodes().size() > 0 && singleNodePlanner.getScanNodes().get(0) - instanceof IcebergScanNode) && (((IcebergScanNode) getScanNodes().get(0)).rowCount > 0)) { - SelectListItem item = selectItems.get(0); - Expr expr = item.getExpr(); - String columnName = columnLabels.get(0); - columns.add(new Column(columnName, expr.getType())); - data.add(String.valueOf(((IcebergScanNode) getScanNodes().get(0)).rowCount)); - ResultSetMetaData metadata = new CommonResultSet.CommonResultSetMetaData(columns); - ResultSet resultSet = new CommonResultSet(metadata, Collections.singletonList(data)); - // only support one iceberg scan node and one count, e.g. select count(*) from icetbl; - return Optional.of(resultSet); - } - if (!parsedSelectStmt.getTableRefs().isEmpty()) { - return Optional.empty(); - } FormatOptions options = FormatOptions.getDefault(); for (int i = 0; i < selectItems.size(); i++) { SelectListItem item = selectItems.get(i);