database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](nereids) Abstract interface of statistics framework for new optimizer reuse (#10240)

Browse Source 
As the statistics framework could not be reused by new optmizer before, so I abstract some interface to make it reusable.

1. Make Slot extends the Id
2. Add new interface:ExprStats,PlanStats
3. Move definition of PlanNode.NodeType to statistics sub-directory
This commit is contained in:
Kikyou1997
2022-06-20 15:16:06 +08:00
committed by GitHub
parent 57327e6236
commit 8531dcb885
45 changed files with 395 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.statistics.ExprStats;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprOpcode;
@ -60,7 +61,8 @@ import java.util.Set;
/**
* Root of the expr node hierarchy.
*/
public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneable, Writable {
public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneable, Writable, ExprStats {
private static final Logger LOG = LogManager.getLogger(Expr.class);
// Name of the function that needs to be implemented by every Expr that

38
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java
View File

@ -17,23 +17,18 @@
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.common.Id;
import org.apache.doris.common.IdGenerator;
import java.util.Objects;
import java.util.UUID;
/**
* UUID for Expression in Nereids.
*/
public class ExprId {
private final long id;
private final UUID jvmId;
public class ExprId extends Id<ExprId> {
public ExprId(long id, UUID jvmId) {
this.id = id;
this.jvmId = jvmId;
}
public long getId() {
return id;
public ExprId(int id) {
super(id);
}
@Override
@ -45,12 +40,29 @@ public class ExprId {
return false;
}
ExprId exprId = (ExprId) o;
return id == exprId.id && jvmId.equals(exprId.jvmId);
return id == exprId.id;
}
/**
* Should be only called by {@link org.apache.doris.nereids.trees.expressions.NamedExpressionUtil}.
*/
public static IdGenerator<ExprId> createGenerator() {
return new IdGenerator<ExprId>() {
@Override
public ExprId getNextId() {
return new ExprId(nextId++);
}
@Override
public ExprId getMaxId() {
return new ExprId(nextId++);
}
};
}
@Override
public int hashCode() {
return Objects.hash(id, jvmId);
return Objects.hash(id);
}
@Override

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpressionUtil.java
View File

@ -17,8 +17,7 @@
package org.apache.doris.nereids.trees.expressions;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.doris.common.IdGenerator;
/**
* The util of named expression.
@ -27,10 +26,9 @@ public class NamedExpressionUtil {
/**
* Tool class for generate next ExprId.
*/
private static final UUID JVM_ID = UUID.randomUUID();
private static final AtomicLong CURRENT_ID = new AtomicLong();
private static IdGenerator<ExprId> idIdGenerator = ExprId.createGenerator();
public static ExprId newExprId() {
return new ExprId(CURRENT_ID.getAndIncrement(), JVM_ID);
return idIdGenerator.getNextId();
}
}

36
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
View File

@ -22,10 +22,14 @@ import org.apache.doris.nereids.operators.plans.PlanOperator;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.AbstractTreeNode;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.statistics.ExprStats;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsDeriveResult;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@ -37,6 +41,8 @@ public abstract class AbstractPlan<OP_TYPE extends PlanOperator>
extends AbstractTreeNode<Plan> implements Plan {
public final OP_TYPE operator;
protected StatsDeriveResult statsDeriveResult;
protected long limit;
protected final LogicalProperties logicalProperties;
@ -92,4 +98,34 @@ public abstract class AbstractPlan<OP_TYPE extends PlanOperator>
treeString(lines, depth + 1, newLasts, children.get(i));
}
}
@Override
public List<StatsDeriveResult> getChildrenStats() {
return Collections.emptyList();
}
@Override
public StatsDeriveResult getStatsDeriveResult() {
return statsDeriveResult;
}
@Override
public StatisticalType getStatisticalType() {
return null;
}
@Override
public void setStatsDeriveResult(StatsDeriveResult result) {
this.statsDeriveResult = result;
}
@Override
public long getLimit() {
return limit;
}
@Override
public List<? extends ExprStats> getConjuncts() {
return Collections.emptyList();
}
}

34
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceHolderPlan.java
View File

@ -24,6 +24,9 @@ import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.NodeType;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.logical.LogicalLeafPlan;
import org.apache.doris.statistics.ExprStats;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsDeriveResult;
import com.google.common.base.Preconditions;
@ -81,4 +84,35 @@ public class PlaceHolderPlan extends LogicalLeafPlan<PlaceHolderPlan.PlaceHolder
Preconditions.checkArgument(children.size() == 0);
return new PlaceHolderPlan(logicalProperties);
}
@Override
public List<StatsDeriveResult> getChildrenStats() {
throw new RuntimeException("Unsupported Method");
}
@Override
public StatsDeriveResult getStatsDeriveResult() {
throw new RuntimeException("Unsupported Method");
}
@Override
public StatisticalType getStatisticalType() {
throw new RuntimeException("Unsupported Method");
}
@Override
public void setStatsDeriveResult(StatsDeriveResult result) {
throw new RuntimeException("Unsupported Method");
}
@Override
public long getLimit() {
throw new RuntimeException("Unsupported Method");
}
@Override
public List<? extends ExprStats> getConjuncts() {
throw new RuntimeException("Unsupported Method");
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
View File

@ -21,13 +21,14 @@ import org.apache.doris.nereids.operators.plans.PlanOperator;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.TreeNode;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.statistics.PlanStats;
import java.util.List;
/**
* Abstract class for all plan node.
*/
public interface Plan extends TreeNode<Plan> {
public interface Plan extends TreeNode<Plan>, PlanStats {
PlanOperator getOperator();

1
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java
View File

@ -77,4 +77,5 @@ public abstract class AbstractPhysicalPlan<OP_TYPE extends PhysicalOperator>
public PhysicalProperties getPhysicalProperties() {
return physicalProperties;
}
}

5
fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.analysis.SlotId;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TAggregationNode;
import org.apache.doris.thrift.TExplainLevel;
@ -65,7 +66,7 @@ public class AggregationNode extends PlanNode {
* isIntermediate is true if it is a slave node in a 2-part agg plan.
*/
public AggregationNode(PlanNodeId id, PlanNode input, AggregateInfo aggInfo) {
super(id, aggInfo.getOutputTupleId().asList(), "AGGREGATE", NodeType.AGG_NODE);
super(id, aggInfo.getOutputTupleId().asList(), "AGGREGATE", StatisticalType.AGG_NODE);
this.aggInfo = aggInfo;
this.children.add(input);
this.needsFinalize = true;
@ -76,7 +77,7 @@ public class AggregationNode extends PlanNode {
* Copy c'tor used in clone().
*/
private AggregationNode(PlanNodeId id, AggregationNode src) {
super(id, src, "AGGREGATE", NodeType.AGG_NODE);
super(id, src, "AGGREGATE", StatisticalType.AGG_NODE);
aggInfo = src.aggInfo;
needsFinalize = src.needsFinalize;
}

3
fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.OrderByElement;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TAnalyticNode;
import org.apache.doris.thrift.TExplainLevel;
@ -81,7 +82,7 @@ public class AnalyticEvalNode extends PlanNode {
AnalyticWindow analyticWindow, TupleDescriptor intermediateTupleDesc,
TupleDescriptor outputTupleDesc, ExprSubstitutionMap logicalToPhysicalSmap,
Expr partitionByEq, Expr orderByEq, TupleDescriptor bufferedTupleDesc) {
super(id, input.getTupleIds(), "ANALYTIC", NodeType.ANALYTIC_EVAL_NODE);
super(id, input.getTupleIds(), "ANALYTIC", StatisticalType.ANALYTIC_EVAL_NODE);
Preconditions.checkState(!tupleIds.contains(outputTupleDesc.getId()));
// we're materializing the input row augmented with the analytic output tuple
tupleIds.add(outputTupleDesc.getId());

3
fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.AssertNumRowsElement;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TAssertNumRowsNode;
import org.apache.doris.thrift.TExplainLevel;
@ -43,7 +44,7 @@ public class AssertNumRowsNode extends PlanNode {
private AssertNumRowsElement.Assertion assertion;
public AssertNumRowsNode(PlanNodeId id, PlanNode input, AssertNumRowsElement assertNumRowsElement) {
super(id, "ASSERT NUMBER OF ROWS", NodeType.ASSERT_NUM_ROWS_NODE);
super(id, "ASSERT NUMBER OF ROWS", StatisticalType.ASSERT_NUM_ROWS_NODE);
this.desiredNumOfRows = assertNumRowsElement.getDesiredNumOfRows();
this.subqueryString = assertNumRowsElement.getSubqueryString();
this.assertion = assertNumRowsElement.getAssertion();

7
fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
View File

@ -45,6 +45,7 @@ import org.apache.doris.load.loadv2.LoadTask;
import org.apache.doris.mysql.privilege.UserProperty;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.resource.Tag;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
import org.apache.doris.system.BeSelectionPolicy;
import org.apache.doris.task.LoadTaskInfo;
@ -148,7 +149,7 @@ public class BrokerScanNode extends LoadScanNode {
// For broker load and external broker table
public BrokerScanNode(PlanNodeId id, TupleDescriptor destTupleDesc, String planNodeName,
List<List<TBrokerFileStatus>> fileStatusesList, int filesAdded) {
super(id, destTupleDesc, planNodeName, NodeType.BROKER_SCAN_NODE);
super(id, destTupleDesc, planNodeName, StatisticalType.BROKER_SCAN_NODE);
this.fileStatusesList = fileStatusesList;
this.filesAdded = filesAdded;
if (ConnectContext.get() != null) {
@ -158,8 +159,8 @@ public class BrokerScanNode extends LoadScanNode {
// For hive and iceberg scan node
public BrokerScanNode(PlanNodeId id, TupleDescriptor destTupleDesc, String planNodeName,
List<List<TBrokerFileStatus>> fileStatusesList, int filesAdded, NodeType nodeType) {
super(id, destTupleDesc, planNodeName, nodeType);
List<List<TBrokerFileStatus>> fileStatusesList, int filesAdded, StatisticalType statisticalType) {
super(id, destTupleDesc, planNodeName, statisticalType);
this.fileStatusesList = fileStatusesList;
this.filesAdded = filesAdded;
if (ConnectContext.get() != null) {

3
fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.TableRef;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
@ -41,7 +42,7 @@ public class CrossJoinNode extends PlanNode {
private final TableRef innerRef;
public CrossJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef innerRef) {
super(id, "CROSS JOIN", NodeType.CROSS_JOIN_NODE);
super(id, "CROSS JOIN", StatisticalType.CROSS_JOIN_NODE);
this.innerRef = innerRef;
tupleIds.addAll(outer.getTupleIds());
tupleIds.addAll(inner.getTupleIds());

3
fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@ -40,7 +41,7 @@ public class EmptySetNode extends PlanNode {
private static final Logger LOG = LogManager.getLogger(EmptySetNode.class);
public EmptySetNode(PlanNodeId id, ArrayList<TupleId> tupleIds) {
super(id, tupleIds, "EMPTYSET", NodeType.EMPTY_SET_NODE);
super(id, tupleIds, "EMPTYSET", StatisticalType.EMPTY_SET_NODE);
Preconditions.checkArgument(tupleIds.size() > 0);
}

3
fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
View File

@ -30,6 +30,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.external.elasticsearch.EsShardPartitions;
import org.apache.doris.external.elasticsearch.EsShardRouting;
import org.apache.doris.external.elasticsearch.EsTablePartitions;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TEsScanNode;
import org.apache.doris.thrift.TEsScanRange;
@ -71,7 +72,7 @@ public class EsScanNode extends ScanNode {
boolean isFinalized = false;
public EsScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) {
super(id, desc, planNodeName, NodeType.ES_SCAN_NODE);
super(id, desc, planNodeName, StatisticalType.ES_SCAN_NODE);
table = (EsTable) (desc.getTable());
esTablePartitions = table.getEsTablePartitions();
}

3
fe/fe-core/src/main/java/org/apache/doris/planner/ExceptNode.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@ -31,7 +32,7 @@ public class ExceptNode extends SetOperationNode {
protected ExceptNode(PlanNodeId id, TupleId tupleId,
List<Expr> setOpResultExprs, boolean isInSubplan) {
super(id, tupleId, "EXCEPT", setOpResultExprs, isInSubplan, NodeType.EXCEPT_NODE);
super(id, tupleId, "EXCEPT", setOpResultExprs, isInSubplan, StatisticalType.EXCEPT_NODE);
}
@Override

3
fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.analysis.SortInfo;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TExchangeNode;
import org.apache.doris.thrift.TPlanNode;
@ -71,7 +72,7 @@ public class ExchangeNode extends PlanNode {
* need to compute the cardinality here.
*/
public ExchangeNode(PlanNodeId id, PlanNode inputNode, boolean copyConjuncts) {
super(id, inputNode, EXCHANGE_NODE, NodeType.EXCHANGE_NODE);
super(id, inputNode, EXCHANGE_NODE, StatisticalType.EXCHANGE_NODE);
offset = 0;
children.add(inputNode);
if (!copyConjuncts) {

5
fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
View File

@ -38,6 +38,7 @@ import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TEqJoinCondition;
import org.apache.doris.thrift.TExplainLevel;
@ -89,7 +90,7 @@ public class HashJoinNode extends PlanNode {
*/
public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef innerRef, List<Expr> eqJoinConjuncts,
List<Expr> otherJoinConjuncts) {
super(id, "HASH JOIN", NodeType.HASH_JOIN_NODE);
super(id, "HASH JOIN", StatisticalType.HASH_JOIN_NODE);
Preconditions.checkArgument(eqJoinConjuncts != null && !eqJoinConjuncts.isEmpty());
Preconditions.checkArgument(otherJoinConjuncts != null);
tblRefIds.addAll(outer.getTblRefIds());
@ -148,7 +149,7 @@ public class HashJoinNode extends PlanNode {
*/
public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator joinOp, List<Expr> eqJoinConjuncts,
List<Expr> otherJoinConjuncts) {
super(id, "HASH JOIN", NodeType.HASH_JOIN_NODE);
super(id, "HASH JOIN", StatisticalType.HASH_JOIN_NODE);
Preconditions.checkArgument(eqJoinConjuncts != null && !eqJoinConjuncts.isEmpty());
Preconditions.checkArgument(otherJoinConjuncts != null);
tblRefIds.addAll(outer.getTblRefIds());

3
fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.catalog.HiveTable;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
import org.apache.doris.load.BrokerFileGroup;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TBrokerFileStatus;
import org.apache.doris.thrift.TExplainLevel;
@ -101,7 +102,7 @@ public class HiveScanNode extends BrokerScanNode {
public HiveScanNode(PlanNodeId id, TupleDescriptor destTupleDesc, String planNodeName,
List<List<TBrokerFileStatus>> fileStatusesList, int filesAdded) {
super(id, destTupleDesc, planNodeName, fileStatusesList, filesAdded, NodeType.HIVE_SCAN_NODE);
super(id, destTupleDesc, planNodeName, fileStatusesList, filesAdded, StatisticalType.HIVE_SCAN_NODE);
this.hiveTable = (HiveTable) destTupleDesc.getTable();
}

3
fe/fe-core/src/main/java/org/apache/doris/planner/IcebergScanNode.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.catalog.IcebergTable;
import org.apache.doris.common.UserException;
import org.apache.doris.external.iceberg.util.IcebergUtils;
import org.apache.doris.load.BrokerFileGroup;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TBrokerFileStatus;
import org.apache.doris.thrift.TExplainLevel;
@ -46,7 +47,7 @@ public class IcebergScanNode extends BrokerScanNode {
public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName,
List<List<TBrokerFileStatus>> fileStatusesList, int filesAdded) {
super(id, desc, planNodeName, fileStatusesList, filesAdded, NodeType.ICEBERG_SCAN_NODE);
super(id, desc, planNodeName, fileStatusesList, filesAdded, StatisticalType.ICEBERG_SCAN_NODE);
icebergTable = (IcebergTable) desc.getTable();
}

7
fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
View File

@ -38,6 +38,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.load.loadv2.LoadTask;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TBrokerScanNode;
import org.apache.doris.thrift.TBrokerScanRangeParams;
import org.apache.doris.thrift.TPlanNode;
@ -55,11 +56,11 @@ public abstract class LoadScanNode extends ScanNode {
protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND;
public LoadScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) {
super(id, desc, planNodeName, NodeType.LOAD_SCAN_NODE);
super(id, desc, planNodeName, StatisticalType.LOAD_SCAN_NODE);
}
public LoadScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, NodeType nodeType) {
super(id, desc, planNodeName, nodeType);
public LoadScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType) {
super(id, desc, planNodeName, statisticalType);
}
protected void initAndSetWhereExpr(Expr whereExpr, TupleDescriptor tupleDesc,

3
fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TMySQLScanNode;
@ -56,7 +57,7 @@ public class MysqlScanNode extends ScanNode {
* Constructs node to scan given data files of table 'tbl'.
*/
public MysqlScanNode(PlanNodeId id, TupleDescriptor desc, MysqlTable tbl) {
super(id, desc, "SCAN MYSQL", NodeType.MYSQL_SCAN_NODE);
super(id, desc, "SCAN MYSQL", StatisticalType.MYSQL_SCAN_NODE);
tblName = "`" + tbl.getMysqlTableName() + "`";
}

3
fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TOdbcScanNode;
@ -73,7 +74,7 @@ public class OdbcScanNode extends ScanNode {
* Constructs node to scan given data files of table 'tbl'.
*/
public OdbcScanNode(PlanNodeId id, TupleDescriptor desc, OdbcTable tbl) {
super(id, desc, "SCAN ODBC", NodeType.ODBC_SCAN_NODE);
super(id, desc, "SCAN ODBC", StatisticalType.ODBC_SCAN_NODE);
connectString = tbl.getConnectString();
odbcType = tbl.getOdbcTableType();
tblName = OdbcTable.databaseProperName(odbcType, tbl.getOdbcTableName());

3
fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -53,6 +53,7 @@ import org.apache.doris.common.util.Util;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.resource.Tag;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TExplainLevel;
@ -146,7 +147,7 @@ public class OlapScanNode extends ScanNode {
// Constructs node to scan given data files of table 'tbl'.
public OlapScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) {
super(id, desc, planNodeName, NodeType.OLAP_SCAN_NODE);
super(id, desc, planNodeName, StatisticalType.OLAP_SCAN_NODE);
olapTable = (OlapTable) desc.getTable();
}

61
fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
View File

@ -36,6 +36,8 @@ import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.statistics.PlanStats;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsDeriveResult;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TFunctionBinaryType;
@ -71,7 +73,7 @@ import java.util.Set;
* this node, ie, they only reference tuples materialized by this node or one of
* its children (= are bound by tupleIds).
*/
public abstract class PlanNode extends TreeNode<PlanNode> {
public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
private static final Logger LOG = LogManager.getLogger(PlanNode.class);
protected String planNodeName;
@ -137,10 +139,11 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
protected List<SlotId> outputSlotIds;
protected NodeType nodeType = NodeType.DEFAULT;
protected StatisticalType statisticalType = StatisticalType.DEFAULT;
protected StatsDeriveResult statsDeriveResult;
protected PlanNode(PlanNodeId id, ArrayList<TupleId> tupleIds, String planNodeName, NodeType nodeType) {
protected PlanNode(PlanNodeId id, ArrayList<TupleId> tupleIds, String planNodeName,
StatisticalType statisticalType) {
this.id = id;
this.limit = -1;
// make a copy, just to be on the safe side
@ -149,10 +152,10 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
this.cardinality = -1;
this.planNodeName = VectorizedUtil.isVectorized() ? "V" + planNodeName : planNodeName;
this.numInstances = 1;
this.nodeType = nodeType;
this.statisticalType = statisticalType;
}
protected PlanNode(PlanNodeId id, String planNodeName, NodeType nodeType) {
protected PlanNode(PlanNodeId id, String planNodeName, StatisticalType statisticalType) {
this.id = id;
this.limit = -1;
this.tupleIds = Lists.newArrayList();
@ -160,13 +163,13 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
this.cardinality = -1;
this.planNodeName = VectorizedUtil.isVectorized() ? "V" + planNodeName : planNodeName;
this.numInstances = 1;
this.nodeType = nodeType;
this.statisticalType = statisticalType;
}
/**
* Copy ctor. Also passes in new id.
*/
protected PlanNode(PlanNodeId id, PlanNode node, String planNodeName, NodeType nodeType) {
protected PlanNode(PlanNodeId id, PlanNode node, String planNodeName, StatisticalType statisticalType) {
this.id = id;
this.limit = node.limit;
this.tupleIds = Lists.newArrayList(node.tupleIds);
@ -177,36 +180,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
this.compactData = node.compactData;
this.planNodeName = VectorizedUtil.isVectorized() ? "V" + planNodeName : planNodeName;
this.numInstances = 1;
this.nodeType = nodeType;
}
public enum NodeType {
DEFAULT,
AGG_NODE,
ANALYTIC_EVAL_NODE,
ASSERT_NUM_ROWS_NODE,
BROKER_SCAN_NODE,
CROSS_JOIN_NODE,
EMPTY_SET_NODE,
ES_SCAN_NODE,
EXCEPT_NODE,
EXCHANGE_NODE,
HASH_JOIN_NODE,
HIVE_SCAN_NODE,
ICEBERG_SCAN_NODE,
INTERSECT_NODE,
LOAD_SCAN_NODE,
MYSQL_SCAN_NODE,
ODBC_SCAN_NODE,
OLAP_SCAN_NODE,
REPEAT_NODE,
SELECT_NODE,
SET_OPERATION_NODE,
SCHEMA_SCAN_NODE,
SORT_NODE,
STREAM_LOAD_SCAN_NODE,
TABLE_FUNCTION_NODE,
UNION_NODE,
this.statisticalType = statisticalType;
}
public String getPlanNodeName() {
@ -217,8 +191,8 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
return statsDeriveResult;
}
public NodeType getNodeType() {
return nodeType;
public StatisticalType getStatisticalType() {
return statisticalType;
}
public void setStatsDeriveResult(StatsDeriveResult statsDeriveResult) {
@ -353,6 +327,15 @@ public abstract class PlanNode extends TreeNode<PlanNode> {
return conjuncts;
}
@Override
public List<StatsDeriveResult> getChildrenStats() {
List<StatsDeriveResult> statsDeriveResultList = Lists.newArrayList();
for (PlanNode child : children) {
statsDeriveResultList.add(child.getStatsDeriveResult());
}
return statsDeriveResultList;
}
void initCompoundPredicate(Expr expr) {
if (expr instanceof CompoundPredicate) {
CompoundPredicate compoundPredicate = (CompoundPredicate) expr;

5
fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java
View File

@ -30,6 +30,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.analysis.VirtualSlotRef;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
@ -67,7 +68,7 @@ public class RepeatNode extends PlanNode {
private GroupByClause groupByClause;
protected RepeatNode(PlanNodeId id, PlanNode input, GroupingInfo groupingInfo, GroupByClause groupByClause) {
super(id, input.getTupleIds(), "REPEAT_NODE", NodeType.REPEAT_NODE);
super(id, input.getTupleIds(), "REPEAT_NODE", StatisticalType.REPEAT_NODE);
this.children.add(input);
this.groupingInfo = groupingInfo;
this.input = input;
@ -78,7 +79,7 @@ public class RepeatNode extends PlanNode {
// only for unittest
protected RepeatNode(PlanNodeId id, PlanNode input, List<Set<SlotId>> repeatSlotIdList,
TupleDescriptor outputTupleDesc, List<List<Long>> groupingList) {
super(id, input.getTupleIds(), "REPEAT_NODE", NodeType.REPEAT_NODE);
super(id, input.getTupleIds(), "REPEAT_NODE", StatisticalType.REPEAT_NODE);
this.children.add(input);
this.repeatSlotIdList = buildIdSetList(repeatSlotIdList);
this.groupingList = groupingList;

5
fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
View File

@ -35,6 +35,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TScanRangeLocations;
@ -63,8 +64,8 @@ public abstract class ScanNode extends PlanNode {
protected String sortColumn = null;
protected Analyzer analyzer;
public ScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, NodeType nodeType) {
super(id, desc.getId().asList(), planNodeName, nodeType);
public ScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType) {
super(id, desc.getId().asList(), planNodeName, statisticalType);
this.desc = desc;
}

3
fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
import org.apache.doris.thrift.TScanRangeLocations;
@ -56,7 +57,7 @@ public class SchemaScanNode extends ScanNode {
* Constructs node to scan given data files of table 'tbl'.
*/
public SchemaScanNode(PlanNodeId id, TupleDescriptor desc) {
super(id, desc, "SCAN SCHEMA", NodeType.SCHEMA_SCAN_NODE);
super(id, desc, "SCAN SCHEMA", StatisticalType.SCHEMA_SCAN_NODE);
this.tableName = desc.getTable().getName();
}

5
fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java
View File

@ -23,6 +23,7 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.Expr;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
@ -40,13 +41,13 @@ public class SelectNode extends PlanNode {
private static final Logger LOG = LogManager.getLogger(SelectNode.class);
protected SelectNode(PlanNodeId id, PlanNode child) {
super(id, child.getTupleIds(), "SELECT", NodeType.SELECT_NODE);
super(id, child.getTupleIds(), "SELECT", StatisticalType.SELECT_NODE);
addChild(child);
this.nullableTupleIds = child.nullableTupleIds;
}
protected SelectNode(PlanNodeId id, PlanNode child, List<Expr> conjuncts) {
super(id, child.getTupleIds(), "SELECT", NodeType.SELECT_NODE);
super(id, child.getTupleIds(), "SELECT", StatisticalType.SELECT_NODE);
addChild(child);
this.tblRefIds = child.tblRefIds;
this.nullableTupleIds = child.nullableTupleIds;

13
fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.analysis.TupleId;
import org.apache.doris.common.CheckedMath;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TExceptNode;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TExpr;
@ -83,23 +84,23 @@ public abstract class SetOperationNode extends PlanNode {
protected final TupleId tupleId;
protected SetOperationNode(PlanNodeId id, TupleId tupleId, String planNodeName, NodeType nodeType) {
super(id, tupleId.asList(), planNodeName, nodeType);
protected SetOperationNode(PlanNodeId id, TupleId tupleId, String planNodeName, StatisticalType statisticalType) {
super(id, tupleId.asList(), planNodeName, statisticalType);
this.setOpResultExprs = Lists.newArrayList();
this.tupleId = tupleId;
this.isInSubplan = false;
}
protected SetOperationNode(PlanNodeId id, TupleId tupleId, String planNodeName,
List<Expr> setOpResultExprs, boolean isInSubplan, NodeType nodeType) {
super(id, tupleId.asList(), planNodeName, nodeType);
List<Expr> setOpResultExprs, boolean isInSubplan, StatisticalType statisticalType) {
super(id, tupleId.asList(), planNodeName, statisticalType);
this.setOpResultExprs = setOpResultExprs;
this.tupleId = tupleId;
this.isInSubplan = isInSubplan;
}
protected SetOperationNode(PlanNodeId id, TupleId tupleId, String planNodeName) {
super(id, tupleId.asList(), planNodeName, NodeType.SET_OPERATION_NODE);
super(id, tupleId.asList(), planNodeName, StatisticalType.SET_OPERATION_NODE);
this.setOpResultExprs = Lists.newArrayList();
this.tupleId = tupleId;
this.isInSubplan = false;
@ -107,7 +108,7 @@ public abstract class SetOperationNode extends PlanNode {
protected SetOperationNode(PlanNodeId id, TupleId tupleId, String planNodeName,
List<Expr> setOpResultExprs, boolean isInSubplan) {
super(id, tupleId.asList(), planNodeName, NodeType.SET_OPERATION_NODE);
super(id, tupleId.asList(), planNodeName, StatisticalType.SET_OPERATION_NODE);
this.setOpResultExprs = setOpResultExprs;
this.tupleId = tupleId;
this.isInSubplan = isInSubplan;

5
fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.SortInfo;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
@ -84,7 +85,7 @@ public class SortNode extends PlanNode {
public SortNode(PlanNodeId id, PlanNode input, SortInfo info, boolean useTopN,
boolean isDefaultLimit, long offset) {
super(id, useTopN ? "TOP-N" : "SORT", NodeType.SORT_NODE);
super(id, useTopN ? "TOP-N" : "SORT", StatisticalType.SORT_NODE);
this.info = info;
this.useTopN = useTopN;
this.isDefaultLimit = isDefaultLimit;
@ -100,7 +101,7 @@ public class SortNode extends PlanNode {
* Clone 'inputSortNode' for distributed Top-N
*/
public SortNode(PlanNodeId id, SortNode inputSortNode, PlanNode child) {
super(id, inputSortNode, inputSortNode.useTopN ? "TOP-N" : "SORT", NodeType.SORT_NODE);
super(id, inputSortNode, inputSortNode.useTopN ? "TOP-N" : "SORT", StatisticalType.SORT_NODE);
this.info = inputSortNode.info;
this.useTopN = inputSortNode.useTopN;
this.isDefaultLimit = inputSortNode.isDefaultLimit;

3
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
View File

@ -30,6 +30,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.load.Load;
import org.apache.doris.load.loadv2.LoadTask;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.task.LoadTaskInfo;
import org.apache.doris.thrift.TBrokerRangeDesc;
import org.apache.doris.thrift.TBrokerScanRange;
@ -75,7 +76,7 @@ public class StreamLoadScanNode extends LoadScanNode {
// used to construct for streaming loading
public StreamLoadScanNode(
TUniqueId loadId, PlanNodeId id, TupleDescriptor tupleDesc, Table dstTable, LoadTaskInfo taskInfo) {
super(id, tupleDesc, "StreamLoadScanNode", NodeType.STREAM_LOAD_SCAN_NODE);
super(id, tupleDesc, "StreamLoadScanNode", StatisticalType.STREAM_LOAD_SCAN_NODE);
this.loadId = loadId;
this.dstTable = dstTable;
this.taskInfo = taskInfo;

3
fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPlanNode;
@ -51,7 +52,7 @@ public class TableFunctionNode extends PlanNode {
private List<SlotId> outputSlotIds = Lists.newArrayList();
protected TableFunctionNode(PlanNodeId id, PlanNode inputNode, List<LateralViewRef> lateralViewRefs) {
super(id, "TABLE FUNCTION NODE", NodeType.TABLE_FUNCTION_NODE);
super(id, "TABLE FUNCTION NODE", StatisticalType.TABLE_FUNCTION_NODE);
tupleIds.addAll(inputNode.getTupleIds());
tblRefIds.addAll(inputNode.getTupleIds());
lateralViewTupleIds = lateralViewRefs.stream().map(e -> e.getDesc().getId())

5
fe/fe-core/src/main/java/org/apache/doris/planner/UnionNode.java
View File

@ -22,6 +22,7 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TPlanNodeType;
@ -29,12 +30,12 @@ import java.util.List;
public class UnionNode extends SetOperationNode {
protected UnionNode(PlanNodeId id, TupleId tupleId) {
super(id, tupleId, "UNION", NodeType.UNION_NODE);
super(id, tupleId, "UNION", StatisticalType.UNION_NODE);
}
protected UnionNode(PlanNodeId id, TupleId tupleId,
List<Expr> setOpResultExprs, boolean isInSubplan) {
super(id, tupleId, "UNION", setOpResultExprs, isInSubplan, NodeType.UNION_NODE);
super(id, tupleId, "UNION", setOpResultExprs, isInSubplan, StatisticalType.UNION_NODE);
}
@Override

5
fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java vendored
View File

@ -20,6 +20,7 @@ package org.apache.doris.planner.external;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TScanRangeLocations;
@ -34,8 +35,8 @@ import java.util.List;
*/
public class ExternalScanNode extends ScanNode {
public ExternalScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, NodeType nodeType) {
super(id, desc, planNodeName, nodeType);
public ExternalScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType) {
super(id, desc, planNodeName, statisticalType);
}
@Override

3
fe/fe-core/src/main/java/org/apache/doris/statistics/AggStatsDerive.java
View File

@ -20,7 +20,6 @@ package org.apache.doris.statistics;
import org.apache.doris.analysis.Expr;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.AggregationNode;
import org.apache.doris.planner.PlanNode;
import com.google.common.base.Preconditions;
import org.apache.logging.log4j.LogManager;
@ -37,7 +36,7 @@ public class AggStatsDerive extends BaseStatsDerive {
List<Expr> groupingExprs = new ArrayList<>();
@Override
public void init(PlanNode node) throws UserException {
public void init(PlanStats node) throws UserException {
Preconditions.checkState(node instanceof AggregationNode);
super.init(node);
groupingExprs.addAll(((AggregationNode) node).getAggInfo().getGroupingExprs());

27
fe/fe-core/src/main/java/org/apache/doris/statistics/BaseStatsDerive.java
View File

@ -18,9 +18,8 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.common.Id;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.PlanNode;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@ -42,19 +41,17 @@ public class BaseStatsDerive {
protected long rowCount = -1;
protected long limit = -1;
protected List<Expr> conjuncts = Lists.newArrayList();
protected List<ExprStats> conjuncts = Lists.newArrayList();
protected List<StatsDeriveResult> childrenStatsResult = Lists.newArrayList();
protected void init(PlanNode node) throws UserException {
protected void init(PlanStats node) throws UserException {
limit = node.getLimit();
conjuncts.addAll(node.getConjuncts());
for (PlanNode childNode : node.getChildren()) {
StatsDeriveResult result = childNode.getStatsDeriveResult();
for (StatsDeriveResult result : node.getChildrenStats()) {
if (result == null) {
throw new UserException(
"childNode statsDeriveResult is null, childNodeType is " + childNode.getNodeType()
+ "parentNodeType is " + node.getNodeType());
"childNode statsDeriveResult is null.");
}
childrenStatsResult.add(result);
}
@ -87,7 +84,7 @@ public class BaseStatsDerive {
}
protected double computeSelectivity() {
for (Expr expr : conjuncts) {
for (ExprStats expr : conjuncts) {
expr.setSelectivity();
}
return computeCombinedSelectivity(conjuncts);
@ -110,10 +107,10 @@ public class BaseStatsDerive {
* * additional selectivity into the final result.
* </p>
*/
protected double computeCombinedSelectivity(List<Expr> conjuncts) {
protected double computeCombinedSelectivity(List<ExprStats> conjuncts) {
// Collect all estimated selectivities.
List<Double> selectivities = new ArrayList<>();
for (Expr e : conjuncts) {
for (ExprStats e : conjuncts) {
if (e.hasSelectivity()) {
selectivities.add(e.getSelectivity());
}
@ -155,16 +152,16 @@ public class BaseStatsDerive {
}
protected HashMap<SlotId, Float> deriveColumnToDataSize() {
HashMap<SlotId, Float> columnToDataSize = new HashMap<>();
protected HashMap<Id, Float> deriveColumnToDataSize() {
HashMap<Id, Float> columnToDataSize = new HashMap<>();
for (StatsDeriveResult child : childrenStatsResult) {
columnToDataSize.putAll(child.getColumnToDataSize());
}
return columnToDataSize;
}
protected HashMap<SlotId, Long> deriveColumnToNdv() {
HashMap<SlotId, Long> columnToNdv = new HashMap<>();
protected HashMap<Id, Long> deriveColumnToNdv() {
HashMap<Id, Long> columnToNdv = new HashMap<>();
for (StatsDeriveResult child : childrenStatsResult) {
columnToNdv.putAll(child.getColumnToNdv());
}

6
fe/fe-core/src/main/java/org/apache/doris/statistics/DeriveFactory.java
View File

@ -17,12 +17,10 @@
package org.apache.doris.statistics;
import org.apache.doris.planner.PlanNode;
public class DeriveFactory {
public BaseStatsDerive getStatsDerive(PlanNode.NodeType nodeType) {
switch (nodeType) {
public BaseStatsDerive getStatsDerive(StatisticalType statisticalType) {
switch (statisticalType) {
case AGG_NODE:
return new AggStatsDerive();
case ANALYTIC_EVAL_NODE:

33
fe/fe-core/src/main/java/org/apache/doris/statistics/ExprStats.java Normal file
View File

@ -0,0 +1,33 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.statistics;
/**
* Used to abstract a common expression interface for statistics deduction to fit both optimizers.
*/
public interface ExprStats {
boolean hasSelectivity();
double getSelectivity();
void setSelectivity();
long getNumDistinctValues();
}

3
fe/fe-core/src/main/java/org/apache/doris/statistics/HashJoinStatsDerive.java
View File

@ -26,7 +26,6 @@ import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.common.CheckedMath;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.HashJoinNode;
import org.apache.doris.planner.PlanNode;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@ -42,7 +41,7 @@ public class HashJoinStatsDerive extends BaseStatsDerive {
private List<BinaryPredicate> eqJoinConjuncts = Lists.newArrayList();
@Override
public void init(PlanNode node) throws UserException {
public void init(PlanStats node) throws UserException {
Preconditions.checkState(node instanceof HashJoinNode);
super.init(node);
joinOp = ((HashJoinNode) node).getJoinOp();

13
fe/fe-core/src/main/java/org/apache/doris/statistics/OlapScanStatsDerive.java
View File

@ -18,12 +18,11 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.Id;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanNode;
import com.google.common.base.Preconditions;
@ -40,12 +39,12 @@ public class OlapScanStatsDerive extends BaseStatsDerive {
// The rowCount here is the number of rows.
private long inputRowCount = -1;
private Map<SlotId, Float> slotIdToDataSize;
private Map<SlotId, Long> slotIdToNdv;
private Map<SlotId, Pair<Long, String>> slotIdToTableIdAndColumnName;
private Map<Id, Float> slotIdToDataSize;
private Map<Id, Long> slotIdToNdv;
private Map<Id, Pair<Long, String>> slotIdToTableIdAndColumnName;
@Override
public void init(PlanNode node) throws UserException {
public void init(PlanStats node) throws UserException {
Preconditions.checkState(node instanceof OlapScanNode);
super.init(node);
buildStructure((OlapScanNode) node);
@ -61,7 +60,7 @@ public class OlapScanStatsDerive extends BaseStatsDerive {
* - So only an inaccurate cardinality can be calculated here.
*/
rowCount = inputRowCount;
for (Map.Entry<SlotId, Pair<Long, String>> pairEntry : slotIdToTableIdAndColumnName.entrySet()) {
for (Map.Entry<Id, Pair<Long, String>> pairEntry : slotIdToTableIdAndColumnName.entrySet()) {
Pair<Long, Float> ndvAndDataSize = getNdvAndDataSizeFromStatistics(pairEntry.getValue());
long ndv = ndvAndDataSize.first;
float dataSize = ndvAndDataSize.second;

39
fe/fe-core/src/main/java/org/apache/doris/statistics/PlanStats.java Normal file
View File

@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.statistics;
import java.util.List;
/**
* Used to abstract a common operator interface for statistics deduction to fit both optimizers.
*/
public interface PlanStats {
List<StatsDeriveResult> getChildrenStats();
StatsDeriveResult getStatsDeriveResult();
StatisticalType getStatisticalType();
void setStatsDeriveResult(StatsDeriveResult result);
long getLimit();
List<? extends ExprStats> getConjuncts();
}

52
fe/fe-core/src/main/java/org/apache/doris/statistics/SlotStatsDeriveResult.java Normal file
View File

@ -0,0 +1,52 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.statistics;
import org.apache.doris.nereids.trees.expressions.Literal;
public class SlotStatsDeriveResult {
// number of distinct value
private long ndv;
private Literal max;
private Literal min;
public long getNdv() {
return ndv;
}
public void setNdv(long ndv) {
this.ndv = ndv;
}
public Literal getMax() {
return max;
}
public void setMax(Literal max) {
this.max = max;
}
public Literal getMin() {
return min;
}
public void setMin(Literal min) {
this.min = min;
}
}

47
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java Normal file
View File

@ -0,0 +1,47 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.statistics;
public enum StatisticalType {
DEFAULT,
AGG_NODE,
ANALYTIC_EVAL_NODE,
ASSERT_NUM_ROWS_NODE,
BROKER_SCAN_NODE,
CROSS_JOIN_NODE,
EMPTY_SET_NODE,
ES_SCAN_NODE,
EXCEPT_NODE,
EXCHANGE_NODE,
HASH_JOIN_NODE,
HIVE_SCAN_NODE,
ICEBERG_SCAN_NODE,
INTERSECT_NODE,
LOAD_SCAN_NODE,
MYSQL_SCAN_NODE,
ODBC_SCAN_NODE,
OLAP_SCAN_NODE,
REPEAT_NODE,
SELECT_NODE,
SET_OPERATION_NODE,
SCHEMA_SCAN_NODE,
SORT_NODE,
STREAM_LOAD_SCAN_NODE,
TABLE_FUNCTION_NODE,
UNION_NODE,
}

12
fe/fe-core/src/main/java/org/apache/doris/statistics/StatsDeriveResult.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.common.Id;
import com.google.common.collect.Maps;
@ -28,12 +28,12 @@ public class StatsDeriveResult {
private long rowCount = -1;
// The data size of the corresponding column in the operator
// The actual key is slotId
private final Map<SlotId, Float> columnToDataSize = Maps.newHashMap();
private final Map<Id, Float> columnToDataSize = Maps.newHashMap();
// The ndv of the corresponding column in the operator
// The actual key is slotId
private final Map<SlotId, Long> columnToNdv = Maps.newHashMap();
private final Map<Id, Long> columnToNdv = Maps.newHashMap();
public StatsDeriveResult(long rowCount, Map<SlotId, Float> columnToDataSize, Map<SlotId, Long> columnToNdv) {
public StatsDeriveResult(long rowCount, Map<Id, Float> columnToDataSize, Map<Id, Long> columnToNdv) {
this.rowCount = rowCount;
this.columnToDataSize.putAll(columnToDataSize);
this.columnToNdv.putAll(columnToNdv);
@ -47,11 +47,11 @@ public class StatsDeriveResult {
return rowCount;
}
public Map<SlotId, Long> getColumnToNdv() {
public Map<Id, Long> getColumnToNdv() {
return columnToNdv;
}
public Map<SlotId, Float> getColumnToDataSize() {
public Map<Id, Float> getColumnToDataSize() {
return columnToDataSize;
}
}

3
fe/fe-core/src/main/java/org/apache/doris/statistics/StatsRecursiveDerive.java
View File

@ -20,7 +20,6 @@ package org.apache.doris.statistics;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.PlanNode;
public class StatsRecursiveDerive {
private StatsRecursiveDerive() {}
@ -48,7 +47,7 @@ public class StatsRecursiveDerive {
}
}
DeriveFactory deriveFactory = new DeriveFactory();
BaseStatsDerive deriveStats = deriveFactory.getStatsDerive(node.getNodeType());
BaseStatsDerive deriveStats = deriveFactory.getStatsDerive(node.getStatisticalType());
deriveStats.init(node);
StatsDeriveResult result = deriveStats.deriveStats();
node.setStatsDeriveResult(result);