database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Opt](load) use batching to optimize auto partition (#26915)

Browse Source 
use batching to optimize auto partition
This commit is contained in:
zclllyybb
2023-11-23 19:12:28 +08:00
committed by GitHub
parent 511eedb4ff
commit 2ea33518b0
27 changed files with 398 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
View File

@ -368,23 +368,30 @@ public class OlapTableSink extends DataSink {
}
}
}
boolean enableAutomaticPartition = partitionInfo.enableAutomaticPartition();
boolean enableAutomaticPartition = partitionInfo.enableAutomaticPartition();
// for auto create partition by function expr, there is no any partition firstly,
// But this is required in thrift struct.
if (enableAutomaticPartition && partitionIds.isEmpty()) {
partitionParam.setDistributedColumns(getDistColumns(table.getDefaultDistributionInfo()));
partitionParam.setPartitions(new ArrayList<TOlapTablePartition>());
}
ArrayList<Expr> exprs = partitionInfo.getPartitionExprs();
if (enableAutomaticPartition && exprs != null && analyzer != null) {
ArrayList<Expr> exprSource = partitionInfo.getPartitionExprs();
if (enableAutomaticPartition && exprSource != null && analyzer != null) {
Analyzer funcAnalyzer = new Analyzer(analyzer.getEnv(), analyzer.getContext());
tupleDescriptor.setTable(table);
funcAnalyzer.registerTupleDescriptor(tupleDescriptor);
// we must clone the exprs. otherwise analyze will influence the origin exprs.
ArrayList<Expr> exprs = new ArrayList<Expr>();
for (Expr e : exprSource) {
exprs.add(e.clone());
}
for (Expr e : exprs) {
e.analyze(funcAnalyzer);
}
partitionParam.setPartitionFunctionExprs(Expr.treesToThrift(exprs));
}
partitionParam.setEnableAutomaticPartition(enableAutomaticPartition);
break;
}