database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](export) filter empty partition before export table to remote storage (#35389) (#35542)

Browse Source 
## Proposed changes

Linked PR : #35389 

<!--Describe your changes.-->

## Further comments

If this is a relatively large or complex change, kick off the discussion
at [dev@doris.apache.org](mailto:dev@doris.apache.org) by explaining why
you chose the solution you did and what alternatives you considered,
etc...
This commit is contained in:
caiconghui
2024-05-28 18:11:12 +08:00
committed by GitHub
parent b78dae040a
commit 27cf5a667f
2 changed files with 28 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
View File

@ -407,6 +407,13 @@ public class ExportJob implements Writable {
ExportTaskExecutor executor = new ExportTaskExecutor(selectStmts, this);
jobExecutorList.add(executor);
}
// add empty task to make export job could be finished finally if jobExecutorList is empty
// which means that export table without data
if (jobExecutorList.isEmpty()) {
ExportTaskExecutor executor = new ExportTaskExecutor(Lists.newArrayList(), this);
jobExecutorList.add(executor);
}
}
/**
@ -511,15 +518,23 @@ public class ExportJob implements Writable {
// get partitions
// user specifies partitions, already checked in ExportCommand
if (!this.partitionNames.isEmpty()) {
this.partitionNames.forEach(partitionName -> partitions.add(table.getPartition(partitionName)));
this.partitionNames.forEach(partitionName -> {
Partition partition = table.getPartition(partitionName);
if (partition.hasData()) {
partitions.add(partition);
}
});
} else {
if (table.getPartitions().size() > Config.maximum_number_of_export_partitions) {
throw new UserException("The partitions number of this export job is larger than the maximum number"
+ " of partitions allowed by a export job");
}
partitions.addAll(table.getPartitions());
table.getPartitions().forEach(partition -> {
if (partition.hasData()) {
partitions.add(partition);
}
});
}
if (partitions.size() > Config.maximum_number_of_export_partitions) {
throw new UserException("The partitions number of this export job is larger than the maximum number"
+ " of partitions allowed by a export job");
}
// get tablets
for (Partition partition : partitions) {
// Partition data consistency is not need to verify partition version.
@ -589,8 +604,7 @@ public class ExportJob implements Writable {
List<Long> tabletsList = new ArrayList<>(flatTabletIdList.subList(start, start + tabletsNum));
List<List<Long>> tablets = new ArrayList<>();
for (int i = 0; i < tabletsList.size(); i += MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT) {
int end = i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT < tabletsList.size()
? i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT : tabletsList.size();
int end = Math.min(i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT, tabletsList.size());
tablets.add(new ArrayList<>(tabletsList.subList(i, end)));
}