database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](spark load)Temp partition with spark load (#14648)

Browse Source 
* [fix](spark load)losing temporary partition item entry

* [fix](spark load)Temp partition with spark load
This commit is contained in:
xiaoDjun
2022-11-29 15:21:44 +08:00
committed by GitHub
parent c5f9fd5619
commit facb7cf4e2
2 changed files with 15 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.common.util.RangeUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import java.io.DataInput;
@ -77,6 +78,19 @@ public class RangePartitionInfo extends PartitionInfo {
return itemEntryList;
}
public List<Map.Entry<Long, PartitionItem>> getAllPartitionItemEntryList(boolean isSorted) {
Map<Long, PartitionItem> tmpMap = Maps.newHashMap();
tmpMap.putAll(idToItem);
tmpMap.putAll(idToTempItem);
List<Map.Entry<Long, PartitionItem>> itemEntryList = Lists.newArrayList(tmpMap.entrySet());
if (isSorted) {
Collections.sort(itemEntryList, RangeUtils.RANGE_MAP_ENTRY_COMPARATOR);
}
return itemEntryList;
}
// create a new range and check it.
private Range<PartitionKey> createAndCheckNewRange(PartitionKeyDesc partKeyDesc, boolean isTemp)
throws AnalysisException, DdlException {

2
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
View File

@ -334,7 +334,7 @@ public class SparkLoadPendingTask extends LoadTask {
partitionColumnRefs.add(column.getName());
}
for (Map.Entry<Long, PartitionItem> entry : rangePartitionInfo.getPartitionItemEntryList(false, true)) {
for (Map.Entry<Long, PartitionItem> entry : rangePartitionInfo.getAllPartitionItemEntryList(true)) {
long partitionId = entry.getKey();
if (!partitionIds.contains(partitionId)) {
continue;