database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](Load)allow load data to the other partitions when some partitions are restoring (#39915)

Browse Source 
If broker load or stream load task execute in one table that is
restoring data, load task will failed with Exception.
Exception info :"Table [xxx] is under restore" or "Table [xxx] is in
restore process, can't load into it".

But mostly restoreJob only effects some partitions in this table, not
all of them, so that the other partitions still need to load data
successfully.
To achieve this goal, before checking olap table state, check partition
state first.

cherry pick from master branch, pr has been merged:
https://github.com/apache/doris/pull/39595

Co-authored-by: shenshoucheng <shenshoucheng@jd.com>
This commit is contained in:
shouchengShen
2024-08-26 21:04:34 +08:00
committed by GitHub
parent fb5002d723
commit e4bf10ee96
2 changed files with 26 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
@ -139,11 +140,20 @@ public class BrokerFileGroup implements Writable {
throw new DdlException("Unknown partition '" + pName
+ "' in table '" + olapTable.getName() + "'");
}
// partition which need load data
if (partition.getState() == PartitionState.RESTORE) {
throw new DdlException("Table [" + olapTable.getName()
+ "], Partition[" + partition.getName() + "] is under restore");
}
partitionIds.add(partition.getId());
}
}
if (olapTable.getState() == OlapTableState.RESTORE) {
boolean isPartitionRestoring = olapTable.getPartitions().stream().anyMatch(
partition -> partition.getState() == PartitionState.RESTORE
);
// restore table
if (!isPartitionRestoring && olapTable.getState() == OlapTableState.RESTORE) {
throw new DdlException("Table [" + olapTable.getName() + "] is under restore");
}

20
fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
View File

@ -23,7 +23,9 @@ import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.Replica;
@ -516,16 +518,24 @@ public class DatabaseTransactionMgr {
continue;
}
if (tbl.getState() == OlapTable.OlapTableState.RESTORE) {
throw new LoadException("Table " + tbl.getName() + " is in restore process. "
+ "Can not load into it");
}
long partitionId = tabletMeta.getPartitionId();
if (tbl.getPartition(partitionId) == null) {
// this can happen when partitionId == -1 (tablet being dropping)
// or partition really not exist.
continue;
} else if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) {
// partition in restore process which can not load data
throw new LoadException("Table [" + tbl.getName() + "], Partition ["
+ tbl.getPartition(partitionId).getName() + "] is in restore process. Can not load into it");
}
boolean isPartitionRestoring = tbl.getPartitions().stream().anyMatch(
partition -> partition.getState() == PartitionState.RESTORE
);
// restore table
if (!isPartitionRestoring && tbl.getState() == OlapTableState.RESTORE) {
throw new LoadException("Table " + tbl.getName() + " is in restore process. "
+ "Can not load into it");
}
if (!tableToPartition.containsKey(tableId)) {