database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[branch-2.1](load) fix commit txn timeout when loading to table with many tablet (#40031) (#40139)

Browse Source 
pick https://github.com/apache/doris/pull/40031
This commit is contained in:
zclllhhjj
2024-08-30 10:34:43 +08:00
committed by GitHub
parent 70daa1f85d
commit e4d543f060
2 changed files with 38 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
View File

@ -149,12 +149,16 @@ public class BrokerFileGroup implements Writable {
}
}
boolean isPartitionRestoring = olapTable.getPartitions().stream().anyMatch(
partition -> partition.getState() == PartitionState.RESTORE
);
// restore table
if (!isPartitionRestoring && olapTable.getState() == OlapTableState.RESTORE) {
throw new DdlException("Table [" + olapTable.getName() + "] is under restore");
// only do check when here's restore on this table now
if (olapTable.getState() == OlapTableState.RESTORE) {
boolean hasPartitionRestoring = olapTable.getPartitions().stream()
.anyMatch(partition -> partition.getState() == PartitionState.RESTORE);
// tbl RESTORE && all partition NOT RESTORE -> whole table restore
// tbl RESTORE && some partition RESTORE -> just partitions restore, NOT WHOLE TABLE
// so check wether the whole table restore here
if (!hasPartitionRestoring) {
throw new DdlException("Table [" + olapTable.getName() + "] is under restore");
}
}
if (olapTable.getKeysType() != KeysType.AGG_KEYS && dataDescription.isNegative()) {

40
fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
View File

@ -504,7 +504,9 @@ public class DatabaseTransactionMgr {
List<Long> tabletIds = tabletCommitInfos.stream()
.map(TabletCommitInfo::getTabletId).collect(Collectors.toList());
List<TabletMeta> tabletMetaList = tabletInvertedIndex.getTabletMetaList(tabletIds);
HashMap<Long, Boolean> tableIdtoRestoring = new HashMap<>();
for (int i = 0; i < tabletMetaList.size(); i++) {
// get partition and table of this tablet
TabletMeta tabletMeta = tabletMetaList.get(i);
if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
continue;
@ -513,29 +515,43 @@ public class DatabaseTransactionMgr {
long tableId = tabletMeta.getTableId();
OlapTable tbl = (OlapTable) idToTable.get(tableId);
if (tbl == null) {
// this can happen when tableId == -1 (tablet being dropping)
// or table really not exist.
// this can happen when tableId == -1 (tablet being dropping) or table really not exist.
continue;
}
// check relative partition restore here
long partitionId = tabletMeta.getPartitionId();
if (tbl.getPartition(partitionId) == null) {
// this can happen when partitionId == -1 (tablet being dropping)
// or partition really not exist.
// this can happen when partitionId == -1 (tablet being dropping) or partition really not exist.
continue;
} else if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) {
}
if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) {
// partition in restore process which can not load data
throw new LoadException("Table [" + tbl.getName() + "], Partition ["
+ tbl.getPartition(partitionId).getName() + "] is in restore process. Can not load into it");
}
boolean isPartitionRestoring = tbl.getPartitions().stream().anyMatch(
partition -> partition.getState() == PartitionState.RESTORE
);
// restore table
if (!isPartitionRestoring && tbl.getState() == OlapTableState.RESTORE) {
throw new LoadException("Table " + tbl.getName() + " is in restore process. "
+ "Can not load into it");
// only do check when here's restore on this table now
if (tbl.getState() == OlapTableState.RESTORE) {
boolean hasPartitionRestoring = false;
if (tableIdtoRestoring.containsKey(tableId)) {
hasPartitionRestoring = tableIdtoRestoring.get(tableId);
} else {
for (Partition partition : tbl.getPartitions()) {
if (partition.getState() == PartitionState.RESTORE) {
hasPartitionRestoring = true;
break;
}
}
tableIdtoRestoring.put(tableId, hasPartitionRestoring);
}
// tbl RESTORE && all partition NOT RESTORE -> whole table restore
// tbl RESTORE && some partition RESTORE -> just partitions restore, NOT WHOLE TABLE
// so check wether the whole table restore here
if (!hasPartitionRestoring) {
throw new LoadException(
"Table " + tbl.getName() + " is in restore process. " + "Can not load into it");
}
}
if (!tableToPartition.containsKey(tableId)) {