database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[branch-2.1] Pick "[Fix](autoinc) try fix concurrent load problem with auto inc column #36421" (#37027)

Browse Source 
## Proposed changes

pick https://github.com/apache/doris/pull/36421
This commit is contained in:
bobhan1
2024-06-30 13:10:03 +08:00
committed by GitHub
parent d237a4d303
commit 12dddfc26c
7 changed files with 203 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/catalog/AutoIncrementGenerator.java
View File

@ -40,7 +40,7 @@ public class AutoIncrementGenerator implements Writable, GsonPostProcessable {
public static final long NEXT_ID_INIT_VALUE = 1;
// _MIN_BATCH_SIZE = 4064 in load task
private static final long BATCH_ID_INTERVAL = 50000;
private static final long BATCH_ID_INTERVAL = 500000;
@SerializedName(value = "dbId")
private Long dbId;
@ -48,7 +48,6 @@ public class AutoIncrementGenerator implements Writable, GsonPostProcessable {
private Long tableId;
@SerializedName(value = "columnId")
private Long columnId;
@SerializedName(value = "nextId")
private long nextId;
@SerializedName(value = "batchEndId")
private long batchEndId;
@ -86,10 +85,10 @@ public class AutoIncrementGenerator implements Writable, GsonPostProcessable {
long endId = startId + length;
nextId = startId + length;
if (endId > batchEndId) {
batchEndId = (endId / BATCH_ID_INTERVAL + 1) * BATCH_ID_INTERVAL;
Preconditions.checkState(editLog != null);
AutoIncrementIdUpdateLog info = new AutoIncrementIdUpdateLog(dbId, tableId, columnId, batchEndId);
editLog.logUpdateAutoIncrementId(info);
batchEndId = (endId / BATCH_ID_INTERVAL + 1) * BATCH_ID_INTERVAL;
}
LOG.info("[getAutoIncrementRange result][{}, {}]", startId, length);
return Pair.of(startId, length);

10
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -2769,6 +2769,16 @@ public class FrontendServiceImpl implements FrontendService.Iface {
TAutoIncrementRangeResult result = new TAutoIncrementRangeResult();
TStatus status = new TStatus(TStatusCode.OK);
result.setStatus(status);
if (!Env.getCurrentEnv().isMaster()) {
status.setStatusCode(TStatusCode.NOT_MASTER);
status.addToErrorMsgs(NOT_MASTER_ERR_MSG);
result.setMasterAddress(getMasterAddress());
LOG.error("failed to getAutoIncrementRange:{}, request:{}, backend:{}",
NOT_MASTER_ERR_MSG, request, getClientAddrAsString());
return result;
}
try {
Env env = Env.getCurrentEnv();
Database db = env.getInternalCatalog().getDbOrMetaException(request.getDbId());