[fix](fe) Add check editlog size mechanism for backupJob (#35653) (#37466)

* When creating a backupJob with huge of tables in a database, it can
cause backupJob editlog size over 2GB and bdbje will throw exception
because of ByteBuffer overflow

## Proposed changes

Issue Number: close #xxx

<!--Describe your changes.-->
This commit is contained in:
Lei Zhang
2024-07-09 10:33:28 +08:00
committed by GitHub
parent ca0e44f83f
commit 4426d6d80f
5 changed files with 50 additions and 0 deletions

View File

@ -615,6 +615,13 @@ public class BackupJob extends AbstractJob {
private void waitingAllSnapshotsFinished() {
if (unfinishedTaskIds.isEmpty()) {
if (env.getEditLog().exceedMaxJournalSize(this)) {
status = new Status(ErrCode.COMMON_ERROR, "backupJob is too large ");
return;
}
snapshotFinishedTime = System.currentTimeMillis();
state = BackupJobState.UPLOAD_SNAPSHOT;

View File

@ -66,4 +66,6 @@ public interface Journal {
// Get all the dbs' name
public List<Long> getDatabaseNames();
public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException;
}

View File

@ -703,4 +703,27 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B
}
return bdbEnvironment.getNotReadyReason();
}
@Override
public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
JournalEntity entity = new JournalEntity();
entity.setOpCode(op);
entity.setData(writable);
DataOutputBuffer buffer = new DataOutputBuffer(OUTPUT_BUFFER_INIT_SIZE);
entity.write(buffer);
DatabaseEntry theData = new DatabaseEntry(buffer.getData());
if (LOG.isDebugEnabled()) {
LOG.debug("opCode = {}, journal size = {}", op, theData.getSize());
}
// 1GB
if (theData.getSize() > (1 << 30)) {
return true;
}
return false;
}
}

View File

@ -210,4 +210,9 @@ public class LocalJournal implements Journal {
public List<Long> getDatabaseNames() {
throw new RuntimeException("Not Support");
}
@Override
public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
return false;
}
}

View File

@ -2080,4 +2080,17 @@ public class EditLog {
}
return "";
}
public boolean exceedMaxJournalSize(BackupJob job) {
try {
return exceedMaxJournalSize(OperationType.OP_BACKUP_JOB, job);
} catch (Exception e) {
LOG.warn("exceedMaxJournalSize exception:", e);
}
return true;
}
private boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
return journal.exceedMaxJournalSize(op, writable);
}
}