* When creating a backupJob with huge of tables in a database, it can cause backupJob editlog size over 2GB and bdbje will throw exception because of ByteBuffer overflow ## Proposed changes Issue Number: close #xxx <!--Describe your changes.-->
This commit is contained in:
@ -615,6 +615,13 @@ public class BackupJob extends AbstractJob {
|
||||
|
||||
private void waitingAllSnapshotsFinished() {
|
||||
if (unfinishedTaskIds.isEmpty()) {
|
||||
|
||||
if (env.getEditLog().exceedMaxJournalSize(this)) {
|
||||
status = new Status(ErrCode.COMMON_ERROR, "backupJob is too large ");
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
snapshotFinishedTime = System.currentTimeMillis();
|
||||
state = BackupJobState.UPLOAD_SNAPSHOT;
|
||||
|
||||
|
||||
@ -66,4 +66,6 @@ public interface Journal {
|
||||
// Get all the dbs' name
|
||||
public List<Long> getDatabaseNames();
|
||||
|
||||
public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException;
|
||||
|
||||
}
|
||||
|
||||
@ -703,4 +703,27 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B
|
||||
}
|
||||
return bdbEnvironment.getNotReadyReason();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
|
||||
JournalEntity entity = new JournalEntity();
|
||||
entity.setOpCode(op);
|
||||
entity.setData(writable);
|
||||
|
||||
DataOutputBuffer buffer = new DataOutputBuffer(OUTPUT_BUFFER_INIT_SIZE);
|
||||
entity.write(buffer);
|
||||
|
||||
DatabaseEntry theData = new DatabaseEntry(buffer.getData());
|
||||
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("opCode = {}, journal size = {}", op, theData.getSize());
|
||||
}
|
||||
|
||||
// 1GB
|
||||
if (theData.getSize() > (1 << 30)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@ -210,4 +210,9 @@ public class LocalJournal implements Journal {
|
||||
public List<Long> getDatabaseNames() {
|
||||
throw new RuntimeException("Not Support");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@ -2080,4 +2080,17 @@ public class EditLog {
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public boolean exceedMaxJournalSize(BackupJob job) {
|
||||
try {
|
||||
return exceedMaxJournalSize(OperationType.OP_BACKUP_JOB, job);
|
||||
} catch (Exception e) {
|
||||
LOG.warn("exceedMaxJournalSize exception:", e);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
|
||||
return journal.exceedMaxJournalSize(op, writable);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user