diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 0342ba70b8..fc846bf182 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -615,6 +615,13 @@ public class BackupJob extends AbstractJob { private void waitingAllSnapshotsFinished() { if (unfinishedTaskIds.isEmpty()) { + + if (env.getEditLog().exceedMaxJournalSize(this)) { + status = new Status(ErrCode.COMMON_ERROR, "backupJob is too large "); + return; + } + + snapshotFinishedTime = System.currentTimeMillis(); state = BackupJobState.UPLOAD_SNAPSHOT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java index b5b37a80ef..1efba2ff92 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java @@ -66,4 +66,6 @@ public interface Journal { // Get all the dbs' name public List getDatabaseNames(); + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException; + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java index 55b63e18d6..c557db9e02 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java @@ -703,4 +703,27 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B } return bdbEnvironment.getNotReadyReason(); } + + @Override + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + JournalEntity entity = new JournalEntity(); + entity.setOpCode(op); + entity.setData(writable); + + DataOutputBuffer buffer = new DataOutputBuffer(OUTPUT_BUFFER_INIT_SIZE); + entity.write(buffer); + + DatabaseEntry theData = new DatabaseEntry(buffer.getData()); + + if (LOG.isDebugEnabled()) { + LOG.debug("opCode = {}, journal size = {}", op, theData.getSize()); + } + + // 1GB + if (theData.getSize() > (1 << 30)) { + return true; + } + + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java index 8e39f8a6a6..51b6b0f3f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java @@ -210,4 +210,9 @@ public class LocalJournal implements Journal { public List getDatabaseNames() { throw new RuntimeException("Not Support"); } + + @Override + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 5abda5dc45..d95b8a0a8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -2080,4 +2080,17 @@ public class EditLog { } return ""; } + + public boolean exceedMaxJournalSize(BackupJob job) { + try { + return exceedMaxJournalSize(OperationType.OP_BACKUP_JOB, job); + } catch (Exception e) { + LOG.warn("exceedMaxJournalSize exception:", e); + } + return true; + } + + private boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + return journal.exceedMaxJournalSize(op, writable); + } }