diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 0342ba70b80fec..fc846bf1820ffe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -615,6 +615,13 @@ private void removeUnsupportProperties(OlapTable tbl) { private void waitingAllSnapshotsFinished() { if (unfinishedTaskIds.isEmpty()) { + + if (env.getEditLog().exceedMaxJournalSize(this)) { + status = new Status(ErrCode.COMMON_ERROR, "backupJob is too large "); + return; + } + + snapshotFinishedTime = System.currentTimeMillis(); state = BackupJobState.UPLOAD_SNAPSHOT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java index b5b37a80ef0ff2..1efba2ff92336e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java @@ -66,4 +66,6 @@ public interface Journal { // Get all the dbs' name public List getDatabaseNames(); + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException; + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java index 0d342a986cd40f..92e900170b8817 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java @@ -707,4 +707,27 @@ public String getNotReadyReason() { } return bdbEnvironment.getNotReadyReason(); } + + @Override + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + JournalEntity entity = new JournalEntity(); + entity.setOpCode(op); + entity.setData(writable); + + DataOutputBuffer buffer = new DataOutputBuffer(OUTPUT_BUFFER_INIT_SIZE); + entity.write(buffer); + + DatabaseEntry theData = new DatabaseEntry(buffer.getData()); + + if (LOG.isDebugEnabled()) { + LOG.debug("opCode = {}, journal size = {}", op, theData.getSize()); + } + + // 1GB + if (theData.getSize() > (1 << 30)) { + return true; + } + + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java index 45ae377a40171c..77e15893845740 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java @@ -210,4 +210,9 @@ private long getCurrentJournalId(List editFileNames) { public List getDatabaseNames() { throw new RuntimeException("Not Support"); } + + @Override + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index d440cf77cac5e7..597421853b198c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -2102,4 +2102,17 @@ public String getNotReadyReason() { } return ""; } + + public boolean exceedMaxJournalSize(BackupJob job) { + try { + return exceedMaxJournalSize(OperationType.OP_BACKUP_JOB, job); + } catch (Exception e) { + LOG.warn("exceedMaxJournalSize exception:", e); + } + return true; + } + + private boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + return journal.exceedMaxJournalSize(op, writable); + } }