diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index b6f031ab7d42eb..ec22d6eafc708c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -568,6 +568,13 @@ private void removeUnsupportProperties(OlapTable tbl) { private void waitingAllSnapshotsFinished() { if (unfinishedTaskIds.isEmpty()) { + + if (env.getEditLog().exceedMaxJournalSize(this)) { + status = new Status(ErrCode.COMMON_ERROR, "backupJob is too large "); + return; + } + + snapshotFinishedTime = System.currentTimeMillis(); state = BackupJobState.UPLOAD_SNAPSHOT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java index 8fca299df19489..b0929818eede7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/Journal.java @@ -61,4 +61,6 @@ public interface Journal { // Get all the dbs' name public List getDatabaseNames(); + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException; + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java index 3746e769f341b4..4105da113b3521 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java @@ -586,4 +586,27 @@ public List getDatabaseNames() { public BDBEnvironment getBDBEnvironment() { return this.bdbEnvironment; } + + @Override + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + JournalEntity entity = new JournalEntity(); + entity.setOpCode(op); + entity.setData(writable); + + DataOutputBuffer buffer = new DataOutputBuffer(OUTPUT_BUFFER_INIT_SIZE); + entity.write(buffer); + + DatabaseEntry theData = new DatabaseEntry(buffer.getData()); + + if (LOG.isDebugEnabled()) { + LOG.debug("opCode = {}, journal size = {}", op, theData.getSize()); + } + + // 1GB + if (theData.getSize() > (1 << 30)) { + return true; + } + + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java index 9ba3274e0f0282..4467c53a368d4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java @@ -198,4 +198,9 @@ private long getCurrentJournalId(List editFileNames) { public List getDatabaseNames() { throw new RuntimeException("Not Support"); } + + @Override + public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 79dde3e58743f4..a1e033d6d1f9c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -1905,4 +1905,17 @@ public void logAutoJob(AnalysisInfo analysisInfo) { public void logDeleteTableStats(TableStatsDeletionLog log) { logEdit(OperationType.OP_DELETE_TABLE_STATS, log); } + + public boolean exceedMaxJournalSize(BackupJob job) { + try { + return exceedMaxJournalSize(OperationType.OP_BACKUP_JOB, job); + } catch (Exception e) { + LOG.warn("exceedMaxJournalSize exception:", e); + } + return true; + } + + private boolean exceedMaxJournalSize(short op, Writable writable) throws IOException { + return journal.exceedMaxJournalSize(op, writable); + } }