Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,13 @@ private void removeUnsupportProperties(OlapTable tbl) {

private void waitingAllSnapshotsFinished() {
if (unfinishedTaskIds.isEmpty()) {

if (env.getEditLog().exceedMaxJournalSize(this)) {
status = new Status(ErrCode.COMMON_ERROR, "backupJob is too large ");
return;
}


snapshotFinishedTime = System.currentTimeMillis();
state = BackupJobState.UPLOAD_SNAPSHOT;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,6 @@ public interface Journal {
// Get all the dbs' name
public List<Long> getDatabaseNames();

public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException;

}
Original file line number Diff line number Diff line change
Expand Up @@ -707,4 +707,27 @@ public String getNotReadyReason() {
}
return bdbEnvironment.getNotReadyReason();
}

@Override
public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is param op mandatory for this check?

JournalEntity entity = new JournalEntity();
entity.setOpCode(op);
entity.setData(writable);

DataOutputBuffer buffer = new DataOutputBuffer(OUTPUT_BUFFER_INIT_SIZE);
entity.write(buffer);

DatabaseEntry theData = new DatabaseEntry(buffer.getData());

if (LOG.isDebugEnabled()) {
LOG.debug("opCode = {}, journal size = {}", op, theData.getSize());
}

// 1GB
if (theData.getSize() > (1 << 30)) {
return true;
}

return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -210,4 +210,9 @@ private long getCurrentJournalId(List<Long> editFileNames) {
public List<Long> getDatabaseNames() {
throw new RuntimeException("Not Support");
}

@Override
public boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
return false;
}
}
13 changes: 13 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
Original file line number Diff line number Diff line change
Expand Up @@ -2102,4 +2102,17 @@ public String getNotReadyReason() {
}
return "";
}

public boolean exceedMaxJournalSize(BackupJob job) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems that we can change it to the following form

Suggested change
public boolean exceedMaxJournalSize(BackupJob job) {
public boolean exceedMaxJournalSize(Writable w) {

try {
return exceedMaxJournalSize(OperationType.OP_BACKUP_JOB, job);
} catch (Exception e) {
LOG.warn("exceedMaxJournalSize exception:", e);
}
return true;
}

private boolean exceedMaxJournalSize(short op, Writable writable) throws IOException {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function is redundant if we do not care about op

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function is redundant if we do not care about op

opCode is important, we will print opCode

return journal.exceedMaxJournalSize(op, writable);
}
}