Skip to content

avoiding a corrupt image file when there is image.ckpt with non-zero … #9180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lide-reed merged 3 commits into from
Apr 25, 2022
Merged

avoiding a corrupt image file when there is image.ckpt with non-zero … #9180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3a70112
avoiding a corrupt image file when there is image.ckpt with non-zero …
dataroaring Apr 23, 2022
36b6c62
append other data after MetaHeader
dataroaring Apr 24, 2022
e86aefb
use channel.force instead of sync
dataroaring Apr 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1966,8 +1966,13 @@ public void saveImage() throws IOException {
}

public void saveImage(File curFile, long replayedJournalId) throws IOException {
if (!curFile.exists()) {
curFile.createNewFile();
if (curFile.exists()) {
if (!curFile.delete()) {
throw new IOException(curFile.getName() + " can not be deleted.");
}
}
if (!curFile.createNewFile()) {
throw new IOException(curFile.getName() + " can not be created.");
}
MetaWriter.write(curFile, this);
}
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/common/MetaFooter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ public static void write(File imageFile, List<MetaIndex> metaIndices, long check
long endIndex = raf.length();
raf.writeLong(endIndex - startIndex);
MetaMagicNumber.write(raf);
raf.getChannel.force(true);
}
}

Expand Down
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/common/MetaHeader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,15 @@ public static MetaHeader read(File imageFile) throws IOException {
}

public static long write(File imageFile) throws IOException {
if (imageFile.length() != 0) {
throw new IOException("Meta header has to be written to an empty file.");
}

try (RandomAccessFile raf = new RandomAccessFile(imageFile, "rw")) {
raf.seek(0);
MetaMagicNumber.write(raf);
MetaJsonHeader.write(raf);
raf.getChannel.force(true);
return raf.getFilePointer();
}
}
Expand Down
8 changes: 5 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/common/MetaWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,15 @@ public long doWork(String name, WriteMethod method) throws IOException {

public static void write(File imageFile, Catalog catalog) throws IOException {
// save image does not need any lock. because only checkpoint thread will call this method.
LOG.info("start save image to {}. is ckpt: {}", imageFile.getAbsolutePath(), Catalog.isCheckpointThread());

LOG.info("start to save image to {}. is ckpt: {}", imageFile.getAbsolutePath(), Catalog.isCheckpointThread());
final Reference<Long> checksum = new Reference<>(0L);
long saveImageStartTime = System.currentTimeMillis();
// MetaHeader should use output stream in the future.
long startPosition = MetaHeader.write(imageFile);
List<MetaIndex> metaIndices = Lists.newArrayList();
FileOutputStream imageFileOut = new FileOutputStream(imageFile, true);
try (CountingDataOutputStream dos = new CountingDataOutputStream(new BufferedOutputStream(
new FileOutputStream(imageFile, true)), startPosition)) {
imageFileOut), startPosition)) {
writer.setDelegate(dos, metaIndices);
long replayedJournalId = catalog.getReplayedJournalId();
checksum.setRef(writer.doWork("header", () -> catalog.saveHeader(dos, replayedJournalId, checksum.getRef())));
Expand All @@ -128,6 +129,7 @@ public static void write(File imageFile, Catalog catalog) throws IOException {
checksum.setRef(writer.doWork("plugins", () -> catalog.savePlugins(dos, checksum.getRef())));
checksum.setRef(writer.doWork("deleteHandler", () -> catalog.saveDeleteHandler(dos, checksum.getRef())));
checksum.setRef(writer.doWork("sqlBlockRule", () -> catalog.saveSqlBlockRule(dos, checksum.getRef())));
imageFileOut.getChannel().force(true);
}
MetaFooter.write(imageFile, metaIndices, checksum.getRef());

Expand Down