Skip to content

keep at least one validated image file #9192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lide-reed merged 2 commits into from
Apr 25, 2022
Merged

keep at least one validated image file #9192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6d56866
rename ImageSeq to LatestImageSeq in Storage
dataroaring Apr 24, 2022
b4a6428
keep at least one validated image file
dataroaring Apr 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1531,7 +1531,7 @@ private boolean getVersionFileFromHelper(Pair<String, Integer> helperNode) throw
private void getNewImage(Pair<String, Integer> helperNode) throws IOException {
long localImageVersion = 0;
Storage storage = new Storage(this.imageDir);
localImageVersion = storage.getImageSeq();
localImageVersion = storage.getLatestImageSeq();

try {
URL infoUrl = new URL("http://" + helperNode.first + ":" + Config.http_port + "/info");
Expand Down Expand Up @@ -1617,7 +1617,7 @@ public void loadImage(String imageDir) throws IOException, DdlException {
LOG.info("image does not exist: {}", curFile.getAbsolutePath());
return;
}
replayedJournalId.set(storage.getImageSeq());
replayedJournalId.set(storage.getLatestImageSeq());
MetaReader.read(curFile, this);
}

Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ private void appendImageInfo(Map<String, Object> result) {
Map<String, Object> checkPoint = new HashMap<>();
Storage storage = new Storage(Config.meta_dir + "/image");
checkPoint.put("Name", "Version");
checkPoint.put("Value", storage.getImageSeq());
checkPoint.put("Value", storage.getLatestImageSeq());
list.add(checkPoint);
long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
Date date = new Date(lastCheckpointTime);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public Object info(HttpServletRequest request, HttpServletResponse response) thr
try {
Storage currentStorageInfo = new Storage(imageDir.getAbsolutePath());
StorageInfo storageInfo = new StorageInfo(currentStorageInfo.getClusterID(),
currentStorageInfo.getImageSeq(), currentStorageInfo.getEditsSeq());
currentStorageInfo.getLatestImageSeq(), currentStorageInfo.getEditsSeq());
return ResponseEntityBuilder.ok(storageInfo);
} catch (IOException e) {
return ResponseEntityBuilder.internalError(e.getMessage());
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ private Map<String, String> getHaInfo() throws IOException {
feInfo.put("is_ready", String.valueOf(Catalog.getCurrentCatalog().isReady()));

Storage storage = new Storage(Config.meta_dir + "/image");
feInfo.put("last_checkpoint_version", String.valueOf(storage.getImageSeq()));
feInfo.put("last_checkpoint_version", String.valueOf(storage.getLatestImageSeq()));
long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
feInfo.put("last_checkpoint_time", String.valueOf(lastCheckpointTime));

Expand Down
8 changes: 5 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public synchronized void doCheckpoint() {
try {
storage = new Storage(imageDir);
// get max image version
imageVersion = storage.getImageSeq();
imageVersion = storage.getLatestImageSeq();
// get max finalized journal id
checkPointVersion = editLog.getFinalizedJournalId();
LOG.info("last checkpoint journal id: {}, current finalized journal id: {}", imageVersion, checkPointVersion);
Expand Down Expand Up @@ -184,7 +184,9 @@ public synchronized void doCheckpoint() {
if (successPushed == otherNodesCount) {
try {
long minOtherNodesJournalId = Long.MAX_VALUE;
long deleteVersion = checkPointVersion;
// Actually, storage.getLatestValidatedImageSeq returns number before this
// checkpoint.
long deleteVersion = storage.getLatestValidatedImageSeq();
if (successPushed > 0) {
for (Frontend fe : allFrontends) {
String host = fe.getHost();
Expand Down Expand Up @@ -220,7 +222,7 @@ public synchronized void doCheckpoint() {
}
}
}
deleteVersion = Math.min(minOtherNodesJournalId, checkPointVersion);
deleteVersion = Math.min(minOtherNodesJournalId, deleteVersion);
}

editLog.deleteJournals(deleteVersion + 1);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/persist/MetaCleaner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public MetaCleaner(String imageDir) {

public void clean() throws IOException {
Storage storage = new Storage(imageDir);
long currentVersion = storage.getImageSeq();
long currentVersion = storage.getLatestValidatedImageSeq();
long imageDeleteVersion = currentVersion - 1;

File currentImage = storage.getImageFile(currentVersion);
Expand Down
55 changes: 31 additions & 24 deletions fe/fe-core/src/main/java/org/apache/doris/persist/Storage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ public class Storage {
private FrontendNodeType role = FrontendNodeType.UNKNOWN;
private String nodeName;
private long editsSeq;
private long imageSeq;
private long latestImageSeq;
private long latestValidatedImageSeq;
private String metaDir;
private List<Long> editsFileSequenceNumbers;

Expand All @@ -70,11 +71,11 @@ public Storage(int clusterID, String token, String metaDir) {
this.metaDir = metaDir;
}

public Storage(int clusterID, String token, long imageSeq, long editsSeq, String metaDir) {
public Storage(int clusterID, String token, long latestImageSeq, long editsSeq, String metaDir) {
this.clusterID = clusterID;
this.token = token;
this.editsSeq = editsSeq;
this.imageSeq = imageSeq;
this.latestImageSeq = latestImageSeq;
this.metaDir = metaDir;
}

Expand Down Expand Up @@ -114,31 +115,33 @@ public void reload() throws IOException {
nodeName = prop.getProperty(NODE_NAME, null);
}

// Find the latest image
// Find the latest two images
File dir = new File(metaDir);
File[] children = dir.listFiles();
if (children == null) {
return;
} else {
for (File child : children) {
String name = child.getName();
try {
if (!name.equals(EDITS) && !name.equals(IMAGE_NEW)
&& !name.endsWith(".part") && name.contains(".")) {
if (name.startsWith(IMAGE)) {
imageSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), imageSeq);
} else if (name.startsWith(EDITS)) {
// Just record the sequence part of the file name
editsFileSequenceNumbers.add(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)));
editsSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), editsSeq);
}
for (File child : children) {
String name = child.getName();
try {
if (!name.equals(EDITS) && !name.equals(IMAGE_NEW)
&& !name.endsWith(".part") && name.contains(".")) {
if (name.startsWith(IMAGE)) {
long fileSeq = Long.parseLong(name.substring(name.lastIndexOf('.') + 1));
if (latestImageSeq < fileSeq) {
latestValidatedImageSeq = latestImageSeq;
latestImageSeq = fileSeq;
}
} else if (name.startsWith(EDITS)) {
// Just record the sequence part of the file name
editsFileSequenceNumbers.add(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)));
editsSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), editsSeq);
}
} catch (Exception e) {
LOG.warn(name + " is not a validate meta file, ignore it");
}
} catch (Exception e) {
LOG.warn(name + " is not a validate meta file, ignore it");
}
}

}

public int getClusterID() {
Expand Down Expand Up @@ -173,12 +176,16 @@ public void setMetaDir(String metaDir) {
this.metaDir = metaDir;
}

public long getImageSeq() {
return imageSeq;
public long getLatestImageSeq() {
return latestImageSeq;
}

public long getLatestValidatedImageSeq() {
return latestValidatedImageSeq;
}

public void setImageSeq(long imageSeq) {
this.imageSeq = imageSeq;
public void setLatestImageSeq(long latestImageSeq) {
this.latestImageSeq = latestImageSeq;
}

public void setEditsSeq(long editsSeq) {
Expand Down Expand Up @@ -272,7 +279,7 @@ public static void rename(File from, File to) throws IOException {
}

public File getCurrentImageFile() {
return getImageFile(imageSeq);
return getImageFile(latestImageSeq);
}

public File getImageFile(long version) {
Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/test/java/org/apache/doris/persist/StorageTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public void testConstruct() {

Storage storage2 = new Storage(1, "token", 2, 3, "test");
Assert.assertEquals(1, storage2.getClusterID());
Assert.assertEquals(2, storage2.getImageSeq());
Assert.assertEquals(2, storage2.getLatestImageSeq());
Assert.assertEquals(3, storage2.getEditsSeq());
Assert.assertEquals("test", storage2.getMetaDir());
}
Expand All @@ -116,7 +116,7 @@ public void testStorage() throws Exception {
Assert.assertEquals(966271669, storage.getClusterID());
storage.setClusterID(1234);
Assert.assertEquals(1234, storage.getClusterID());
Assert.assertEquals(0, storage.getImageSeq());
Assert.assertEquals(0, storage.getLatestImageSeq());
Assert.assertEquals(10, Storage.getMetaSeq(new File("storageTestDir/edits.10")));
Assert.assertTrue(Storage.getCurrentEditsFile(new File("storageTestDir"))
.equals(new File("storageTestDir/edits")));
Expand All @@ -133,8 +133,8 @@ public void testStorage() throws Exception {

Assert.assertTrue(storage.getVersionFile().equals(new File("storageTestDir/VERSION")));

storage.setImageSeq(100);
Assert.assertEquals(100, storage.getImageSeq());
storage.setLatestImageSeq(100);
Assert.assertEquals(100, storage.getLatestImageSeq());

storage.setEditsSeq(100);
Assert.assertEquals(100, storage.getEditsSeq());
Expand Down