Skip to content

add and check indexDirs in Cookie meta #3372

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eolivelli merged 1 commit into from
Jul 1, 2022
Merged

add and check indexDirs in Cookie meta #3372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions bookkeeper-proto/src/main/proto/DataFormats.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ message CookieFormat {
required string journalDir = 2;
required string ledgerDirs = 3;
optional string instanceId = 4;
optional string indexDirs = 5;
}

/**
Expand Down
60 changes: 52 additions & 8 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Cookie.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,20 +68,23 @@
public class Cookie {
private static final Logger LOG = LoggerFactory.getLogger(Cookie.class);

static final int CURRENT_COOKIE_LAYOUT_VERSION = 4;
static final int CURRENT_COOKIE_LAYOUT_VERSION = 5;
private final int layoutVersion;
private final String bookieId;
private final String journalDirs;
private final String ledgerDirs;
private final String indexDirs;
private final String instanceId;
private static final String SEPARATOR = "\t";

private Cookie(int layoutVersion, String bookieId, String journalDirs, String ledgerDirs, String instanceId) {
private Cookie(int layoutVersion, String bookieId, String journalDirs, String ledgerDirs, String instanceId,
String indexDirs) {
this.layoutVersion = layoutVersion;
this.bookieId = bookieId;
this.journalDirs = journalDirs;
this.ledgerDirs = ledgerDirs;
this.instanceId = instanceId;
this.indexDirs = indexDirs;
}

public static String encodeDirPaths(String[] dirs) {
Expand All @@ -104,6 +107,13 @@ String[] getLedgerDirPathsFromCookie() {
return decodeDirPathFromCookie(ledgerDirs);
}

String[] getIndexDirPathsFromCookie() {
if (null == indexDirs) {
return null;
}
return decodeDirPathFromCookie(indexDirs);
}

/**
* Receives 2 String arrays, that each contain a list of directory paths,
* and checks if first is a super set of the second.
Expand All @@ -126,14 +136,31 @@ private boolean verifyLedgerDirs(Cookie c, boolean checkIfSuperSet) {
}
}

private boolean verifyIndexDirs(Cookie c, boolean checkIfSuperSet) {
// Compatible with old cookie
if (null == indexDirs && null == c.indexDirs) {
return true;
}
if (null == indexDirs || null == c.indexDirs) {
return false;
}

if (!checkIfSuperSet) {
return indexDirs.equals(c.indexDirs);
} else {
return isSuperSet(decodeDirPathFromCookie(indexDirs), decodeDirPathFromCookie(c.indexDirs));
}
}

private void verifyInternal(Cookie c, boolean checkIfSuperSet) throws BookieException.InvalidCookieException {
String errMsg;
if (c.layoutVersion < 3 && c.layoutVersion != layoutVersion) {
errMsg = "Cookie is of too old version " + c.layoutVersion;
LOG.error(errMsg);
throw new BookieException.InvalidCookieException(errMsg);
} else if (!(c.layoutVersion >= 3 && c.bookieId.equals(bookieId)
&& c.journalDirs.equals(journalDirs) && verifyLedgerDirs(c, checkIfSuperSet))) {
&& c.journalDirs.equals(journalDirs) && verifyLedgerDirs(c, checkIfSuperSet)
&& verifyIndexDirs(c, checkIfSuperSet))) {
errMsg = "Cookie [" + this + "] is not matching with [" + c + "]";
throw new BookieException.InvalidCookieException(errMsg);
} else if ((instanceId == null && c.instanceId != null)
Expand Down Expand Up @@ -165,6 +192,10 @@ public String toString() {
if (null != instanceId) {
builder.setInstanceId(instanceId);
}
if (null != indexDirs) {
builder.setIndexDirs(indexDirs);
}

StringBuilder b = new StringBuilder();
b.append(CURRENT_COOKIE_LAYOUT_VERSION).append("\n");
b.append(builder.build());
Expand Down Expand Up @@ -209,6 +240,9 @@ private static Builder parse(BufferedReader reader) throws IOException {
if (null != data.getInstanceId() && !data.getInstanceId().isEmpty()) {
cBuilder.setInstanceId(data.getInstanceId());
}
if (null != data.getIndexDirs() && !data.getIndexDirs().isEmpty()) {
cBuilder.setIndexDirs(data.getIndexDirs());
}
}
return cBuilder;
}
Expand Down Expand Up @@ -304,6 +338,9 @@ public static Builder generateCookie(ServerConfiguration conf)
builder.setBookieId(BookieImpl.getBookieId(conf).toString());
builder.setJournalDirs(Joiner.on(',').join(conf.getJournalDirNames()));
builder.setLedgerDirs(encodeDirPaths(conf.getLedgerDirNames()));
if (null != conf.getIndexDirNames()) {
builder.setIndexDirs(encodeDirPaths(conf.getIndexDirNames()));
}
return builder;
}

Expand Down Expand Up @@ -398,17 +435,19 @@ public static class Builder {
private String journalDirs = null;
private String ledgerDirs = null;
private String instanceId = null;
private String indexDirs = null;

private Builder() {
}

private Builder(int layoutVersion, String bookieId, String journalDirs, String ledgerDirs,
String instanceId) {
String instanceId, String indexDirs) {
this.layoutVersion = layoutVersion;
this.bookieId = bookieId;
this.journalDirs = journalDirs;
this.ledgerDirs = ledgerDirs;
this.instanceId = instanceId;
this.indexDirs = indexDirs;
}

public Builder setLayoutVersion(int layoutVersion) {
Expand Down Expand Up @@ -436,8 +475,12 @@ public Builder setInstanceId(String instanceId) {
return this;
}

public void setIndexDirs(String indexDirs) {
this.indexDirs = indexDirs;
}

public Cookie build() {
return new Cookie(layoutVersion, bookieId, journalDirs, ledgerDirs, instanceId);
return new Cookie(layoutVersion, bookieId, journalDirs, ledgerDirs, instanceId, indexDirs);
}
}

Expand All @@ -458,7 +501,7 @@ public static Builder newBuilder() {
*/
public static Builder newBuilder(Cookie oldCookie) {
return new Builder(oldCookie.layoutVersion, oldCookie.bookieId, oldCookie.journalDirs, oldCookie.ledgerDirs,
oldCookie.instanceId);
oldCookie.instanceId, oldCookie.indexDirs);
}

@Override
Expand All @@ -469,14 +512,15 @@ public boolean equals(Object other) {
&& Objects.equals(bookieId, otherCookie.bookieId)
&& Objects.equals(journalDirs, otherCookie.journalDirs)
&& Objects.equals(ledgerDirs, otherCookie.ledgerDirs)
&& Objects.equals(instanceId, otherCookie.instanceId);
&& Objects.equals(instanceId, otherCookie.instanceId)
&& Objects.equals(indexDirs, otherCookie.indexDirs);
} else {
return false;
}
}

@Override
public int hashCode() {
return Objects.hash(bookieId, journalDirs, ledgerDirs, instanceId);
return Objects.hash(bookieId, journalDirs, ledgerDirs, instanceId, indexDirs);
}
}
10 changes: 7 additions & 3 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LegacyCookieValidation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,9 +240,13 @@ private static void stampNewCookie(ServerConfiguration conf,

private static Set<File> getKnownDirs(List<Cookie> cookies) {
return cookies.stream()
.flatMap((c) -> Arrays.stream(c.getLedgerDirPathsFromCookie()))
.map((s) -> new File(s))
.collect(Collectors.toSet());
.flatMap((c) -> {
List<String> dirs = new ArrayList<>(Arrays.asList(c.getLedgerDirPathsFromCookie()));
if (null != c.getIndexDirPathsFromCookie()) {
dirs.addAll(Arrays.asList(c.getIndexDirPathsFromCookie()));
}
return Arrays.stream(dirs.toArray(new String[]{}));
}).map((s) -> new File(s)).collect(Collectors.toSet());
}

private static void verifyDirsForStorageExpansion(
Expand Down
Loading