Skip to content

Commit 2f75b52

Browse files
gmagnugmagnu
authored
fix(ENGKNOW-2961): Feature flag, and refactor. (#85)
1 parent 58748f7 commit 2f75b52

File tree

6 files changed

+123
-65
lines changed

6 files changed

+123
-65
lines changed

‎gortools/src/test/java/gorsat/UTestGorWrite.java‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.gorpipe.gor.driver.GorDriverConfig;
3131
import org.gorpipe.gor.driver.linkfile.LinkFile;
3232
import org.gorpipe.gor.driver.linkfile.LinkFileMeta;
33+
import org.gorpipe.gor.driver.linkfile.LinkFileV1;
3334
import org.gorpipe.gor.driver.meta.DataType;
3435
import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource;
3536
import org.gorpipe.gor.model.BaseMeta;
@@ -80,7 +81,7 @@ public void setupTest() throws IOException {
8081
tempRootPath = tempRoot.getRoot().toPath();
8182

8283
var meta = new LinkFileMeta();
83-
meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1");
84+
meta.loadAndMergeMeta(LinkFileV1.getDefaultMetaContent());
8485
meta.setProperty(BaseMeta.HEADER_SERIAL_KEY, "1");
8586
defaultV1LinkFileHeader = meta.formatHeader();
8687
}
@@ -158,19 +159,21 @@ public void testWritePathWithExistingBadLinkFile() throws IOException {
158159
Files.copy(Paths.get("../tests/data/gor/dbsnp_test.gor"), workDirPath.resolve("dbsnp.gor"));
159160
Files.writeString(link, "");
160161
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
161-
var linkUrl = LinkFile.load(new FileSource(link)).getLatestEntryUrl();
162-
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkUrl);
162+
var linkFile = LinkFile.load(new FileSource(link));
163+
Assert.assertEquals("1", linkFile.getMeta().getVersion());
164+
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkFile.getLatestEntryUrl());
163165
}
164166

165167
@Test
166168
public void testWritePathWithExistingBadVersionedLinkFile() throws IOException {
167-
Path p = Paths.get("../tests/data/gor/dbsnp_test.gor");
168-
Files.copy(p, workDirPath.resolve("dbsnp.gor"));
169-
Files.writeString(workDirPath.resolve("dbsnp3.gor.link"), "");
169+
Path link = workDirPath.resolve("dbsnp3.gor.link");
170+
Files.copy(Paths.get("../tests/data/gor/dbsnp_test.gor"), workDirPath.resolve("dbsnp.gor"));
171+
Files.writeString(link, "## VERSION = 1");
170172
TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString());
171173

172-
Assert.assertTrue(Files.readString(workDirPath.resolve("dbsnp3.gor.link")).startsWith(
173-
defaultV1LinkFileHeader + workDirPath.resolve("dbsnp2.gor") + "\t"));
174+
var linkFile = LinkFile.load(new FileSource(link));
175+
Assert.assertEquals("1", linkFile.getMeta().getVersion());
176+
Assert.assertEquals(workDirPath.resolve("dbsnp2.gor").toString(), linkFile.getLatestEntryUrl());
174177
}
175178

176179
@Test

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -66,17 +66,39 @@ public abstract class LinkFile {
6666

6767
public static LinkFile load(StreamSource source) throws IOException {
6868
var content = loadContentFromSource(source);
69-
return create(source, content);
69+
var meta = LinkFileMeta.createOrLoad(content, null, false);
70+
return create(source, meta, content);
7071
}
7172

7273
public static LinkFile create(StreamSource source, String content) {
73-
var meta = LinkFileMeta.createAndLoad(content);
74+
var meta = LinkFileMeta.createOrLoad(content, null, true);
75+
return create(source, meta, content);
76+
}
7477

75-
if ("0".equals(meta.getVersion())) {
76-
return new LinkFileV0(source, meta, content);
77-
} else {
78-
return new LinkFileV1(source, meta, content);
79-
}
78+
public static LinkFile create(StreamSource source, LinkFileMeta meta, String content) {
79+
return switch (meta.getVersion()) {
80+
case "0" -> new LinkFileV0(source, meta, content);
81+
case "1" -> new LinkFileV1(source, meta, content);
82+
default -> throw new GorResourceException("Unsupported link file version: " + meta.getVersion(), source.getFullPath());
83+
};
84+
}
85+
86+
public static LinkFile loadV0(StreamSource source) throws IOException {
87+
var content = loadContentFromSource(source);
88+
return new LinkFileV0(source, LinkFileMeta.createOrLoad(content, LinkFileV0.VERSION, true), content);
89+
}
90+
91+
public static LinkFile loadV1(StreamSource source) throws IOException {
92+
var content = loadContentFromSource(source);
93+
return new LinkFileV1(source, LinkFileMeta.createOrLoad(content, LinkFileV1.VERSION, true), content);
94+
}
95+
96+
public static LinkFile createV0(StreamSource source, String content) throws IOException {
97+
return new LinkFileV0(source, LinkFileMeta.createOrLoad(content, LinkFileV0.VERSION, true), content);
98+
}
99+
100+
public static LinkFile createV1(StreamSource source, String content) throws IOException {
101+
return new LinkFileV1(source, LinkFileMeta.createOrLoad(content, LinkFileV1.VERSION, true), content);
80102
}
81103

82104
public static String validateAndUpdateLinkFileName(String linkFilePath) {
@@ -161,15 +183,6 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
161183
protected final LinkFileMeta meta;
162184
protected final List<LinkFileEntry> entries; // Entries sorted by time (oldest first)
163185

164-
/**
165-
* Create a new link file from source and content.
166-
*
167-
* @param source the source to create the link file from
168-
* @param content the content of the link file, can be empty or null to create an empty link file.
169-
*/
170-
protected LinkFile(StreamSource source, String content) {
171-
this(source, LinkFileMeta.createAndLoad(content), content);
172-
}
173186

174187
protected LinkFile(StreamSource source, LinkFileMeta meta, String content) {
175188
this.source = source;
@@ -334,6 +347,7 @@ private void save(OutputStream os, long timestamp) {
334347
.filter(entry -> entry.timestamp() <= 0 || currentTimestamp - entry.timestamp() <= getEntriesAgeMax())
335348
.forEach(entry -> content.append(entry.format()).append("\n"));
336349
}
350+
337351
try {
338352
os.write(content.toString().getBytes());
339353
} catch (IOException e) {

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import org.gorpipe.gor.model.FileReader;
66
import org.gorpipe.util.Strings;
77

8+
import java.util.stream.Collectors;
9+
810
public class LinkFileMeta extends BaseMeta {
911

1012
// Max number of entries to keep track of in the link file.
@@ -16,24 +18,39 @@ public class LinkFileMeta extends BaseMeta {
1618
// Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false).
1719
public static final String HEADER_CONTENT_LIFECYCLE_MANAGED_KEY = "CONTENT_LIFECYCLE_MANAGED";
1820

19-
public static final String[] DEFAULT_TABLE_HEADER = new String[] {"File", "Timestamp", "MD5", "Serial", "Info"};
21+
private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", "1");
2022

2123
public static final int DEFAULT_ENTRIES_COUNT_MAX = 100;
2224
public static final long DEFAULT_ENTRIES_AGE_MAX = Long.MAX_VALUE;
2325

24-
public static LinkFileMeta createAndLoad(String metaContent) {
26+
/**
27+
* Create or load link file meta from content.
28+
* @param content
29+
* @param version version if known, otherwise null. Only used if content is null or empty.
30+
* @param isNew true if creating new link file meta, false if loading existing.
31+
* @return
32+
*/
33+
public static LinkFileMeta createOrLoad(String content, String version, boolean isNew) {
34+
var metaContent = !Strings.isNullOrEmpty(content) ? content.lines().filter(line -> line.startsWith("#")).collect(Collectors.joining("\n")) : "";
2535
LinkFileMeta meta = new LinkFileMeta();
26-
if (Strings.isNullOrEmpty(metaContent)) {
27-
meta.loadAndMergeMeta(getDefaultMetaContent());
28-
} else {
29-
meta.loadAndMergeMeta(metaContent);
36+
if (Strings.isNullOrEmpty(metaContent) ) {
37+
// No meta, determine version to use
38+
if (Strings.isNullOrEmpty(version)) {
39+
version = Strings.isNullOrEmpty(content) || isNew ? DEFAULT_VERSION : LinkFileV0.VERSION;
40+
}
41+
42+
metaContent = switch(version) {
43+
case "0" -> LinkFileV0.getDefaultMetaContent();
44+
case "1" -> LinkFileV1.getDefaultMetaContent();
45+
default -> throw new IllegalArgumentException("Unsupported link file version: " + meta.getVersion());
46+
};
3047
}
48+
meta.loadAndMergeMeta(metaContent);
3149
return meta;
3250
}
3351

3452
public LinkFileMeta() {
3553
super();
36-
setFileHeader(DEFAULT_TABLE_HEADER);
3754
saveHeaderLine = true;
3855
}
3956

@@ -63,13 +80,8 @@ public void setEntriesAgeMax(int entriesAgeMax) {
6380

6481
@Override
6582
public String getVersion() {
66-
return getProperty(HEADER_VERSION_KEY, "0");
83+
return getProperty(HEADER_VERSION_KEY, DEFAULT_VERSION);
6784
}
6885

69-
public static String getDefaultMetaContent() {
70-
return String.format("""
71-
## SERIAL = 0
72-
## VERSION = 1
73-
""");
74-
}
86+
7587
}

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV0.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,8 @@
1111
*/
1212
public class LinkFileV0 extends LinkFile {
1313

14-
/**
15-
* Load from a source, if it exists, otherwise create an empty link file.
16-
*
17-
* @param source the source to load from
18-
*/
19-
public LinkFileV0(StreamSource source) throws IOException {
20-
super(source, loadContentFromSource(source));
21-
}
14+
public static final String VERSION = "0";
15+
2216

2317
protected LinkFileV0(StreamSource source, LinkFileMeta meta, String content) {
2418
super(source, meta, content);
@@ -40,4 +34,10 @@ public LinkFile appendEntry(String link, String md5, String info, FileReader rea
4034
entries.add(new LinkFileEntryV0(link));
4135
return this;
4236
}
37+
38+
public static String getDefaultMetaContent() {
39+
return String.format("""
40+
## VERSION = 0
41+
""");
42+
}
4343
}

‎model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,12 @@
1212
*/
1313
public class LinkFileV1 extends LinkFile {
1414

15+
public static final String VERSION = "1";
16+
public static final String DEFAULT_TABLE_HEADER = "#File\tTimestamp\tMD5\tSerial\tInfo";
17+
1518
private static boolean allowOverwriteOfTargets
1619
= Boolean.parseBoolean(System.getProperty("gor.link.versioned.allow.overwrite", "false"));
1720

18-
/**
19-
* Load from a source, if it exists, otherwise create an empty link file.
20-
*
21-
* @param source the source to load from
22-
*/
23-
public LinkFileV1(StreamSource source) throws IOException {
24-
super(source, loadContentFromSource(source));
25-
checkDefaultMeta();
26-
}
27-
2821
protected LinkFileV1(StreamSource source, LinkFileMeta meta, String content) {
2922
super(source, meta, content);
3023
checkDefaultMeta();
@@ -79,9 +72,16 @@ private boolean canReuseEntryWithSameUrl(LinkFileEntry oldEntry, LinkFileEntry n
7972
}
8073

8174
private void checkDefaultMeta() {
82-
if (!meta.containsProperty(BaseMeta.HEADER_VERSION_KEY)) {
83-
getMeta().loadAndMergeMeta(LinkFileMeta.getDefaultMetaContent());
84-
meta.setProperty(BaseMeta.HEADER_VERSION_KEY, "1");
75+
if (!meta.getVersion().equals(VERSION)) {
76+
meta.loadAndMergeMeta(getDefaultMetaContent());
8577
}
8678
}
79+
80+
public static String getDefaultMetaContent() {
81+
return String.format("""
82+
## SERIAL = 0
83+
## VERSION = %s
84+
%s
85+
""", VERSION, DEFAULT_TABLE_HEADER);
86+
}
8787
}

‎model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import org.gorpipe.gor.driver.providers.stream.sources.StreamSource;
66
import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource;
77
import org.junit.Before;
8+
import org.junit.Ignore;
89
import org.junit.Rule;
910
import org.junit.Test;
1011
import org.junit.contrib.java.lang.system.EnvironmentVariables;
@@ -34,6 +35,10 @@ public class LinkFileTest {
3435
public final EnvironmentVariables environmentVariables
3536
= new EnvironmentVariables();
3637

38+
@Rule
39+
public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
40+
41+
3742
private StreamSource mockSource;
3843
private final String v1LinkFileContent = """
3944
## SERIAL = 1
@@ -59,10 +64,34 @@ public void setUp() {
5964
public void testCreateLinkFile() {
6065
LinkFile linkFile = LinkFile.create(mockSource, v1LinkFileContent);
6166
assertNotNull(linkFile);
67+
assertTrue(linkFile instanceof LinkFileV1);
68+
assertEquals("1", linkFile.getMeta().getVersion());
6269
assertEquals(2, linkFile.getEntries().size());
6370
assertEquals(100, linkFile.getEntriesCountMax());
6471
}
6572

73+
@Test
74+
public void testCreateLinkFileSimple() {
75+
LinkFile linkFile = LinkFile.create(mockSource, "test.gorz");
76+
assertNotNull(linkFile);
77+
assertTrue(linkFile instanceof LinkFileV1);
78+
assertEquals("1", linkFile.getMeta().getVersion());
79+
assertEquals(1, linkFile.getEntries().size());
80+
assertEquals(100, linkFile.getEntriesCountMax());
81+
}
82+
83+
@Ignore("Fiddly test depending on system properties, ignore for now. Can run in isolation to verify.")
84+
@Test
85+
public void testCreateLinkFileSimpleWithDefault0() {
86+
System.setProperty("gor.driver.link.default.version", "0");
87+
LinkFile linkFile = LinkFile.create(mockSource, "test.gorz");
88+
assertNotNull(linkFile);
89+
assertTrue(linkFile instanceof LinkFileV0);
90+
assertEquals("0", linkFile.getMeta().getVersion());
91+
assertEquals(1, linkFile.getEntries().size());
92+
assertEquals(100, linkFile.getEntriesCountMax());
93+
}
94+
6695
@Test
6796
public void testLoadLinkFile() throws IOException {
6897
when(mockSource.exists()).thenReturn(true);
@@ -105,7 +134,7 @@ public void testGetTimedPath() {
105134
@Test
106135
public void testSaveNewV1LinkFile() throws IOException {
107136
var linkPath = workPath.resolve("test.link");
108-
LinkFile linkFile = new LinkFileV1(new FileSource(linkPath.toString()));
137+
LinkFile linkFile = LinkFile.createV1(new FileSource(linkPath.toString()), "");
109138
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
110139
linkFile.save();
111140
String savedContent = Files.readString(linkPath);
@@ -116,7 +145,7 @@ public void testSaveNewV1LinkFile() throws IOException {
116145
@Test
117146
public void testSaveNewV0LinkFile() throws IOException {
118147
var linkPath = workPath.resolve("test.link");
119-
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
148+
LinkFile linkFile = LinkFile.createV0(new FileSource(linkPath.toString()), "");
120149
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
121150
linkFile.save();
122151
String savedContent = Files.readString(linkPath);
@@ -139,7 +168,7 @@ public void testSaveLinkFileV1ToV1() throws IOException {
139168
public void testSaveLinkFileV0ToV0() throws IOException {
140169
var linkPath = workPath.resolve("test.link");
141170
Files.writeString(linkPath, "a/b/c.gorz");
142-
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
171+
LinkFile linkFile = LinkFile.load(new FileSource(linkPath.toString()));
143172
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
144173
linkFile.save();
145174
String savedContent = Files.readString(linkPath);
@@ -150,26 +179,26 @@ public void testSaveLinkFileV0ToV0() throws IOException {
150179
public void testSaveLinkFileV0ToV1() throws IOException {
151180
var linkPath = workPath.resolve("test.link");
152181
Files.writeString(linkPath, "a/b/c.gorz");
153-
LinkFile linkFile = new LinkFileV1(new FileSource(linkPath.toString()));
182+
LinkFile linkFile = LinkFile.loadV1(new FileSource(linkPath.toString()));
154183
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
155184
linkFile.save();
156185
String savedContent = Files.readString(linkPath);
157186
assertTrue(savedContent.contains("## VERSION = 1"));
187+
assertEquals(2, linkFile.getEntries().size());
158188
assertTrue(savedContent.contains(simpleFile));
159189
}
160190

161191
@Test
162192
public void testSaveLinkFileV1ToV0() throws IOException {
163193
var linkPath = workPath.resolve("test.link");
164194
Files.writeString(linkPath, v1LinkFileContent);
165-
LinkFile linkFile = new LinkFileV0(new FileSource(linkPath.toString()));
195+
LinkFile linkFile = LinkFile.loadV0(new FileSource(linkPath.toString()));
166196
linkFile.appendEntry(simpleFile, "NEWMD5SUM");
167197
linkFile.save();
168198
String savedContent = Files.readString(linkPath);
169199
assertEquals(simpleFile, savedContent.trim());
170200
}
171201

172-
173202
@Test(expected = IllegalArgumentException.class)
174203
public void testInferDataFileNameFromLinkFile_NullOrEmptyPath() throws Exception {
175204
LinkFile.inferDataFileNameFromLinkFile(new FileSource(""), null);

0 commit comments

Comments
 (0)