diff --git a/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java b/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java index 042df1e2..4e825046 100644 --- a/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java +++ b/drivers/src/test/java/org/gorpipe/s3/shared/ITestS3Shared.java @@ -128,7 +128,7 @@ public void testProjectRead() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/projects/some_project/BVL_FATHER_SLC52A2/BVL_FATHER_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHUUUART", iterator.getHeader()); } } @@ -144,7 +144,7 @@ public void testProjectReadUserData() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/projects/some_project/user_data/BVL_INDEX_SLC52A2/BVL_INDEX_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNVKUKNN", iterator.getHeader()); } } @@ -161,7 +161,7 @@ public void testProjectSharedRead() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_MOTHER_SLC52A2/BVL_MOTHER_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNKHARLQ", iterator.getHeader()); } } @@ -177,7 +177,7 @@ public void testRegionSharedRead() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_MOTHER_SLC52A2/BVL_MOTHER_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNKHARLQ", iterator.getHeader()); } } @@ -193,7 +193,7 @@ public void testGlobalSharedRead() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_MOTHER_SLC52A2/BVL_MOTHER_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNKHARLQ", iterator.getHeader()); } } @@ -209,7 +209,7 @@ public void testProjectFileRead() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/projects/some_project/BVL_MOTHER_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNKHARLQ", iterator.getHeader()); } } @@ -225,7 +225,7 @@ public void testProjectSharedFileRead() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_FATHER_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHUUUART", iterator.getHeader()); } } @@ -241,7 +241,7 @@ public void testRegionSharedFileRead() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_FATHER_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHUUUART", iterator.getHeader()); } } @@ -257,14 +257,14 @@ public void testGlobalSharedFileRead() throws IOException { Assert.assertEquals("S3", source.getSourceType().getName()); Assert.assertEquals("s3://gdb-unit-test-data/shared/BVL_FATHER_SLC52A2.vcf.gz.gorz", source.getFullPath()); - try(GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { + try (GenomicIterator iterator = PluggableGorDriver.instance().createIterator(source)) { Assert.assertEquals("CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHUUUART", iterator.getHeader()); } } @Test public void testReadWithLinkFile() throws IOException { - Path gorRoot = workDirPath.resolve("some_project"); + Path gorRoot = workDirPath.resolve("some_project"); Path linkFile = gorRoot.resolve(DataUtil.toLinkFile("a", DataType.GORZ)); Files.createDirectory(gorRoot); Files.writeString(linkFile, DataUtil.toFile(DataUtil.toFile("s3data://project/user_data/BVL_INDEX_SLC52A2", DataType.VCFGZ), DataType.GORZ)); @@ -290,7 +290,7 @@ public void testReadWithLinkFile() throws IOException { @Test public void testReadWithUnaccessableLinkFile() throws IOException { - Path gorRoot = workDirPath.resolve("some_project"); + Path gorRoot = workDirPath.resolve("some_project"); Path linkFile = workDirPath.resolve(DataUtil.toLinkFile("a", DataType.GORZ)); Files.createDirectory(gorRoot); Files.writeString(linkFile, DataUtil.toFile(DataUtil.toFile("s3data://project/user_data/BVL_INDEX_SLC52A2", DataType.VCFGZ), DataType.GORZ)); @@ -303,12 +303,12 @@ public void testReadWithUnaccessableLinkFile() throws IOException { var e2 = Assert.assertThrows("Should not be resolved, link outside project", GorSecurityException.class, () -> fileReader.resolveUrl(DataUtil.toLinkFile("../a", DataType.GORZ))); Assert.assertTrue(e2.getMessage().contains("File paths must be within project scope")); - } + } @Test @Ignore("Slow test, meant to be manually run") public void testReadServer() throws IOException { - Path gorRoot = workDirPath.resolve("some_project"); + Path gorRoot = workDirPath.resolve("some_project"); Path linkFile = gorRoot.resolve(DataUtil.toLinkFile("a", DataType.GORZ)); Files.createDirectory(gorRoot); //Files.write(linkFile, "s3data://project/user_data/BVL_INDEX_SLC52A2.vcf.gz.gorz".getBytes(StandardCharsets.UTF_8)); @@ -324,7 +324,7 @@ public void testReadServer() throws IOException { @Test public void testProjectWriteRootCLI() throws IOException { String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET); - String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); String dataPath = DataUtil.toFile("dummy", DataType.GOR); runGorPipeCLI("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext); @@ -341,7 +341,7 @@ public void testProjectWriteRootCLI() throws IOException { @Test public void testProjectWriteUserDataCLI() throws IOException { String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET); - String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); String dataPath = DataUtil.toFile("user_data/dummy", DataType.GOR); runGorPipeCLI("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext); @@ -358,7 +358,7 @@ public void testProjectWriteUserDataCLI() throws IOException { @Test public void testProjectWriteRootServer() { String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET); - String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); String dataPath = DataUtil.toFile("dummy", DataType.GOR); var e = Assert.assertThrows("Server context should not allow write to root", GorSecurityException.class, () -> runGorPipeServer("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext)); @@ -368,7 +368,7 @@ public void testProjectWriteRootServer() { @Test public void testProjectWriteUserDataServer() throws IOException { String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET); - String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); String dataPath = DataUtil.toFile("user_data/dummy", DataType.GOR); runGorPipeServer("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext); @@ -385,7 +385,7 @@ public void testProjectWriteUserDataServer() throws IOException { @Test public void testProjectWriteUserDataWithIndexServer() throws IOException { String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET); - String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); String dataPath = "user_data/dummy.gorz"; runGorPipeServer("gorrow 1,2,3 | write s3data://project/" + dataPath, gorRoot, securityContext); @@ -408,7 +408,7 @@ public void testProjectWriteUserDataWithIndexServer() throws IOException { @Test public void testSharedWriteRootServer() { String securityContext = createSecurityContext("s3region", Credentials.OwnerType.System, "some_env", S3_KEY, S3_SECRET); - String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); String dataPath = DataUtil.toFile("dummy", DataType.GOR); @@ -419,7 +419,7 @@ public void testSharedWriteRootServer() { @Test public void testSharedWriteUserDataServer() throws IOException { String securityContext = createSecurityContext("s3region", Credentials.OwnerType.System, "some_env", S3_KEY, S3_SECRET); - String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); String dataPath = DataUtil.toFile("user_data/dummy", DataType.GOR); runGorPipeServer("gorrow 1,2,3 | write s3region://shared/" + dataPath, gorRoot, securityContext); @@ -433,6 +433,51 @@ public void testSharedWriteUserDataServer() throws IOException { Assert.assertTrue(Files.exists(Path.of(gorRoot, DataUtil.toFile(dataPath, DataType.LINK)))); } + @Test + public void testSharedWriteUserDataServerTeeed() throws IOException { + String securityContext = createSecurityContext("s3region", Credentials.OwnerType.System, "some_env", S3_KEY, S3_SECRET); + String gorRoot = Path.of(workDir.getRoot().toString(), "some_project").toString(); + String dataPath = DataUtil.toFile("user_data/dummy", DataType.GOR); + String dataPath2 = DataUtil.toFile("user_data/dummy2", DataType.GOR); + String dataPath3 = DataUtil.toFile("user_data/dummy3", DataType.GOR); + + runGorPipeServer("gorrows -p chr1:10000-10005| tee >(where 1 = 1 | write -c s3region://shared/" + dataPath + + ") | tee >(where 1 = 1 | write -c s3region://shared/" + dataPath2 + + ") | where 1 = 1 | write -c s3region://shared/" + dataPath3 + , gorRoot, securityContext); + + runGorPipeServer("gorrows -p chr1:10000-10005| tee >(where 1 = 1 | write -c s3region://shared/" + dataPath + + ") | tee >(where 1 = 1 | write -c s3region://shared/" + dataPath2 + + ") | where 1 = 1 | write -c s3region://shared/" + dataPath3 + , gorRoot, securityContext); + + Assert.assertTrue(Files.exists(Path.of(gorRoot, DataUtil.toFile(dataPath, DataType.LINK)))); + Assert.assertTrue(Files.exists(Path.of(gorRoot, DataUtil.toFile(dataPath2, DataType.LINK)))); + Assert.assertTrue(Files.exists(Path.of(gorRoot, DataUtil.toFile(dataPath3, DataType.LINK)))); + + // Clean up.. + S3SharedSourceProvider provider = new S3RegionSharedSourceProvider(); + provider.setConfig(ConfigManager.getPrefixConfig("gor",GorDriverConfig .class)); + try( + DataSource source = getDataSourceFromProvider(provider, dataPath, Credentials.OwnerType.System, "some_env")) + + { + source.delete(); + } + try( + DataSource source = getDataSourceFromProvider(provider, dataPath2, Credentials.OwnerType.System, "some_env")) + + { + source.delete(); + } +} + + + +/* + | tee >(where depth >= 4 | segspan -maxseg 10000 | write -c ${OUTPUT_WGS_SEG_GOODCOV_4}) + | tee >(where depth >= 4 | segspan -maxseg 10000 | hide segCount | join -segseg ${JOIN_EXONS} | write -c ${OUTPUT_WES_SEG_GOODCOV_4}) +*/ @Test public void testProjecSharedProjecttWriteUserDataServer() throws IOException { String securityContext = createSecurityContext("s3data", Credentials.OwnerType.Project, "some_project", S3_KEY, S3_SECRET); diff --git a/gortools/src/main/java/gorsat/process/GorJavaUtilities.java b/gortools/src/main/java/gorsat/process/GorJavaUtilities.java index 6f7d8365..c4087c08 100644 --- a/gortools/src/main/java/gorsat/process/GorJavaUtilities.java +++ b/gortools/src/main/java/gorsat/process/GorJavaUtilities.java @@ -498,13 +498,13 @@ public static void writeDictionaryFromMeta(String commandToExecute, FileReader f } else writeDummyHeader(dictionarypathwriter); } - localFileReader.writeLinkIfNeeded(dictionarypath); - var linkOptions = LinkFileUtil.extractLinkOptionData(commandToExecute); if (!Strings.isNullOrEmpty(linkOptions)) { var linkMetaOption = LinkFileUtil.extractLinkMetaOptionData(commandToExecute); var linkData = LinkFileUtil.extractLink(fileReader, outfolderpath, linkOptions, linkMetaOption, null); LinkFileUtil.writeLinkFile(fileReader, linkData); + } else { + localFileReader.writeLinkIfNeeded(dictionarypath); } } diff --git a/gortools/src/test/java/gorsat/UTestGorWrite.java b/gortools/src/test/java/gorsat/UTestGorWrite.java index fea6684a..7d5ba8d5 100644 --- a/gortools/src/test/java/gorsat/UTestGorWrite.java +++ b/gortools/src/test/java/gorsat/UTestGorWrite.java @@ -178,23 +178,30 @@ public void testWritePathWithExistingBadVersionedLinkFile() throws IOException { @Test public void testOverwritePathWithExistingVersionedLinkFile() throws IOException { - Path p = Paths.get("../tests/data/gor/dbsnp_test.gor"); - Files.copy(p, workDirPath.resolve("dbsnp.gor")); - - TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString()); - LinkFile linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gor.link").toString())); - Assert.assertEquals(1, linkFile.getEntriesCount()); + var oldAllow = LinkFileV1.allowOverwriteOfTargets; + LinkFileV1.allowOverwriteOfTargets = false; - // Test with same file. - TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString()); - linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gor.link").toString())); - Assert.assertEquals(2, linkFile.getEntriesCount()); - - // Test with different file - Assert.assertThrows( "Overwriting link with same path, throws exception", - GorSystemException.class, - () -> TestUtils.runGorPipe("gor dbsnp.gor | top 1 | write dbsnp2.gor -link dbsnp3.gor", - "-gorroot", workDirPath.toString())); + try { + Path p = Paths.get("../tests/data/gor/dbsnp_test.gor"); + Files.copy(p, workDirPath.resolve("dbsnp.gor")); + + TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString()); + LinkFile linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gor.link").toString())); + Assert.assertEquals(1, linkFile.getEntriesCount()); + + // Test with same file. + TestUtils.runGorPipe("gor dbsnp.gor | write dbsnp2.gor -link dbsnp3.gor", "-gorroot", workDirPath.toString()); + linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gor.link").toString())); + Assert.assertEquals(2, linkFile.getEntriesCount()); + + // Test with different file + Assert.assertThrows("Overwriting link with same path, throws exception", + GorSystemException.class, + () -> TestUtils.runGorPipe("gor dbsnp.gor | top 1 | write dbsnp2.gor -link dbsnp3.gor", + "-gorroot", workDirPath.toString())); + } finally { + LinkFileV1.allowOverwriteOfTargets = oldAllow; + } } @Test diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java index ef96d488..d499afae 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java @@ -67,6 +67,12 @@ public static LinkFile load(StreamSource source) throws IOException { return create(source, meta, content); } + public static LinkFile createOrLoad(StreamSource source, String version) throws IOException { + var content = loadContentFromSource(source); + var meta = LinkFileMeta.createOrLoad(content, version, false); + return create(source, meta, content); + } + public static LinkFile create(StreamSource source, String content) { var meta = LinkFileMeta.createOrLoad(content, null, true); return create(source, meta, content); diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java index eca7c7fa..d79a5a01 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java @@ -17,7 +17,7 @@ public class LinkFileMeta extends BaseMeta { // Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false). public static final String HEADER_DATA_LIFECYCLE_MANAGED_KEY = "DATA_LIFECYCLE_MANAGED"; - private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", LinkFileV0.VERSION); + static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", LinkFileV0.VERSION); public static final int DEFAULT_ENTRIES_COUNT_MAX = 100; public static final long DEFAULT_ENTRIES_AGE_MAX = Long.MAX_VALUE; diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java index 39c0b613..0597ec07 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java @@ -95,7 +95,7 @@ public static String extractLinkMetaOptionData(String options) { return ""; } - public record LinkData(String linkFile, String linkFileContent, String linkFileMeta, String linkFileInfo, String md5) {} + public record LinkData(String linkFile, String linkFileContent, String linkFileMeta, String linkFileInfo, String md5, String version) {} /** @@ -111,6 +111,7 @@ public record LinkData(String linkFile, String linkFileContent, String linkFileM public static LinkData extractLink(FileReader fileReader, String source, String optLinkFile, String optLinkFileMeta, String md5) { var linkFile = LinkFile.validateAndUpdateLinkFileName(optLinkFile); var linkFileContent = !Strings.isNullOrEmpty(linkFile) ? PathUtils.resolve(fileReader.getCommonRoot(), source) : ""; + var version = LinkFileV1.VERSION; // Default to V1 if (Strings.isNullOrEmpty(linkFile) && !Strings.isNullOrEmpty(source)) { // Check if link file is forced from the source @@ -118,10 +119,11 @@ public static LinkData extractLink(FileReader fileReader, String source, String if (dataSource != null && dataSource.forceLink()) { linkFile = dataSource.getProjectLinkFile(); linkFileContent = dataSource.getProjectLinkFileContent(); + version = LinkFileMeta.DEFAULT_VERSION; // Use default version when forced from source. } } var metaInfo = extractLinkMetaInfo(optLinkFileMeta); - return new LinkData(linkFile, linkFileContent, metaInfo.linkFileMeta, metaInfo.linkFileInfo, md5); + return new LinkData(linkFile, linkFileContent, metaInfo.linkFileMeta, metaInfo.linkFileInfo, md5, version); } public static LinkData extractLinkMetaInfo(String optLinkFileMeta) { @@ -139,7 +141,7 @@ public static LinkData extractLinkMetaInfo(String optLinkFileMeta) { } } - return new LinkData("", "", linkFileMeta, linkFileInfo, ""); + return new LinkData("", "", linkFileMeta, linkFileInfo, "", ""); } public static void writeLinkFile(FileReader fileReader, LinkData linkData) throws IOException { @@ -149,7 +151,7 @@ public static void writeLinkFile(FileReader fileReader, LinkData linkData) throw // Use the nonsecure driver file reader as this is an exception from the write no links rule. var unsecureFileReader = fileReader.unsecure(); - LinkFile.loadV1((StreamSource)unsecureFileReader.resolveUrl(linkData.linkFile, true)) + LinkFile.createOrLoad((StreamSource)unsecureFileReader.resolveUrl(linkData.linkFile, true), linkData.version) .appendMeta(linkData.linkFileMeta) .appendEntry(linkData.linkFileContent, linkData.md5, linkData.linkFileInfo, unsecureFileReader) .save(unsecureFileReader.getQueryTime()); diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java index 7bc162e6..f6d70c1c 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileV1.java @@ -15,8 +15,8 @@ public class LinkFileV1 extends LinkFile { public static final String VERSION = "1"; public static final String DEFAULT_TABLE_HEADER = "#File\tTimestamp\tMD5\tSerial\tInfo"; - private static boolean allowOverwriteOfTargets - = Boolean.parseBoolean(System.getProperty("gor.link.versioned.allow.overwrite", "false")); + public static boolean allowOverwriteOfTargets + = Boolean.parseBoolean(System.getProperty("gor.link.versioned.allow.overwrite", "true")); protected LinkFileV1(StreamSource source, LinkFileMeta meta, String content) { super(source, meta, content); diff --git a/model/src/main/java/org/gorpipe/gor/driver/meta/DataType.java b/model/src/main/java/org/gorpipe/gor/driver/meta/DataType.java index 5d3c4d54..990fdd23 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/meta/DataType.java +++ b/model/src/main/java/org/gorpipe/gor/driver/meta/DataType.java @@ -55,6 +55,7 @@ public enum DataType { GORQ(REPORT, ".gorq"), LINK(REFERENCE, ".link"), VERSIONED_LINK(REFERENCE, ".versioned.link"), + GORD_INTERNAL_LINK(REFERENCE, ".internal.link"), // Gord internal link type CRAM(VARIANTS, ".cram", true), CRAI(INDEX, ".crai"), SPEC(VARIANTS, ".spec"), @@ -112,7 +113,7 @@ public static boolean isOfType(String file, DataType type) { private static boolean isLinkToType(String file, DataType type) { var base = PathUtils.stripTrailingSlash(file.trim()).toLowerCase(); return base.endsWith(type.suffix + DataType.LINK.suffix) - || base.endsWith(type.suffix + DataType.VERSIONED_LINK.suffix); + || base.endsWith(type.suffix + DataType.GORD_INTERNAL_LINK.suffix); } public static boolean isOfTypeOrLinksToType(String file, DataType type) { diff --git a/model/src/main/java/org/gorpipe/gor/table/livecycle/TableInfoBase.java b/model/src/main/java/org/gorpipe/gor/table/livecycle/TableInfoBase.java index d6bbcc8e..19814bf9 100644 --- a/model/src/main/java/org/gorpipe/gor/table/livecycle/TableInfoBase.java +++ b/model/src/main/java/org/gorpipe/gor/table/livecycle/TableInfoBase.java @@ -72,12 +72,12 @@ protected TableInfoBase(String uri, FileReader inputFileReader, TableHeader head var fileName = PathUtils.getFileName(source.getFullPath()); this.name = FilenameUtils.removeExtension(fileName); - if (safeCheckExists(PathUtils.resolve(realUri, DataUtil.toLink(GorOptions.DEFAULT_FOLDER_DICTIONARY_NAME)))) { + if (safeCheckExists(PathUtils.resolve(realUri, DataUtil.toFile(GorOptions.DEFAULT_FOLDER_DICTIONARY_NAME, DataType.GORD_INTERNAL_LINK)))) { // Not all data sources support isDirectory (so just check for the dict file) // GORDFOLDER VERSIONED: with folder (containing versioned link file) passed in. this.rootUri = realUri; this.folderPath = rootUri; - this.linkPath = PathUtils.resolve(rootUri, DataUtil.toLink(GorOptions.DEFAULT_FOLDER_DICTIONARY_NAME)); + this.linkPath = PathUtils.resolve(rootUri, DataUtil.toFile(GorOptions.DEFAULT_FOLDER_DICTIONARY_NAME, DataType.GORD_INTERNAL_LINK)); try (var linkFileSource = fileReader.resolveUrl(linkPath)) { this.path = linkFileSource != null ? linkFileSource.getFullPath() : getNewVersionedFileName(); } @@ -98,7 +98,7 @@ protected TableInfoBase(String uri, FileReader inputFileReader, TableHeader head // GORDFOLDER VERSIONED: New dict with USE_LINKS=true (versioned). this.rootUri = realUri; this.folderPath = rootUri; - this.linkPath = PathUtils.resolve(rootUri, DataUtil.toLink(GorOptions.DEFAULT_FOLDER_DICTIONARY_NAME)); + this.linkPath = PathUtils.resolve(rootUri, DataUtil.toFile(GorOptions.DEFAULT_FOLDER_DICTIONARY_NAME, DataType.GORD_INTERNAL_LINK)); this.path = getNewVersionedFileName(); } } diff --git a/model/src/main/java/org/gorpipe/gor/util/DataUtil.java b/model/src/main/java/org/gorpipe/gor/util/DataUtil.java index 4d2211a0..66ef7d67 100644 --- a/model/src/main/java/org/gorpipe/gor/util/DataUtil.java +++ b/model/src/main/java/org/gorpipe/gor/util/DataUtil.java @@ -118,21 +118,17 @@ public static boolean isGZip(String file) { } public static String toFile(String name, DataType type) { - return name + type.suffix; + return PathUtils.stripTrailingSlash(name) + type.suffix; } public static String toLinkFile(String name, DataType type) { - return name + type.suffix + DataType.LINK.suffix; + return PathUtils.stripTrailingSlash(name) + type.suffix + DataType.LINK.suffix; } public static String toLink(String path) { return DataUtil.isLink(path) ? path : PathUtils.stripTrailingSlash(path) + DataType.LINK.suffix; } - public static String toVersionedLink(String path) { - return PathUtils.stripTrailingSlash(path) + DataType.VERSIONED_LINK.suffix; - } - public static String toTempTempFile(String file) { var type = DataType.fromFileName(file);