diff --git a/gortools/src/main/java/gorsat/process/GorJavaUtilities.java b/gortools/src/main/java/gorsat/process/GorJavaUtilities.java index 4f0b392a..6f7d8365 100644 --- a/gortools/src/main/java/gorsat/process/GorJavaUtilities.java +++ b/gortools/src/main/java/gorsat/process/GorJavaUtilities.java @@ -432,17 +432,28 @@ public static void createSymbolicLinkSafe(Path resultPath, Path cachePath) throw */ public static String verifyLinkFileLastModified(ProjectContext projectContext, String cacheFile) { if (cacheFile != null && DataUtil.isLink(cacheFile)) { + var invalidCacheFile = false; try { var ds = projectContext.getFileReader().resolveUrl(cacheFile); var linkLastModified = ds.getSourceMetadata().getLinkLastModified(); var lastModified = ds.getSourceMetadata().getLastModified(); if (linkLastModified != null && lastModified > linkLastModified) { - // Delete the link file (from the cache). + // Outdated link file. + invalidCacheFile = true; + } + } catch (Exception e) { + // Can not resolve the file or other errors. + invalidCacheFile = true; + } + + if (invalidCacheFile) { + log.debug("Link file {} is out of date and will be re-created.", cacheFile); + try { Files.delete(Paths.get(cacheFile)); cacheFile = null; + } catch (IOException ioException) { + // Ignore } - } catch (IOException e) { - // Ignore } } return cacheFile; diff --git a/gortools/src/main/scala/gorsat/Commands/Write.scala b/gortools/src/main/scala/gorsat/Commands/Write.scala index 0e27ee96..8ed758cf 100644 --- a/gortools/src/main/scala/gorsat/Commands/Write.scala +++ b/gortools/src/main/scala/gorsat/Commands/Write.scala @@ -51,7 +51,7 @@ class Write extends CommandInfo("WRITE", fileName = if (fileName.isEmpty && linkOpt.nonEmpty) { val linkMetaInfo = LinkFileUtil.extractLinkMetaInfo(linkMetaOpt) - val linkSourceRef = new SourceReference(linkOpt, + val linkSourceRef = new SourceReference(DataUtil.toLink(linkOpt), context.getSession.getProjectContext.getFileReader.getSecurityContext, context.getSession.getProjectContext.getFileReader.getCommonRoot, null, null, true); // Infer the full file name from the link (and defautl locations) diff --git a/gortools/src/test/java/gorsat/UTestGorWrite.java b/gortools/src/test/java/gorsat/UTestGorWrite.java index f0be3484..fea6684a 100644 --- a/gortools/src/test/java/gorsat/UTestGorWrite.java +++ b/gortools/src/test/java/gorsat/UTestGorWrite.java @@ -311,7 +311,7 @@ public void testWriteLinkFileForGordFolderInferFilename() throws IOException { var linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gord.link").toString())); Assert.assertEquals(1, linkFile.getEntriesCount()); - Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3\\..*?\\.gord/")); + Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3_.*?\\.gord/")); String linkresult1 = TestUtils.runGorPipe("gor dbsnp.gor| top 1000", "-gorroot", workDirPath.toString()); String linkresult3 = TestUtils.runGorPipe("gor dbsnp3.gord | top 1000", "-gorroot", workDirPath.toString()); @@ -327,7 +327,7 @@ public void testWriteLinkFileForGordFolderInferFilenameParallel() throws IOExcep var linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gord.link").toString())); Assert.assertEquals(1, linkFile.getEntriesCount()); - Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3\\..*?\\.gord/")); + Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3_.*?\\.gord/")); String linkresult1 = TestUtils.runGorPipe("gor dbsnp.gor | top 500", "-gorroot", workDirPath.toString()); String linkresult3 = TestUtils.runGorPipe("gor dbsnp3.gord | top 500", "-gorroot", workDirPath.toString()); diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java index fa79dbd5..ef96d488 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java @@ -123,6 +123,10 @@ public LinkFileMeta getMeta() { return meta; } + public int getSerial() { + return meta.getPropertyInt(LinkFileMeta.HEADER_SERIAL_KEY, 0); + } + public String getPath() { return source.getFullPath(); } diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java index f8fdb3ea..39c0b613 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java @@ -8,6 +8,7 @@ import org.gorpipe.gor.model.FileReader; import org.gorpipe.gor.table.util.PathUtils; import org.gorpipe.util.Strings; +import org.slf4j.Logger; import java.io.IOException; import java.util.regex.Matcher; @@ -15,9 +16,15 @@ public class LinkFileUtil { + private static Logger log = org.slf4j.LoggerFactory.getLogger(LinkFileUtil.class); + /** * Infer the data file name from the link file name. * + * Notes: The path returned must be idempotent as this is called + * from multiple different places in the code (meaning we + * can not use random or time in the path). + * * @param linkSource the link file path with the link extension * @param linkFileMeta additional link file meta data * @return the data file path @@ -38,7 +45,7 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri if (!Strings.isNullOrEmpty(linkDataFileParentPath)) { dataFileParentPath = linkDataFileParentPath; } else if (link.getLatestEntry() != null) { - dataFileParentPath = PathUtils.getParent(link.getLatestEntryUrl()); + dataFileParentPath = PathUtils.getParent(PathUtils.getParent(link.getLatestEntryUrl())); } if (!Strings.isNullOrEmpty(linkDataFileParentPath)) { @@ -59,9 +66,14 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri } } - var dataFileName = PathUtils.injectRandomStringIntoFileName(PathUtils.getFileName(linkSource.getFullPath())); + var fileName = PathUtils.getFileName(linkSource.getFullPath()); + var extraFolder = PathUtils.removeExtensions(fileName); + var uniqueFileName = PathUtils.injectStringIntoFileName(fileName, Integer.toString(link.getSerial() + 1)); + + log.warn("Inferred file name for link file {} is {}", linkSource.getFullPath(), + PathUtils.resolve(PathUtils.resolve(dataFileParentPath, extraFolder), uniqueFileName)); - return PathUtils.resolve(dataFileParentPath, dataFileName); + return PathUtils.resolve(PathUtils.resolve(dataFileParentPath, extraFolder), uniqueFileName); } private static Pattern linkPattern = Pattern.compile(".* -link ([^\\s]*) ?.*", Pattern.CASE_INSENSITIVE); diff --git a/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java b/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java index daf9ebea..fc625756 100644 --- a/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java +++ b/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java @@ -348,16 +348,19 @@ public static Path getTempFilePath(Path filePath) { } public static String injectRandomStringIntoFileName(String fileName) { + return injectStringIntoFileName(fileName, RandomStringUtils.insecure().next(8, true, true)); + } + + public static String injectStringIntoFileName(String fileName, String injectString) { var tempFileName = ""; - String uniqId = RandomStringUtils.insecure().next(8, true, true); var linkPathSplit = fileName.indexOf('.', fileName.indexOf("/")); if (linkPathSplit > 0) { - tempFileName = "%s.%s.%s".formatted( + tempFileName = "%s_%s.%s".formatted( fileName.substring(0, linkPathSplit), - uniqId, + injectString, fileName.substring(linkPathSplit + 1)); } else { - tempFileName = "%s.%s".formatted(fileName, uniqId); + tempFileName = "%s_%s".formatted(fileName, injectString); } return tempFileName.replaceAll("\\.link$", ""); diff --git a/model/src/main/java/org/gorpipe/gor/util/DataUtil.java b/model/src/main/java/org/gorpipe/gor/util/DataUtil.java index abbffb24..4d2211a0 100644 --- a/model/src/main/java/org/gorpipe/gor/util/DataUtil.java +++ b/model/src/main/java/org/gorpipe/gor/util/DataUtil.java @@ -126,7 +126,7 @@ public static String toLinkFile(String name, DataType type) { } public static String toLink(String path) { - return PathUtils.stripTrailingSlash(path) + DataType.LINK.suffix; + return DataUtil.isLink(path) ? path : PathUtils.stripTrailingSlash(path) + DataType.LINK.suffix; } public static String toVersionedLink(String path) { diff --git a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java index dd2191d1..a8f447d4 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java +++ b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java @@ -201,7 +201,7 @@ public void testInferDataFileNameFromLinkFile_FromEnvVariable_WithProject() thro String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference("x.gor.link", null, "/projects/test", -1, null, null, false, false)), null); assertNotNull(result); - assertTrue(result.matches((root + "/test/x\\..*\\.gor").replace("/", "\\/"))); + assertTrue(result.matches((root + "/test/x/x_.*\\.gor").replace("/", "\\/"))); } @Test @@ -210,7 +210,7 @@ public void testInferDataFileNameFromLinkFile_FromEnvVariable_WithOutProject() t environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_ROOT_URL, root); String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource("x.gor.link"), null); - assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/"))); + assertTrue(result.matches((root + "/x/x_.*\\.gor").replace("/", "\\/"))); } @Test @@ -218,11 +218,11 @@ public void testInferDataFileNameFromLinkFile_FromExiting_File() throws Exceptio String root = "/managed/fromfile"; String linkFilePath = "x.gor.link"; Files.createDirectory(workPath.resolve("test")); - Files.writeString(workPath.resolve("test").resolve(linkFilePath), root + "/source/y.gorz\n"); + Files.writeString(workPath.resolve("test").resolve(linkFilePath), root + "/source/y/y.gorz\n"); String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath, null, workPath.resolve("test").toString(), -1, null, null, false, false)), null); assertNotNull(result); - assertTrue(result.matches((root + "/source/x\\..*\\.gor").replace("/", "\\/"))); + assertTrue(result.matches((root + "/source/x/x_.*\\.gor").replace("/", "\\/"))); } @Test @@ -233,7 +233,7 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam() throws Exception { String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath)), linkFileMeta); assertNotNull(result); - assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/"))); + assertTrue(result.matches((root + "/x/x_..*\\.gor").replace("/", "\\/"))); } @Test @@ -247,7 +247,7 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam_ExistingFile() throw String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath)), linkFileMeta); assertNotNull(result); - assertTrue(result.matches((paramroot + "/x\\..*\\.gor").replace("/", "\\/"))); + assertTrue(result.matches((paramroot + "/x/x_.*\\.gor").replace("/", "\\/"))); } @Test @@ -258,6 +258,6 @@ public void testInferDataFileNameFromLinkFile_AbsolutePath() throws Exception { String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource("/abs/path/x.gor.link"), null); assertNotNull(result); - assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/"))); + assertTrue(result.matches((root + "/x/x_.*\\.gor").replace("/", "\\/"))); } }