fix(ENGKOW-2781): gor support versioning of dictionaries

gortools/src/main/java/gorsat/process/GorJavaUtilities.java

@@ -432,17 +432,28 @@ public static void createSymbolicLinkSafe(Path resultPath, Path cachePath) throw
      */
     public static String verifyLinkFileLastModified(ProjectContext projectContext, String cacheFile) {
         if (cacheFile != null && DataUtil.isLink(cacheFile)) {
+            var invalidCacheFile = false;
             try {
                 var ds = projectContext.getFileReader().resolveUrl(cacheFile);
                 var linkLastModified = ds.getSourceMetadata().getLinkLastModified();
                 var lastModified = ds.getSourceMetadata().getLastModified();
                 if (linkLastModified != null && lastModified > linkLastModified) {
-                    // Delete the link file (from the cache).
+                    // Outdated link file.
+                    invalidCacheFile = true;
+                }
+            } catch (Exception e) {
+                // Can not resolve the file or other errors.
+                invalidCacheFile = true;
+            }
+
+            if (invalidCacheFile) {
+                log.debug("Link file {} is out of date and will be re-created.", cacheFile);
+                try {
                     Files.delete(Paths.get(cacheFile));
                     cacheFile = null;
+                } catch (IOException ioException) {
+                    // Ignore
                 }
-            } catch (IOException e) {
-                // Ignore
             }
         }
         return cacheFile;

gortools/src/main/scala/gorsat/Commands/Write.scala

@@ -51,7 +51,7 @@ class Write extends CommandInfo("WRITE",
 
     fileName = if (fileName.isEmpty && linkOpt.nonEmpty) {
       val linkMetaInfo = LinkFileUtil.extractLinkMetaInfo(linkMetaOpt)
-      val linkSourceRef = new SourceReference(linkOpt,
+      val linkSourceRef = new SourceReference(DataUtil.toLink(linkOpt),
         context.getSession.getProjectContext.getFileReader.getSecurityContext,
         context.getSession.getProjectContext.getFileReader.getCommonRoot, null, null, true);
       // Infer the full file name from the link (and defautl locations)

gortools/src/test/java/gorsat/UTestGorWrite.java

@@ -311,7 +311,7 @@ public void testWriteLinkFileForGordFolderInferFilename() throws IOException {
         var linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gord.link").toString()));
 
         Assert.assertEquals(1, linkFile.getEntriesCount());
-        Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3\\..*?\\.gord/"));
+        Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3_.*?\\.gord/"));
 
         String linkresult1 = TestUtils.runGorPipe("gor dbsnp.gor| top 1000", "-gorroot", workDirPath.toString());
         String linkresult3 = TestUtils.runGorPipe("gor dbsnp3.gord | top 1000", "-gorroot", workDirPath.toString());
@@ -327,7 +327,7 @@ public void testWriteLinkFileForGordFolderInferFilenameParallel() throws IOExcep
         var linkFile = LinkFile.load(new FileSource(workDirPath.resolve("dbsnp3.gord.link").toString()));
 
         Assert.assertEquals(1, linkFile.getEntriesCount());
-        Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3\\..*?\\.gord/"));
+        Assert.assertTrue(linkFile.getLatestEntry().url().matches(".*?dbsnp3_.*?\\.gord/"));
 
         String linkresult1 = TestUtils.runGorPipe("gor dbsnp.gor | top 500", "-gorroot", workDirPath.toString());
         String linkresult3 = TestUtils.runGorPipe("gor dbsnp3.gord | top 500", "-gorroot", workDirPath.toString());

model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java

@@ -123,6 +123,10 @@ public LinkFileMeta getMeta() {
         return meta;
     }
 
+    public int getSerial() {
+        return meta.getPropertyInt(LinkFileMeta.HEADER_SERIAL_KEY, 0);
+    }
+
     public String getPath() {
         return source.getFullPath();
     }

model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileUtil.java

@@ -8,16 +8,23 @@
 import org.gorpipe.gor.model.FileReader;
 import org.gorpipe.gor.table.util.PathUtils;
 import org.gorpipe.util.Strings;
+import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public class LinkFileUtil {
 
+    private static Logger log = org.slf4j.LoggerFactory.getLogger(LinkFileUtil.class);
+
     /**
      * Infer the data file name from the link file name.
      *
+     * Notes: The path returned must be idempotent as this is called
+     *        from multiple different places in the code (meaning we
+     *        can not use random or time in the path).
+     *
      * @param linkSource the link file path with the link extension
      * @param linkFileMeta additional link file meta data
      * @return the data file path
@@ -38,7 +45,7 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
         if (!Strings.isNullOrEmpty(linkDataFileParentPath)) {
             dataFileParentPath = linkDataFileParentPath;
         } else if (link.getLatestEntry() != null) {
-            dataFileParentPath = PathUtils.getParent(link.getLatestEntryUrl());
+            dataFileParentPath = PathUtils.getParent(PathUtils.getParent(link.getLatestEntryUrl()));
         }
 
         if (!Strings.isNullOrEmpty(linkDataFileParentPath)) {
@@ -59,9 +66,14 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
             }
         }
 
-        var dataFileName = PathUtils.injectRandomStringIntoFileName(PathUtils.getFileName(linkSource.getFullPath()));
+        var fileName = PathUtils.getFileName(linkSource.getFullPath());
+        var extraFolder = PathUtils.removeExtensions(fileName);
+        var uniqueFileName = PathUtils.injectStringIntoFileName(fileName, Integer.toString(link.getSerial() + 1));
+
+        log.warn("Inferred file name for link file {} is {}", linkSource.getFullPath(),
+                PathUtils.resolve(PathUtils.resolve(dataFileParentPath, extraFolder), uniqueFileName));
 
-        return PathUtils.resolve(dataFileParentPath, dataFileName);
+        return PathUtils.resolve(PathUtils.resolve(dataFileParentPath, extraFolder), uniqueFileName);
     }
 
     private static Pattern linkPattern = Pattern.compile(".* -link ([^\\s]*) ?.*", Pattern.CASE_INSENSITIVE);

model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java

@@ -348,16 +348,19 @@ public static Path getTempFilePath(Path filePath) {
     }
 
     public static String injectRandomStringIntoFileName(String fileName) {
+        return injectStringIntoFileName(fileName, RandomStringUtils.insecure().next(8, true, true));
+    }
+
+    public static String injectStringIntoFileName(String fileName, String injectString) {
         var tempFileName = "";
-        String uniqId = RandomStringUtils.insecure().next(8, true, true);
         var linkPathSplit = fileName.indexOf('.', fileName.indexOf("/"));
         if (linkPathSplit > 0) {
-            tempFileName = "%s.%s.%s".formatted(
+            tempFileName = "%s_%s.%s".formatted(
                     fileName.substring(0, linkPathSplit),
-                    uniqId,
+                    injectString,
                     fileName.substring(linkPathSplit + 1));
         } else {
-            tempFileName = "%s.%s".formatted(fileName, uniqId);
+            tempFileName = "%s_%s".formatted(fileName, injectString);
         }
 
         return tempFileName.replaceAll("\\.link$", "");

model/src/main/java/org/gorpipe/gor/util/DataUtil.java

@@ -126,7 +126,7 @@ public static String toLinkFile(String name, DataType type) {
     }
 
     public static String toLink(String path) {
-        return PathUtils.stripTrailingSlash(path) + DataType.LINK.suffix;
+        return DataUtil.isLink(path) ? path : PathUtils.stripTrailingSlash(path) + DataType.LINK.suffix;
     }
 
     public static String toVersionedLink(String path) {

model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java

@@ -201,7 +201,7 @@ public void testInferDataFileNameFromLinkFile_FromEnvVariable_WithProject() thro
 
         String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference("x.gor.link", null, "/projects/test", -1, null,  null, false, false)), null);
         assertNotNull(result);
-        assertTrue(result.matches((root + "/test/x\\..*\\.gor").replace("/", "\\/")));
+        assertTrue(result.matches((root + "/test/x/x_.*\\.gor").replace("/", "\\/")));
     }
 
     @Test
@@ -210,19 +210,19 @@ public void testInferDataFileNameFromLinkFile_FromEnvVariable_WithOutProject() t
         environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_ROOT_URL, root);
 
         String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource("x.gor.link"), null);
-        assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/")));
+        assertTrue(result.matches((root + "/x/x_.*\\.gor").replace("/", "\\/")));
     }
 
     @Test
     public void testInferDataFileNameFromLinkFile_FromExiting_File() throws Exception {
         String root = "/managed/fromfile";
         String linkFilePath =  "x.gor.link";
         Files.createDirectory(workPath.resolve("test"));
-        Files.writeString(workPath.resolve("test").resolve(linkFilePath), root + "/source/y.gorz\n");
+        Files.writeString(workPath.resolve("test").resolve(linkFilePath), root + "/source/y/y.gorz\n");
 
         String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath, null, workPath.resolve("test").toString(), -1, null,  null, false, false)), null);
         assertNotNull(result);
-        assertTrue(result.matches((root + "/source/x\\..*\\.gor").replace("/", "\\/")));
+        assertTrue(result.matches((root + "/source/x/x_.*\\.gor").replace("/", "\\/")));
     }
 
     @Test
@@ -233,7 +233,7 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam() throws Exception {
 
         String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath)), linkFileMeta);
         assertNotNull(result);
-        assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/")));
+        assertTrue(result.matches((root + "/x/x_..*\\.gor").replace("/", "\\/")));
     }
 
     @Test
@@ -247,7 +247,7 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam_ExistingFile() throw
 
         String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath)), linkFileMeta);
         assertNotNull(result);
-        assertTrue(result.matches((paramroot + "/x\\..*\\.gor").replace("/", "\\/")));
+        assertTrue(result.matches((paramroot + "/x/x_.*\\.gor").replace("/", "\\/")));
     }
 
     @Test
@@ -258,6 +258,6 @@ public void testInferDataFileNameFromLinkFile_AbsolutePath() throws Exception {
         String result = LinkFileUtil.inferDataFileNameFromLinkFile(new FileSource("/abs/path/x.gor.link"), null);
 
         assertNotNull(result);
-        assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/")));
+        assertTrue(result.matches((root + "/x/x_.*\\.gor").replace("/", "\\/")));
     }
 }