YAML front matter: Limited support for single/double quoted strings

Extend our manual parser to handle string values that use single or
double quotes. The support is limited and doesn't implement the full
YAML spec (e.g. no support for escapes like `\n`).

At some point we should either depend on a real YAML parser or expose
the raw YAML source so that users can parse it themselves.

Fixes #260.

Author: robinst

CHANGELOG.md

@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html),
 with the exception that 0.x versions can break between minor versions.
 
+## Unreleased
+### Added
+- YAML front matter extension: Limited support for single and double
+  quoted string values (#260)
+
 ## [0.18.2] - 2022-02-24
 ### Changed
 - Test against Java 17

commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java

@@ -61,10 +61,11 @@ public BlockContinue tryContinue(ParserState parserState) {
             inLiteral = false;
             currentKey = matcher.group(1);
             currentValues = new ArrayList<>();
-            if ("|".equals(matcher.group(2))) {
+            String value = matcher.group(2);
+            if ("|".equals(value)) {
                 inLiteral = true;
-            } else if (!"".equals(matcher.group(2))) {
-                currentValues.add(matcher.group(2));
+            } else if (!"".equals(value)) {
+                currentValues.add(parseString(value));
             }
 
             return BlockContinue.atIndex(parserState.getIndex());
@@ -81,7 +82,8 @@ public BlockContinue tryContinue(ParserState parserState) {
             } else {
                 matcher = REGEX_METADATA_LIST.matcher(line);
                 if (matcher.matches()) {
-                    currentValues.add(matcher.group(1));
+                    String value = matcher.group(1);
+                    currentValues.add(parseString(value));
                 }
             }
 
@@ -93,6 +95,24 @@ public BlockContinue tryContinue(ParserState parserState) {
     public void parseInlines(InlineParser inlineParser) {
     }
 
+    private static String parseString(String s) {
+        // Limited parsing of https://yaml.org/spec/1.2.2/#73-flow-scalar-styles
+        // We assume input is well-formed and otherwise treat it as a plain string. In a real
+        // parser, e.g. `'foo` would be invalid because it's missing a trailing `'`.
+        if (s.startsWith("'") && s.endsWith("'")) {
+            String inner = s.substring(1, s.length() - 1);
+            return inner.replace("''", "'");
+        } else if (s.startsWith("\"") && s.endsWith("\"")) {
+            String inner = s.substring(1, s.length() - 1);
+            // Only support escaped `\` and `"`, nothing else.
+            return inner
+                    .replace("\\\"", "\"")
+                    .replace("\\\\", "\\");
+        } else {
+            return s;
+        }
+    }
+
     public static class Factory extends AbstractBlockParserFactory {
         @Override
         public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) {

commonmark-ext-yaml-front-matter/src/test/java/org/commonmark/ext/front/matter/YamlFrontMatterTest.java

@@ -30,11 +30,7 @@ public void simpleValue() {
                 "\ngreat";
         final String rendered = "<p>great</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertEquals(1, data.size());
         assertEquals("hello", data.keySet().iterator().next());
@@ -53,11 +49,7 @@ public void emptyValue() {
                 "\ngreat";
         final String rendered = "<p>great</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertEquals(1, data.size());
         assertEquals("key", data.keySet().iterator().next());
@@ -77,11 +69,7 @@ public void listValues() {
                 "\ngreat";
         final String rendered = "<p>great</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertEquals(1, data.size());
         assertTrue(data.containsKey("list"));
@@ -103,11 +91,7 @@ public void literalValue1() {
                 "\ngreat";
         final String rendered = "<p>great</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertEquals(1, data.size());
         assertTrue(data.containsKey("literal"));
@@ -127,11 +111,7 @@ public void literalValue2() {
                 "\ngreat";
         final String rendered = "<p>great</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertEquals(1, data.size());
         assertTrue(data.containsKey("literal"));
@@ -156,11 +136,7 @@ public void complexValues() {
                 "\ngreat";
         final String rendered = "<p>great</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertEquals(3, data.size());
 
@@ -187,11 +163,7 @@ public void empty() {
                 "test";
         final String rendered = "<p>test</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertTrue(data.isEmpty());
 
@@ -207,11 +179,7 @@ public void yamlInParagraph() {
                 "\n---";
         final String rendered = "<h1>hello</h1>\n<h2>hello markdown world!</h2>\n<h2>hello: world</h2>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertTrue(data.isEmpty());
 
@@ -226,11 +194,7 @@ public void yamlOnSecondLine() {
                 "\n---";
         final String rendered = "<p>hello</p>\n<hr />\n<h2>hello: world</h2>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertTrue(data.isEmpty());
 
@@ -243,11 +207,7 @@ public void nonMatchedStartTag() {
                 "test";
         final String rendered = "<hr />\n<p>test</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertTrue(data.isEmpty());
 
@@ -261,11 +221,7 @@ public void inList() {
                 "test";
         final String rendered = "<ul>\n<li>\n<hr />\n<hr />\n</li>\n</ul>\n<p>test</p>\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertTrue(data.isEmpty());
 
@@ -310,31 +266,68 @@ public void nodesCanBeModified() {
         assertTrue(data.containsKey("see"));
         assertEquals(Collections.singletonList("you"), data.get("see"));
     }
-    
+
     @Test
     public void dotInKeys() {
         final String input = "---" +
                 "\nms.author: author" +
                 "\n---" +
                 "\n";
 
-        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
-        Node document = PARSER.parse(input);
-        document.accept(visitor);
-
-        Map<String, List<String>> data = visitor.getData();
+        Map<String, List<String>> data = getFrontMatter(input);
 
         assertEquals(1, data.size());
         assertEquals("ms.author", data.keySet().iterator().next());
         assertEquals(1, data.get("ms.author").size());
         assertEquals("author", data.get("ms.author").get(0));
     }
 
+    @Test
+    public void singleQuotedLiterals() {
+        final String input = "---" +
+                "\nstring: 'It''s me'" +
+                "\nlist:" +
+                "\n  - 'I''m here'" +
+                "\n---" +
+                "\n";
+
+        Map<String, List<String>> data = getFrontMatter(input);
+
+        assertEquals(2, data.size());
+        assertEquals("It's me", data.get("string").get(0));
+        assertEquals("I'm here", data.get("list").get(0));
+    }
+
+    @Test
+    public void doubleQuotedLiteral() {
+        final String input = "---" +
+                "\nstring: \"backslash: \\\\ quote: \\\"\"" +
+                "\nlist:" +
+                "\n  - \"hey\"" +
+                "\n---" +
+                "\n";
+
+        Map<String, List<String>> data = getFrontMatter(input);
+
+        assertEquals(2, data.size());
+        assertEquals("backslash: \\ quote: \"", data.get("string").get(0));
+        assertEquals("hey", data.get("list").get(0));
+    }
+
     @Override
     protected String render(String source) {
         return RENDERER.render(PARSER.parse(source));
     }
 
+    private Map<String, List<String>> getFrontMatter(String input) {
+        YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
+        Node document = PARSER.parse(input);
+        document.accept(visitor);
+
+        Map<String, List<String>> data = visitor.getData();
+        return data;
+    }
+
     // Custom node for tests
     private static class TestNode extends CustomNode {
     }