From 4785decaba4bc38d8666353464af23ee74e1bf21 Mon Sep 17 00:00:00 2001 From: Robin Stocker Date: Wed, 1 Jun 2022 14:43:15 +1000 Subject: [PATCH] YAML front matter: Limited support for single/double quoted strings Extend our manual parser to handle string values that use single or double quotes. The support is limited and doesn't implement the full YAML spec (e.g. no support for escapes like `\n`). At some point we should either depend on a real YAML parser or expose the raw YAML source so that users can parse it themselves. Fixes #260. --- CHANGELOG.md | 5 + .../internal/YamlFrontMatterBlockParser.java | 28 ++++- .../ext/front/matter/YamlFrontMatterTest.java | 115 ++++++++---------- 3 files changed, 83 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc3b7a332..9761edb2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html), with the exception that 0.x versions can break between minor versions. +## Unreleased +### Added +- YAML front matter extension: Limited support for single and double + quoted string values (#260) + ## [0.18.2] - 2022-02-24 ### Changed - Test against Java 17 diff --git a/commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java b/commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java index e9d0901b6..2010b4f71 100644 --- a/commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java +++ b/commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java @@ -61,10 +61,11 @@ public BlockContinue tryContinue(ParserState parserState) { inLiteral = false; currentKey = matcher.group(1); currentValues = new ArrayList<>(); - if ("|".equals(matcher.group(2))) { + String value = matcher.group(2); + if ("|".equals(value)) { inLiteral = true; - } else if (!"".equals(matcher.group(2))) { - currentValues.add(matcher.group(2)); + } else if (!"".equals(value)) { + currentValues.add(parseString(value)); } return BlockContinue.atIndex(parserState.getIndex()); @@ -81,7 +82,8 @@ public BlockContinue tryContinue(ParserState parserState) { } else { matcher = REGEX_METADATA_LIST.matcher(line); if (matcher.matches()) { - currentValues.add(matcher.group(1)); + String value = matcher.group(1); + currentValues.add(parseString(value)); } } @@ -93,6 +95,24 @@ public BlockContinue tryContinue(ParserState parserState) { public void parseInlines(InlineParser inlineParser) { } + private static String parseString(String s) { + // Limited parsing of https://yaml.org/spec/1.2.2/#73-flow-scalar-styles + // We assume input is well-formed and otherwise treat it as a plain string. In a real + // parser, e.g. `'foo` would be invalid because it's missing a trailing `'`. + if (s.startsWith("'") && s.endsWith("'")) { + String inner = s.substring(1, s.length() - 1); + return inner.replace("''", "'"); + } else if (s.startsWith("\"") && s.endsWith("\"")) { + String inner = s.substring(1, s.length() - 1); + // Only support escaped `\` and `"`, nothing else. + return inner + .replace("\\\"", "\"") + .replace("\\\\", "\\"); + } else { + return s; + } + } + public static class Factory extends AbstractBlockParserFactory { @Override public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { diff --git a/commonmark-ext-yaml-front-matter/src/test/java/org/commonmark/ext/front/matter/YamlFrontMatterTest.java b/commonmark-ext-yaml-front-matter/src/test/java/org/commonmark/ext/front/matter/YamlFrontMatterTest.java index 251b1c15f..a5f203b97 100644 --- a/commonmark-ext-yaml-front-matter/src/test/java/org/commonmark/ext/front/matter/YamlFrontMatterTest.java +++ b/commonmark-ext-yaml-front-matter/src/test/java/org/commonmark/ext/front/matter/YamlFrontMatterTest.java @@ -30,11 +30,7 @@ public void simpleValue() { "\ngreat"; final String rendered = "

great

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertEquals(1, data.size()); assertEquals("hello", data.keySet().iterator().next()); @@ -53,11 +49,7 @@ public void emptyValue() { "\ngreat"; final String rendered = "

great

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertEquals(1, data.size()); assertEquals("key", data.keySet().iterator().next()); @@ -77,11 +69,7 @@ public void listValues() { "\ngreat"; final String rendered = "

great

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertEquals(1, data.size()); assertTrue(data.containsKey("list")); @@ -103,11 +91,7 @@ public void literalValue1() { "\ngreat"; final String rendered = "

great

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertEquals(1, data.size()); assertTrue(data.containsKey("literal")); @@ -127,11 +111,7 @@ public void literalValue2() { "\ngreat"; final String rendered = "

great

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertEquals(1, data.size()); assertTrue(data.containsKey("literal")); @@ -156,11 +136,7 @@ public void complexValues() { "\ngreat"; final String rendered = "

great

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertEquals(3, data.size()); @@ -187,11 +163,7 @@ public void empty() { "test"; final String rendered = "

test

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertTrue(data.isEmpty()); @@ -207,11 +179,7 @@ public void yamlInParagraph() { "\n---"; final String rendered = "

hello

\n

hello markdown world!

\n

hello: world

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertTrue(data.isEmpty()); @@ -226,11 +194,7 @@ public void yamlOnSecondLine() { "\n---"; final String rendered = "

hello

\n
\n

hello: world

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertTrue(data.isEmpty()); @@ -243,11 +207,7 @@ public void nonMatchedStartTag() { "test"; final String rendered = "
\n

test

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertTrue(data.isEmpty()); @@ -261,11 +221,7 @@ public void inList() { "test"; final String rendered = "
    \n
  • \n
    \n
    \n
    • \n
\n

test

\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertTrue(data.isEmpty()); @@ -310,7 +266,7 @@ public void nodesCanBeModified() { assertTrue(data.containsKey("see")); assertEquals(Collections.singletonList("you"), data.get("see")); } - + @Test public void dotInKeys() { final String input = "---" + @@ -318,11 +274,7 @@ public void dotInKeys() { "\n---" + "\n"; - YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); - Node document = PARSER.parse(input); - document.accept(visitor); - - Map> data = visitor.getData(); + Map> data = getFrontMatter(input); assertEquals(1, data.size()); assertEquals("ms.author", data.keySet().iterator().next()); @@ -330,11 +282,52 @@ public void dotInKeys() { assertEquals("author", data.get("ms.author").get(0)); } + @Test + public void singleQuotedLiterals() { + final String input = "---" + + "\nstring: 'It''s me'" + + "\nlist:" + + "\n - 'I''m here'" + + "\n---" + + "\n"; + + Map> data = getFrontMatter(input); + + assertEquals(2, data.size()); + assertEquals("It's me", data.get("string").get(0)); + assertEquals("I'm here", data.get("list").get(0)); + } + + @Test + public void doubleQuotedLiteral() { + final String input = "---" + + "\nstring: \"backslash: \\\\ quote: \\\"\"" + + "\nlist:" + + "\n - \"hey\"" + + "\n---" + + "\n"; + + Map> data = getFrontMatter(input); + + assertEquals(2, data.size()); + assertEquals("backslash: \\ quote: \"", data.get("string").get(0)); + assertEquals("hey", data.get("list").get(0)); + } + @Override protected String render(String source) { return RENDERER.render(PARSER.parse(source)); } + private Map> getFrontMatter(String input) { + YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); + Node document = PARSER.parse(input); + document.accept(visitor); + + Map> data = visitor.getData(); + return data; + } + // Custom node for tests private static class TestNode extends CustomNode { }