From e3ab1de7656f589d9f71d75153ac650149a34ed0 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Thu, 28 Mar 2024 15:09:50 -0700 Subject: [PATCH 1/2] [add] yaml parsing and tests. --- go.mod | 2 + go.sum | 3 ++ internal/parsers/markdown.go | 9 ++-- internal/parsers/markdown_test.go | 81 ++++++++++++++++++++++++++----- 4 files changed, 81 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 7bdb5c28..4a3b0cc0 100644 --- a/go.mod +++ b/go.mod @@ -42,10 +42,12 @@ require ( github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect + github.com/yuin/goldmark-meta v1.1.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 7655a0cf..b9797804 100644 --- a/go.sum +++ b/go.sum @@ -94,6 +94,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= +github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc= +github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -123,6 +125,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/parsers/markdown.go b/internal/parsers/markdown.go index ecc576cc..e6f37b37 100644 --- a/internal/parsers/markdown.go +++ b/internal/parsers/markdown.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/InnovationEngine/internal/logging" "github.com/yuin/goldmark" + meta "github.com/yuin/goldmark-meta" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" @@ -17,7 +18,7 @@ import ( ) var markdownParser = goldmark.New( - goldmark.WithExtensions(extension.GFM), + goldmark.WithExtensions(extension.GFM, meta.New(meta.WithStoresInDocument())), goldmark.WithParserOptions( parser.WithAutoHeadingID(), parser.WithBlockParsers(), @@ -33,7 +34,10 @@ func ParseMarkdownIntoAst(source []byte) ast.Node { return document } -// The representation of an expected output block in a markdown file. This is +func ExtractYamlMetadataFromAst(node ast.Node) map[string]interface{} { + return node.OwnerDocument().Meta() +} + // for scenarios that have expected output that should be validated against the // actual output. type ExpectedOutputBlock struct { @@ -237,7 +241,6 @@ func convertScenarioVariablesToMap(variableBlock string) map[string]string { variableMap[key] = value } } - } return variableMap diff --git a/internal/parsers/markdown_test.go b/internal/parsers/markdown_test.go index bb795a32..0b450ff6 100644 --- a/internal/parsers/markdown_test.go +++ b/internal/parsers/markdown_test.go @@ -10,7 +10,6 @@ func TestParsingMarkdownHeaders(t *testing.T) { markdown := []byte(`# Hello World`) document := ParseMarkdownIntoAst(markdown) title, err := ExtractScenarioTitleFromAst(document, markdown) - if err != nil { t.Errorf("Error parsing title: %s", err) } @@ -24,7 +23,6 @@ func TestParsingMarkdownHeaders(t *testing.T) { markdown := []byte("# Hello World \n # Hello again") document := ParseMarkdownIntoAst(markdown) title, err := ExtractScenarioTitleFromAst(document, markdown) - if err != nil { t.Errorf("Error parsing title: %s", err) } @@ -50,8 +48,60 @@ func TestParsingMarkdownHeaders(t *testing.T) { }) } -func TestParsingMarkdownCodeBlocks(t *testing.T) { +func TestParsingYamlMetadata(t *testing.T) { + t.Run("Markdown with valid yaml metadata", func(t *testing.T) { + markdown := []byte(`--- + key: value + array: [1, 2, 3] + --- + `) + + document := ParseMarkdownIntoAst(markdown) + metadata := ExtractYamlMetadataFromAst(document) + + if metadata["key"] != "value" { + t.Errorf("Metadata is wrong: %v", metadata) + } + + array := metadata["array"].([]interface{}) + if array[0] != 1 || array[1] != 2 || array[2] != 3 { + t.Errorf("Metadata is wrong: %v", metadata) + } + }) + + t.Run("Markdown without yaml metadata", func(t *testing.T) { + markdown := []byte(`# Hello World.`) + document := ParseMarkdownIntoAst(markdown) + metadata := ExtractYamlMetadataFromAst(document) + + if len(metadata) != 0 { + t.Errorf("Metadata should be empty") + } + }) + + t.Run("yaml with nested properties", func(t *testing.T) { + markdown := []byte(`--- + nested: + key: value + key.value: otherValue + --- + `) + document := ParseMarkdownIntoAst(markdown) + metadata := ExtractYamlMetadataFromAst(document) + + nested := metadata["nested"].(map[interface{}]interface{}) + if nested["key"] != "value" { + t.Errorf("Metadata is wrong: %v", metadata) + } + + if metadata["key.value"] != "otherValue" { + t.Errorf("Metadata is wrong: %v", metadata) + } + }) +} + +func TestParsingMarkdownCodeBlocks(t *testing.T) { t.Run("Markdown with a valid bash code block", func(t *testing.T) { markdown := []byte(fmt.Sprintf("# Hello World\n ```bash\n%s\n```", "echo Hello")) @@ -74,13 +124,16 @@ func TestParsingMarkdownCodeBlocks(t *testing.T) { ) } }) - } func TestParsingMarkdownExpectedSimilarty(t *testing.T) { - t.Run("Markdown with a expected_similarty tag using float", func(t *testing.T) { - markdown := []byte(fmt.Sprintf("```bash\n%s\n```\n\n```\nHello\n```\n", "echo Hello")) + markdown := []byte( + fmt.Sprintf( + "```bash\n%s\n```\n\n```\nHello\n```\n", + "echo Hello", + ), + ) document := ParseMarkdownIntoAst(markdown) codeBlocks := ExtractCodeBlocksFromAst(document, markdown, []string{"bash"}) @@ -92,16 +145,23 @@ func TestParsingMarkdownExpectedSimilarty(t *testing.T) { block := codeBlocks[0].ExpectedOutput expectedFloat := .8 if block.ExpectedSimilarity != expectedFloat { - t.Errorf("ExpectedSimilarity is wrong, got %f, expected %f", block.ExpectedSimilarity, expectedFloat) + t.Errorf( + "ExpectedSimilarity is wrong, got %f, expected %f", + block.ExpectedSimilarity, + expectedFloat, + ) } }) - } func TestParsingMarkdownExpectedRegex(t *testing.T) { - t.Run("Markdown with a expected_similarty tag using regex", func(t *testing.T) { - markdown := []byte(fmt.Sprintf("```bash\n%s\n```\n\n```\nFoo Bar\n```\n", "echo 'Foo Bar'")) + markdown := []byte( + fmt.Sprintf( + "```bash\n%s\n```\n\n```\nFoo Bar\n```\n", + "echo 'Foo Bar'", + ), + ) document := ParseMarkdownIntoAst(markdown) codeBlocks := ExtractCodeBlocksFromAst(document, markdown, []string{"bash"}) @@ -121,5 +181,4 @@ func TestParsingMarkdownExpectedRegex(t *testing.T) { t.Errorf("ExpectedRegex is wrong, got %q, expected %q", stringRegex, expectedRegex) } }) - } From 579bdb5cc9fb197c12b21d5c85214eb35fe5cd85 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Thu, 28 Mar 2024 16:11:24 -0700 Subject: [PATCH 2/2] [add] comment back. --- internal/parsers/markdown.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/parsers/markdown.go b/internal/parsers/markdown.go index e6f37b37..f777a29b 100644 --- a/internal/parsers/markdown.go +++ b/internal/parsers/markdown.go @@ -38,6 +38,7 @@ func ExtractYamlMetadataFromAst(node ast.Node) map[string]interface{} { return node.OwnerDocument().Meta() } +// The representation of an expected output block in a markdown file. This is // for scenarios that have expected output that should be validated against the // actual output. type ExpectedOutputBlock struct {