From 0a6245abf011ba75312f3c548cf56d5cf63108fd Mon Sep 17 00:00:00 2001 From: vmarcella Date: Mon, 21 Oct 2024 13:15:50 -0700 Subject: [PATCH 1/6] [update] interactive mode to override environment variables with their values after a run for the portal. --- internal/engine/common/scenario.go | 11 ++++-- internal/engine/engine.go | 3 ++ internal/engine/environments/azure.go | 17 +++++---- internal/engine/interactive/interactive.go | 40 ++++++++++++++++++++++ internal/patterns/regex.go | 9 ++++- 5 files changed, 71 insertions(+), 9 deletions(-) diff --git a/internal/engine/common/scenario.go b/internal/engine/common/scenario.go index d51e3344..f0552df2 100644 --- a/internal/engine/common/scenario.go +++ b/internal/engine/common/scenario.go @@ -6,13 +6,13 @@ import ( "net/http" "os" "path/filepath" - "regexp" "strings" "github.com/Azure/InnovationEngine/internal/lib" "github.com/Azure/InnovationEngine/internal/lib/fs" "github.com/Azure/InnovationEngine/internal/logging" "github.com/Azure/InnovationEngine/internal/parsers" + "github.com/Azure/InnovationEngine/internal/patterns" "github.com/yuin/goldmark/ast" ) @@ -29,6 +29,12 @@ type Scenario struct { Steps []Step Properties map[string]interface{} Environment map[string]string + Source []byte +} + +// Get the markdown source for the scenario as a string. +func (s *Scenario) GetSourceAsString() string { + return string(s.Source) } // Groups the codeblocks into steps based on the header of the codeblock. @@ -132,7 +138,7 @@ func CreateScenarioFromMarkdown( for key, value := range environmentVariableOverrides { environmentVariables[key] = value logging.GlobalLogger.Debugf("Attempting to override %s with %s", key, value) - exportRegex := regexp.MustCompile(fmt.Sprintf(`(?m)export %s\s*=\s*(.*?)(;|&&|$)`, key)) + exportRegex := patterns.ExportVariableRegex(key) for index, codeBlock := range codeBlocks { matches := exportRegex.FindAllStringSubmatch(codeBlock.Content, -1) @@ -206,6 +212,7 @@ func CreateScenarioFromMarkdown( Steps: steps, Properties: properties, MarkdownAst: markdown, + Source: source, }, nil } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index fd531a79..a3c67a05 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -157,6 +157,7 @@ func (e *Engine) InteractWithScenario(scenario *common.Scenario) error { e.Configuration.Environment, stepsToExecute, lib.CopyMap(scenario.Environment), + scenario.GetSourceAsString(), ) if err != nil { return err @@ -181,9 +182,11 @@ func (e *Engine) InteractWithScenario(scenario *common.Scenario) error { switch e.Configuration.Environment { case environments.EnvironmentsAzure, environments.EnvironmentsOCD: + logging.GlobalLogger.Info( "Cleaning environment variable file located at /tmp/env-vars", ) + err := lib.CleanEnvironmentStateFile(lib.DefaultEnvironmentStateFile) if err != nil { logging.GlobalLogger.Errorf("Error cleaning environment variables: %s", err.Error()) diff --git a/internal/engine/environments/azure.go b/internal/engine/environments/azure.go index 15a50f16..5f875d6a 100644 --- a/internal/engine/environments/azure.go +++ b/internal/engine/environments/azure.go @@ -23,12 +23,13 @@ type AzureStep struct { // The status of a one-click deployment or learn mode deployment. type AzureDeploymentStatus struct { - Steps []AzureStep `json:"steps"` - CurrentStep int `json:"currentStep"` - Status string `json:"status"` - ResourceURIs []string `json:"resourceURIs"` - Error string `json:"error"` - Output string `json:"output"` + Steps []AzureStep `json:"steps"` + CurrentStep int `json:"currentStep"` + Status string `json:"status"` + ResourceURIs []string `json:"resourceURIs"` + Error string `json:"error"` + Output string `json:"output"` + ConfiguredScript string `json:"configuredScript"` } func NewAzureDeploymentStatus() AzureDeploymentStatus { @@ -72,6 +73,10 @@ func (status *AzureDeploymentStatus) SetOutput(output string) { status.Output = output } +func (status *AzureDeploymentStatus) SetConfiguredScript(script string) { + status.ConfiguredScript = script +} + // Print out the status JSON for azure/cloudshell if in the correct environment. func ReportAzureStatus(status AzureDeploymentStatus, environment string) { if !IsAzureEnvironment(environment) { diff --git a/internal/engine/interactive/interactive.go b/internal/engine/interactive/interactive.go index 83d67e70..45a1f858 100644 --- a/internal/engine/interactive/interactive.go +++ b/internal/engine/interactive/interactive.go @@ -60,6 +60,7 @@ type InteractiveModeModel struct { scenarioCompleted bool components interactiveModeComponents ready bool + markdownSource string CommandLines []string } @@ -344,6 +345,43 @@ func (model InteractiveModeModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { model.resourceGroupName, model.environment, ) + + environmentVariables, err := lib.LoadEnvironmentStateFile(lib.DefaultEnvironmentStateFile) + if err != nil { + logging.GlobalLogger.Errorf("Failed to load environment state file: %s", err) + model.azureStatus.SetError(err) + } + + configuredMarkdownSource := model.markdownSource + + for key, value := range environmentVariables { + exportRegex := patterns.ExportVariableRegex(key) + + matches := exportRegex.FindAllStringSubmatch(model.markdownSource, -1) + + if len(matches) != 0 { + logging.GlobalLogger.Debugf( + "Found %d matches for the environment variable %s, Replacing them in markdown source.", + len(matches), + key, + ) + } + + for _, match := range matches { + oldLine := match[0] + oldValue := match[1] + + // Replace the old export with the new export statement + newLine := strings.Replace(oldLine, oldValue, value+" ", 1) + logging.GlobalLogger.Debugf("Replacing '%s' with '%s'", oldLine, newLine) + + // Update the code block with the new export statement + configuredMarkdownSource = strings.Replace(configuredMarkdownSource, oldLine, newLine, 1) + } + } + + logging.GlobalLogger.Tracef("Configured markdown source: %s", configuredMarkdownSource) + model.azureStatus.SetConfiguredScript(configuredMarkdownSource) model.azureStatus.SetOutput(strings.Join(model.CommandLines, "\n")) commands = append( commands, @@ -570,6 +608,7 @@ func NewInteractiveModeModel( environment string, steps []common.Step, env map[string]string, + markdownSource string, ) (InteractiveModeModel, error) { // TODO: In the future we should just set the current step for the azure status // to one as the default. @@ -666,6 +705,7 @@ func NewInteractiveModeModel( environment: environment, scenarioCompleted: false, ready: false, + markdownSource: markdownSource, CommandLines: commandLines, }, nil } diff --git a/internal/patterns/regex.go b/internal/patterns/regex.go index a67b7241..bedf27a9 100644 --- a/internal/patterns/regex.go +++ b/internal/patterns/regex.go @@ -1,6 +1,9 @@ package patterns -import "regexp" +import ( + "fmt" + "regexp" +) var ( // An SSH command regex where there must be a username@host somewhere present in the command. @@ -18,4 +21,8 @@ var ( // ARM regex AzResourceURI = regexp.MustCompile(`\"id\": \"(/subscriptions/[^\"]+)\"`) AzResourceGroupName = regexp.MustCompile(`resourceGroups/([^\"\\/\ ]+)`) + + ExportVariableRegex = func(key string) *regexp.Regexp { + return regexp.MustCompile(fmt.Sprintf(`(?m)export %s\s*=\s*(.*?)(;|&&|$)`, key)) + } ) From 938aca1f52e3c777de00b5dfd674dcf0d427ceb0 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Mon, 21 Oct 2024 13:26:46 -0700 Subject: [PATCH 2/6] [update] name to configuredMarkdown --- internal/engine/environments/azure.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/engine/environments/azure.go b/internal/engine/environments/azure.go index 5f875d6a..db013bfd 100644 --- a/internal/engine/environments/azure.go +++ b/internal/engine/environments/azure.go @@ -23,13 +23,13 @@ type AzureStep struct { // The status of a one-click deployment or learn mode deployment. type AzureDeploymentStatus struct { - Steps []AzureStep `json:"steps"` - CurrentStep int `json:"currentStep"` - Status string `json:"status"` - ResourceURIs []string `json:"resourceURIs"` - Error string `json:"error"` - Output string `json:"output"` - ConfiguredScript string `json:"configuredScript"` + Steps []AzureStep `json:"steps"` + CurrentStep int `json:"currentStep"` + Status string `json:"status"` + ResourceURIs []string `json:"resourceURIs"` + Error string `json:"error"` + Output string `json:"output"` + ConfiguredMarkdown string `json:"configuredMarkdown"` } func NewAzureDeploymentStatus() AzureDeploymentStatus { @@ -74,7 +74,7 @@ func (status *AzureDeploymentStatus) SetOutput(output string) { } func (status *AzureDeploymentStatus) SetConfiguredScript(script string) { - status.ConfiguredScript = script + status.ConfiguredMarkdown = script } // Print out the status JSON for azure/cloudshell if in the correct environment. From 0d51a41132d26708776991f9a6a30606da4c226e Mon Sep 17 00:00:00 2001 From: vmarcella Date: Mon, 21 Oct 2024 13:27:23 -0700 Subject: [PATCH 3/6] [update] name of function. --- internal/engine/environments/azure.go | 2 +- internal/engine/interactive/interactive.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/engine/environments/azure.go b/internal/engine/environments/azure.go index db013bfd..78a023e9 100644 --- a/internal/engine/environments/azure.go +++ b/internal/engine/environments/azure.go @@ -73,7 +73,7 @@ func (status *AzureDeploymentStatus) SetOutput(output string) { status.Output = output } -func (status *AzureDeploymentStatus) SetConfiguredScript(script string) { +func (status *AzureDeploymentStatus) SetConfiguredMarkdown(script string) { status.ConfiguredMarkdown = script } diff --git a/internal/engine/interactive/interactive.go b/internal/engine/interactive/interactive.go index 45a1f858..72dec03d 100644 --- a/internal/engine/interactive/interactive.go +++ b/internal/engine/interactive/interactive.go @@ -381,7 +381,7 @@ func (model InteractiveModeModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { } logging.GlobalLogger.Tracef("Configured markdown source: %s", configuredMarkdownSource) - model.azureStatus.SetConfiguredScript(configuredMarkdownSource) + model.azureStatus.SetConfiguredMarkdown(configuredMarkdownSource) model.azureStatus.SetOutput(strings.Join(model.CommandLines, "\n")) commands = append( commands, From 6673ffcd8a4878d41c2807c665df6f932155fe54 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Mon, 21 Oct 2024 13:34:59 -0700 Subject: [PATCH 4/6] [update] implementation to exist within the azure specific code. --- internal/engine/environments/azure.go | 43 +++++++++++++++++++++- internal/engine/interactive/interactive.go | 31 +--------------- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/internal/engine/environments/azure.go b/internal/engine/environments/azure.go index 78a023e9..78c07f62 100644 --- a/internal/engine/environments/azure.go +++ b/internal/engine/environments/azure.go @@ -3,9 +3,11 @@ package environments import ( "encoding/json" "fmt" + "strings" "github.com/Azure/InnovationEngine/internal/az" "github.com/Azure/InnovationEngine/internal/logging" + "github.com/Azure/InnovationEngine/internal/patterns" "github.com/Azure/InnovationEngine/internal/ui" ) @@ -73,8 +75,45 @@ func (status *AzureDeploymentStatus) SetOutput(output string) { status.Output = output } -func (status *AzureDeploymentStatus) SetConfiguredMarkdown(script string) { - status.ConfiguredMarkdown = script +// Given a markdown string, replace the environment variables with the values +// provided in the environmentVariables map. Currently this is only used +// by the portal. +func (status *AzureDeploymentStatus) ConfigureMarkdownForDownload( + markdown string, + environmentVariables map[string]string, +) { + for key, value := range environmentVariables { + exportRegex := patterns.ExportVariableRegex(key) + + matches := exportRegex.FindAllStringSubmatch(markdown, -1) + + if len(matches) != 0 { + logging.GlobalLogger.Debugf( + "Found %d matches for the environment variable %s, Replacing them in markdown source.", + len(matches), + key, + ) + } + + for _, match := range matches { + oldLine := match[0] + oldValue := match[1] + + // Replace the old export with the new export statement + newLine := strings.Replace(oldLine, oldValue, value+" ", 1) + logging.GlobalLogger.Debugf("Replacing '%s' with '%s'", oldLine, newLine) + + // Update the code block with the new export statement + markdown = strings.Replace( + markdown, + oldLine, + newLine, + 1, + ) + } + } + + status.ConfiguredMarkdown = markdown } // Print out the status JSON for azure/cloudshell if in the correct environment. diff --git a/internal/engine/interactive/interactive.go b/internal/engine/interactive/interactive.go index 72dec03d..e775ea46 100644 --- a/internal/engine/interactive/interactive.go +++ b/internal/engine/interactive/interactive.go @@ -352,36 +352,7 @@ func (model InteractiveModeModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { model.azureStatus.SetError(err) } - configuredMarkdownSource := model.markdownSource - - for key, value := range environmentVariables { - exportRegex := patterns.ExportVariableRegex(key) - - matches := exportRegex.FindAllStringSubmatch(model.markdownSource, -1) - - if len(matches) != 0 { - logging.GlobalLogger.Debugf( - "Found %d matches for the environment variable %s, Replacing them in markdown source.", - len(matches), - key, - ) - } - - for _, match := range matches { - oldLine := match[0] - oldValue := match[1] - - // Replace the old export with the new export statement - newLine := strings.Replace(oldLine, oldValue, value+" ", 1) - logging.GlobalLogger.Debugf("Replacing '%s' with '%s'", oldLine, newLine) - - // Update the code block with the new export statement - configuredMarkdownSource = strings.Replace(configuredMarkdownSource, oldLine, newLine, 1) - } - } - - logging.GlobalLogger.Tracef("Configured markdown source: %s", configuredMarkdownSource) - model.azureStatus.SetConfiguredMarkdown(configuredMarkdownSource) + model.azureStatus.ConfigureMarkdownForDownload(model.markdownSource, environmentVariables) model.azureStatus.SetOutput(strings.Join(model.CommandLines, "\n")) commands = append( commands, From b0fbbe9aeaa618fae4ee710d519fa8ec54c9e08b Mon Sep 17 00:00:00 2001 From: vmarcella Date: Mon, 21 Oct 2024 13:39:09 -0700 Subject: [PATCH 5/6] [add] environment check. --- internal/engine/environments/azure.go | 5 +++++ internal/engine/interactive/interactive.go | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/engine/environments/azure.go b/internal/engine/environments/azure.go index 78c07f62..7b55b091 100644 --- a/internal/engine/environments/azure.go +++ b/internal/engine/environments/azure.go @@ -81,7 +81,12 @@ func (status *AzureDeploymentStatus) SetOutput(output string) { func (status *AzureDeploymentStatus) ConfigureMarkdownForDownload( markdown string, environmentVariables map[string]string, + environment string, ) { + if !IsAzureEnvironment(environment) { + return + } + for key, value := range environmentVariables { exportRegex := patterns.ExportVariableRegex(key) diff --git a/internal/engine/interactive/interactive.go b/internal/engine/interactive/interactive.go index e775ea46..7a220b63 100644 --- a/internal/engine/interactive/interactive.go +++ b/internal/engine/interactive/interactive.go @@ -352,7 +352,11 @@ func (model InteractiveModeModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { model.azureStatus.SetError(err) } - model.azureStatus.ConfigureMarkdownForDownload(model.markdownSource, environmentVariables) + model.azureStatus.ConfigureMarkdownForDownload( + model.markdownSource, + environmentVariables, + model.environment, + ) model.azureStatus.SetOutput(strings.Join(model.CommandLines, "\n")) commands = append( commands, From af5f4e8623d397e2417710f9560dea6386022446 Mon Sep 17 00:00:00 2001 From: vmarcella Date: Mon, 21 Oct 2024 13:42:58 -0700 Subject: [PATCH 6/6] [update] workflow files to use latest versions of actions. --- .../workflows/build-test-release-tagged.yaml | 40 +++++++++---------- .github/workflows/build-test-release.yaml | 6 +-- .github/workflows/scenario-testing.yaml | 6 +-- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build-test-release-tagged.yaml b/.github/workflows/build-test-release-tagged.yaml index a89aeb93..3762dff4 100644 --- a/.github/workflows/build-test-release-tagged.yaml +++ b/.github/workflows/build-test-release-tagged.yaml @@ -1,32 +1,30 @@ name: build-test-release-tagged - on: push: tags: - v* - jobs: build-test-release: permissions: contents: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Build all targets. - run: | - make build-all - - name: Run unit tests across all targets. - run: | - make test-all - - name: Prepare scenarios to be released. - run: | - sudo apt install zip - zip -r scenarios.zip scenarios - - name: Release Innovation Engine - uses: softprops/action-gh-release@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - generate_release_notes: true - files: | - ./bin/ie - ./scenarios.zip + - uses: actions/checkout@v4 + - name: Build all targets. + run: | + make build-all + - name: Run unit tests across all targets. + run: | + make test-all + - name: Prepare scenarios to be released. + run: | + sudo apt install zip + zip -r scenarios.zip scenarios + - name: Release Innovation Engine + uses: softprops/action-gh-release@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + generate_release_notes: true + files: | + ./bin/ie + ./scenarios.zip diff --git a/.github/workflows/build-test-release.yaml b/.github/workflows/build-test-release.yaml index 118b7270..52c9031f 100644 --- a/.github/workflows/build-test-release.yaml +++ b/.github/workflows/build-test-release.yaml @@ -5,11 +5,11 @@ on: - main jobs: build-test-release: - permissions: - contents: write + permissions: + contents: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build all targets. run: | make build-all diff --git a/.github/workflows/scenario-testing.yaml b/.github/workflows/scenario-testing.yaml index 086f43a3..ec71e034 100644 --- a/.github/workflows/scenario-testing.yaml +++ b/.github/workflows/scenario-testing.yaml @@ -16,7 +16,7 @@ jobs: test-ie-installation: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Check ie installation run: | set -e @@ -38,14 +38,14 @@ jobs: # the testing subscription for any branch in this repository. environment: ScenarioTesting steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build & test all targets. run: | make build-all make test-all WITH_COVERAGE=true make test-local-scenarios ENVIRONMENT=github-action - name: Upload test coverage - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: github.event_name == 'pull_request' with: name: coverage