From 00660d509ddf72015bdc3e71f97ea0f8be1da8ac Mon Sep 17 00:00:00 2001 From: Jeffrey Chien Date: Fri, 13 Feb 2026 16:02:15 +0000 Subject: [PATCH] Add otel_config integration test --- environment/metadata.go | 2 +- generator/test_case_generator.go | 4 +- .../agent_otel_merging_test.go | 129 ----------- test/agent_otel_merging/resources/otel.yaml | 27 --- .../agent_configs/config.json | 2 +- test/otel_config/otel_config_test.go | 203 ++++++++++++++++++ .../resources/metrics.json | 10 +- test/otel_config/resources/otel.yaml | 21 ++ test/otel_config/resources/yaml_only.yaml | 25 +++ .../resources/yaml_only_append.yaml | 17 ++ 10 files changed, 275 insertions(+), 165 deletions(-) delete mode 100644 test/agent_otel_merging/agent_otel_merging_test.go delete mode 100644 test/agent_otel_merging/resources/otel.yaml rename test/{agent_otel_merging => otel_config}/agent_configs/config.json (98%) create mode 100644 test/otel_config/otel_config_test.go rename test/{agent_otel_merging => otel_config}/resources/metrics.json (80%) create mode 100644 test/otel_config/resources/otel.yaml create mode 100644 test/otel_config/resources/yaml_only.yaml create mode 100644 test/otel_config/resources/yaml_only_append.yaml diff --git a/environment/metadata.go b/environment/metadata.go index 4049302e9..caca64642 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -345,7 +345,7 @@ func GetEnvironmentMetaData() *MetaData { return metaDataStorage } - metaDataStorage := &(MetaData{}) + metaDataStorage = &(MetaData{}) fillComputeType(metaDataStorage, registeredMetaDataStrings) fillECSData(metaDataStorage, registeredMetaDataStrings) fillEKSData(metaDataStorage, registeredMetaDataStrings) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 5c25a37df..7fd5d3e99 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -162,7 +162,7 @@ var testTypeToTestConfig = map[string][]testConfig{ targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, }, { - testDir: "./test/agent_otel_merging", + testDir: "./test/otel_config", targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, }, { @@ -235,7 +235,7 @@ var testTypeToTestConfig = map[string][]testConfig{ targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, }, { - testDir: "./test/agent_otel_merging", + testDir: "./test/otel_config", targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, }, { diff --git a/test/agent_otel_merging/agent_otel_merging_test.go b/test/agent_otel_merging/agent_otel_merging_test.go deleted file mode 100644 index e86252e1d..000000000 --- a/test/agent_otel_merging/agent_otel_merging_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package agent_otel_merging - -import ( - "bytes" - "fmt" - "net/http" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" - "github.com/stretchr/testify/require" - - "github.com/aws/amazon-cloudwatch-agent-test/environment" - "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" - "github.com/aws/amazon-cloudwatch-agent-test/util/common" -) - -func init() { - environment.RegisterEnvironmentMetaDataFlags() -} - -// Tests merging of agent and OTEL yaml -// Merges OTLP receiver to Agent yaml -// And Checks if test_metric was generated(YAML merged successfully) -func TestOtelMerging(t *testing.T) { - startAgent(t) - appendOtelConfig(t) - sendPayload(t) - verifyMetricsInCloudWatch(t) - verifyHealthCheck(t) -} - -func startAgent(t *testing.T) { - common.CopyFile(filepath.Join("agent_configs", "config.json"), common.ConfigOutputPath) - require.NoError(t, common.StartAgent(common.ConfigOutputPath, true, false)) - time.Sleep(10 * time.Second) // Wait for the agent to start properly -} - -func appendOtelConfig(t *testing.T) { - cmd := exec.Command("sudo", "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl", - "-a", "append-config", "-s", "-m", "ec2", "-c", "file:"+filepath.Join("resources", "otel.yaml")) - output, err := cmd.CombinedOutput() - require.NoError(t, err, "Failed to append OTEL config: %s", output) - time.Sleep(10 * time.Second) // Wait for the agent to apply the new configuration -} - -func sendPayload(t *testing.T) { - currentTimestamp := time.Now().UnixNano() - - payload, err := os.ReadFile(filepath.Join("resources", "metrics.json")) - require.NoError(t, err, "Failed to read JSON payload from metrics.json") - - payloadStr := string(payload) - payloadStr = strings.ReplaceAll(payloadStr, "$CURRENT_TIMESTAMP", fmt.Sprintf("%d", currentTimestamp)) - payloadStr = strings.ReplaceAll(payloadStr, "$INSTANCE_ID", environment.GetEnvironmentMetaData().InstanceId) - - req, err := http.NewRequest("POST", "http://localhost:4318/v1/metrics", bytes.NewBuffer([]byte(payloadStr))) - require.NoError(t, err, "Failed to create HTTP request") - - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{} - resp, err := client.Do(req) - require.NoError(t, err, "Failed to send HTTP request") - - defer resp.Body.Close() - - require.True(t, resp.StatusCode >= 200 && resp.StatusCode < 300, "Unexpected status code: %d", resp.StatusCode) - time.Sleep(2 * time.Minute) -} - -func verifyMetricsInCloudWatch(t *testing.T) { - metricName := "test_metric" - namespace := "CWAgent-testing-otel" - - serviceDimName := "service.name" - serviceDimValue := "test-service" - instanceDimName := "instance_id" - instanceDimValue := environment.GetEnvironmentMetaData().InstanceId - - dimensions := []types.Dimension{ - { - Name: &serviceDimName, - Value: &serviceDimValue, - }, - { - Name: &instanceDimName, - Value: &instanceDimValue, - }, - } - - startTime := time.Now().Add(-5 * time.Minute) - endTime := time.Now() - periodInSeconds := int32(1) - statType := []types.Statistic{types.StatisticAverage} - - resp, err := awsservice.GetMetricStatistics( - metricName, - namespace, - dimensions, - startTime, - endTime, - periodInSeconds, - statType, - nil, - ) - - require.NoError(t, err, "Failed to fetch metric from CloudWatch") - require.NotEmpty(t, resp.Datapoints, "No data points found for the metric") - -} - -func verifyHealthCheck(t *testing.T) { - endpoint := "http://localhost:13133/health/status" - - resp, err := http.Get(endpoint) - require.NoError(t, err, "Failed to send HTTP request") - - defer resp.Body.Close() - - require.Equal(t, 200, resp.StatusCode, "Expected HTTP status code 200, got %d", resp.StatusCode) -} diff --git a/test/agent_otel_merging/resources/otel.yaml b/test/agent_otel_merging/resources/otel.yaml deleted file mode 100644 index a77e70fbb..000000000 --- a/test/agent_otel_merging/resources/otel.yaml +++ /dev/null @@ -1,27 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - http: - -exporters: - awsemf/otel-merging: - namespace: "CWAgent-testing-otel" - log_group_name: "CWA" - dimension_rollup_option: "NoDimensionRollup" - log_stream_name: "Testing-otel" - resource_to_telemetry_conversion: - enabled: true - version: "0" - -extensions: - health_check: - -service: - extensions: - - health_check - pipelines: - metrics: - receivers: [otlp] - exporters: [awsemf/otel-merging] - diff --git a/test/agent_otel_merging/agent_configs/config.json b/test/otel_config/agent_configs/config.json similarity index 98% rename from test/agent_otel_merging/agent_configs/config.json rename to test/otel_config/agent_configs/config.json index f5e4dcadf..789307009 100644 --- a/test/agent_otel_merging/agent_configs/config.json +++ b/test/otel_config/agent_configs/config.json @@ -8,4 +8,4 @@ } } } -} \ No newline at end of file +} diff --git a/test/otel_config/otel_config_test.go b/test/otel_config/otel_config_test.go new file mode 100644 index 000000000..a4856fa55 --- /dev/null +++ b/test/otel_config/otel_config_test.go @@ -0,0 +1,203 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package otel_config + +import ( + "bytes" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + cwtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + cwltypes "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" +) + +const ( + agentStartupWait = 5 * time.Second + metricIngestion = 30 * time.Second + namespace = "OtelConfigTest" + metricName = "otel_config_test_metric" + otlpEndpoint = "http://localhost:4318/v1/metrics" + validationRetries = 12 + retryInterval = 5 * time.Second + + placeholderInstanceID = "$INSTANCE_ID" + placeholderLogFile = "$LOG_FILE" + placeholderTimestamp = "$CURRENT_TIMESTAMP" + placeholderTestName = "$TEST_NAME" + + testNameFetchJSONAppendYAML = "fetch_json_append_yaml" + testNameFetchYAML = "fetch_yaml" + testNameFetchYAMLAppendJSON = "fetch_yaml_append_json" + testNameFetchYAMLAppendYAML = "fetch_yaml_append_yaml" + + dirResources = "resources" + dirAgentConfigs = "agent_configs" +) + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +// TestFetchJSONAppendYAML tests merging of agent JSON config and OTEL YAML. +func TestFetchJSONAppendYAML(t *testing.T) { + startAgentWithJSON(t) + defer common.StopAgent() + appendConfig(t, prepareConfigWithSubstitutions(t, "otel.yaml", map[string]string{ + placeholderInstanceID: environment.GetEnvironmentMetaData().InstanceId, + })) + runMetricValidation(t, testNameFetchJSONAppendYAML) +} + +// TestFetchYAML tests that the agent can run with only YAML configuration. +func TestFetchYAML(t *testing.T) { + startAgentWithYAML(t, prepareConfigWithSubstitutions(t, "yaml_only.yaml", map[string]string{ + placeholderInstanceID: environment.GetEnvironmentMetaData().InstanceId, + })) + defer common.StopAgent() + runMetricValidation(t, testNameFetchYAML) +} + +// TestFetchYAMLAppendJSON tests fetch-config with YAML followed by append-config with JSON. +func TestFetchYAMLAppendJSON(t *testing.T) { + startAgentWithYAML(t, prepareConfigWithSubstitutions(t, "yaml_only.yaml", map[string]string{ + placeholderInstanceID: environment.GetEnvironmentMetaData().InstanceId, + })) + defer common.StopAgent() + appendConfig(t, filepath.Join(dirAgentConfigs, "config.json")) + runMetricValidation(t, testNameFetchYAMLAppendJSON) +} + +// TestFetchYAMLAppendYAML tests fetch-config + append-config with multiple YAML files. +// Validates append by writing to a log file and verifying it appears in CloudWatch Logs. +func TestFetchYAMLAppendYAML(t *testing.T) { + instanceId := environment.GetEnvironmentMetaData().InstanceId + logGroup := namespace + "/" + instanceId + logStream := testNameFetchYAMLAppendYAML + testLogFile := filepath.Join(t.TempDir(), "test.log") + defer awsservice.DeleteLogGroupAndStream(logGroup, logStream) + + subs := map[string]string{ + placeholderInstanceID: instanceId, + placeholderLogFile: testLogFile, + } + startAgentWithYAML(t, prepareConfigWithSubstitutions(t, "yaml_only.yaml", subs)) + defer common.StopAgent() + appendConfig(t, prepareConfigWithSubstitutions(t, "yaml_only_append.yaml", subs)) + + // Write test log before starting validations + testStart := time.Now() + testMessage := fmt.Sprintf("%s_%d", testNameFetchYAMLAppendYAML, testStart.UnixNano()) + require.NoError(t, os.WriteFile(testLogFile, []byte(testMessage+"\n"), 0644)) + + sendPayload(t, testNameFetchYAMLAppendYAML) + time.Sleep(metricIngestion) + verifyMetricsInCloudWatch(t, testNameFetchYAMLAppendYAML) + assert.NoError(t, verifyLogsInCloudWatch(logGroup, logStream, testMessage, testStart), "Logs validation failed") +} + +func prepareConfigWithSubstitutions(t *testing.T, configFile string, substitutions map[string]string) string { + t.Helper() + tmpFile := filepath.Join(t.TempDir(), configFile) + common.CopyFile(filepath.Join(dirResources, configFile), tmpFile) + require.NoError(t, common.ReplacePlaceholders(tmpFile, substitutions)) + return tmpFile +} + +func verifyLogsInCloudWatch(logGroup, logStream, expectedMessage string, since time.Time) error { + var err error + for i := 0; i < validationRetries; i++ { + err = awsservice.ValidateLogs(logGroup, logStream, &since, nil, + func(events []cwltypes.OutputLogEvent) error { + for _, event := range events { + if strings.Contains(*event.Message, expectedMessage) { + return nil + } + } + return fmt.Errorf("expected message not found in logs") + }) + if err == nil { + return nil + } + time.Sleep(retryInterval) + } + return err +} + +func runMetricValidation(t *testing.T, testName string) { + t.Helper() + sendPayload(t, testName) + time.Sleep(metricIngestion) + verifyMetricsInCloudWatch(t, testName) +} + +func startAgentWithJSON(t *testing.T) { + t.Helper() + common.CopyFile(filepath.Join(dirAgentConfigs, "config.json"), common.ConfigOutputPath) + require.NoError(t, common.StartAgent(common.ConfigOutputPath, false, false)) + time.Sleep(agentStartupWait) +} + +func startAgentWithYAML(t *testing.T, configPath string) { + t.Helper() + require.NoError(t, common.StartAgent(configPath, false, false)) + time.Sleep(agentStartupWait) +} + +func appendConfig(t *testing.T, configPath string) { + t.Helper() + cmd := exec.Command("sudo", "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl", + "-a", "append-config", "-s", "-m", "ec2", "-c", "file:"+configPath) + output, err := cmd.CombinedOutput() + t.Logf("append-config output: %s", output) + require.NoError(t, err, "Failed to append config: %s", output) + time.Sleep(agentStartupWait) +} + +func sendPayload(t *testing.T, testName string) { + t.Helper() + payload, err := os.ReadFile(filepath.Join(dirResources, "metrics.json")) + require.NoError(t, err, "Failed to read payload file") + + payloadStr := string(payload) + payloadStr = strings.ReplaceAll(payloadStr, placeholderTimestamp, fmt.Sprintf("%d", time.Now().UnixNano())) + payloadStr = strings.ReplaceAll(payloadStr, placeholderInstanceID, environment.GetEnvironmentMetaData().InstanceId) + payloadStr = strings.ReplaceAll(payloadStr, placeholderTestName, testName) + + req, err := http.NewRequest(http.MethodPost, otlpEndpoint, bytes.NewBuffer([]byte(payloadStr))) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{Timeout: retryInterval} + resp, err := client.Do(req) + require.NoError(t, err, "Failed to send payload") + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, "Unexpected status code") +} + +func verifyMetricsInCloudWatch(t *testing.T, testName string) { + t.Helper() + instanceDimValue := environment.GetEnvironmentMetaData().InstanceId + + dimensionsFilter := []cwtypes.DimensionFilter{ + {Name: aws.String("test_name"), Value: aws.String(testName)}, + {Name: aws.String("instance_id"), Value: aws.String(instanceDimValue)}, + } + + awsservice.ValidateMetricWithTest(t, metricName, namespace, dimensionsFilter, validationRetries, retryInterval) +} diff --git a/test/agent_otel_merging/resources/metrics.json b/test/otel_config/resources/metrics.json similarity index 80% rename from test/agent_otel_merging/resources/metrics.json rename to test/otel_config/resources/metrics.json index 3afba52a0..46b6f8c10 100644 --- a/test/agent_otel_merging/resources/metrics.json +++ b/test/otel_config/resources/metrics.json @@ -4,9 +4,9 @@ "resource": { "attributes": [ { - "key": "service.name", + "key": "test_name", "value": { - "stringValue": "test-service" + "stringValue": "$TEST_NAME" } }, { @@ -20,19 +20,19 @@ "scopeMetrics": [ { "scope": { - "name": "my.library", + "name": "otel.config.test", "version": "1.0.0" }, "metrics": [ { - "name": "test_metric", + "name": "otel_config_test_metric", "unit": "1", "sum": { "aggregationTemporality": 1, "isMonotonic": true, "dataPoints": [ { - "asDouble": 0.0, + "asDouble": 1.0, "startTimeUnixNano": "$CURRENT_TIMESTAMP", "timeUnixNano": "$CURRENT_TIMESTAMP" } diff --git a/test/otel_config/resources/otel.yaml b/test/otel_config/resources/otel.yaml new file mode 100644 index 000000000..f76e28db8 --- /dev/null +++ b/test/otel_config/resources/otel.yaml @@ -0,0 +1,21 @@ +receivers: + otlp: + protocols: + grpc: + http: + +exporters: + awsemf: + namespace: "OtelConfigTest" + log_group_name: "OtelConfigTest/$INSTANCE_ID" + log_retention: 1 + dimension_rollup_option: "NoDimensionRollup" + log_stream_name: "fetch_json_append_yaml" + resource_to_telemetry_conversion: + enabled: true + +service: + pipelines: + metrics: + receivers: [otlp] + exporters: [awsemf] diff --git a/test/otel_config/resources/yaml_only.yaml b/test/otel_config/resources/yaml_only.yaml new file mode 100644 index 000000000..8d50809d5 --- /dev/null +++ b/test/otel_config/resources/yaml_only.yaml @@ -0,0 +1,25 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + awsemf: + namespace: "OtelConfigTest" + log_group_name: "OtelConfigTest/$INSTANCE_ID" + log_retention: 1 + dimension_rollup_option: "NoDimensionRollup" + log_stream_name: "fetch_yaml" + resource_to_telemetry_conversion: + enabled: true + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [awsemf] diff --git a/test/otel_config/resources/yaml_only_append.yaml b/test/otel_config/resources/yaml_only_append.yaml new file mode 100644 index 000000000..92ad21d32 --- /dev/null +++ b/test/otel_config/resources/yaml_only_append.yaml @@ -0,0 +1,17 @@ +receivers: + filelog: + include: + - $LOG_FILE + start_at: beginning + +exporters: + awscloudwatchlogs: + log_group_name: "OtelConfigTest/$INSTANCE_ID" + log_retention: 1 + log_stream_name: "fetch_yaml_append_yaml" + +service: + pipelines: + logs: + receivers: [filelog] + exporters: [awscloudwatchlogs]