feat: Add cloudwatchlogs concurrency integration tests

.gitallowed

@@ -0,0 +1 @@
+SOFTWARE\\Microsoft\\PowerShell\\1\\ShellIds\\Microsoft\.PowerShell

environment/metadata.go

@@ -69,6 +69,7 @@ type MetaData struct {
 	PerformanceMetricMapName                    string
 	PerformanceTestName                         string
 	IPFamily                                    string
+	IamRoleName                                 string
 }
 
 type MetaDataStrings struct {
@@ -116,6 +117,7 @@ type MetaDataStrings struct {
 	PerformanceMetricMapName                    string
 	PerformanceTestName                         string
 	IPFamily                                    string
+	IamRoleName                                 string
 }
 
 func registerComputeType(dataString *MetaDataStrings) {
@@ -317,6 +319,10 @@ func registerAccountId(dataString *MetaDataStrings) {
 	flag.StringVar(&(dataString.AccountId), "accountId", "", "AWS account Id")
 }
 
+func registerIamRoleName(dataString *MetaDataStrings) {
+	flag.StringVar(&(dataString.IamRoleName), "iamRoleName", "", "IAM role name for the EC2 instance")
+}
+
 func RegisterEnvironmentMetaDataFlags() *MetaDataStrings {
 	registerComputeType(registeredMetaDataStrings)
 	registerECSData(registeredMetaDataStrings)
@@ -336,6 +342,7 @@ func RegisterEnvironmentMetaDataFlags() *MetaDataStrings {
 	registerAgentStartCommand(registeredMetaDataStrings)
 	registerAmpWorkspaceId(registeredMetaDataStrings)
 	registerAccountId(registeredMetaDataStrings)
+	registerIamRoleName(registeredMetaDataStrings)
 
 	return registeredMetaDataStrings
 }
@@ -382,6 +389,7 @@ func GetEnvironmentMetaData() *MetaData {
 	metaDataStorage.SampleApp = registeredMetaDataStrings.SampleApp
 	metaDataStorage.AccountId = registeredMetaDataStrings.AccountId
 	metaDataStorage.IPFamily = registeredMetaDataStrings.IPFamily
+	metaDataStorage.IamRoleName = registeredMetaDataStrings.IamRoleName
 	fillEKSInstallationType(metaDataStorage, registeredMetaDataStrings)
 
 	return metaDataStorage

generator/test_case_generator.go

@@ -89,6 +89,10 @@ var testTypeToTestConfig = map[string][]testConfig{
 	testTypeKeyEc2Linux: {
 		{testDir: "./test/ca_bundle"},
 		{testDir: "./test/cloudwatchlogs"},
+		{
+			testDir:      "./test/cloudwatchlogs_concurrency",
+			terraformDir: "terraform/ec2/cloudwatchlogs_concurrency",
+		},
 		{
 			testDir: "./test/log_state/logfile",
 			targets: map[string]map[string]struct{}{"os": {"al2": {}}},
@@ -192,6 +196,10 @@ var testTypeToTestConfig = map[string][]testConfig{
 	testTypeKeyEc2SELinux: {
 		{testDir: "./test/ca_bundle"},
 		{testDir: "./test/cloudwatchlogs"},
+		{
+			testDir:      "./test/cloudwatchlogs_concurrency",
+			terraformDir: "terraform/ec2/cloudwatchlogs_concurrency",
+		},
 		{
 			testDir: "./test/metrics_number_dimension",
 			targets: map[string]map[string]struct{}{"os": {"al2": {}}},
@@ -430,6 +438,8 @@ type partition struct {
 	configName string
 	tests      []string
 	ami        []string
+	// excludedTestDirs allows excluding specific test directories from this partition
+	excludedTestDirs map[string]struct{}
 	// testConfigOverrides allows partition-specific test configurations
 	// key is testDir, value is the override config
 	testConfigOverrides map[string]testConfig
@@ -445,6 +455,9 @@ var partitionTests = map[string]partition{
 		configName: "_itar",
 		tests:      []string{testTypeKeyEc2Linux},
 		ami:        []string{"cloudwatch-agent-integration-test-aarch64-al2023*"},
+		excludedTestDirs: map[string]struct{}{
+			"./test/cloudwatchlogs_concurrency": {}, // IAM deny policy not deployed to ITAR account
+		},
 		testConfigOverrides: map[string]testConfig{
 			"./test/metric_value_benchmark": {
 				// Exclude DiskIOInstanceStore and DiskIOEBS tests - custom AMI doesn't support NVMe instance store metrics
@@ -460,6 +473,9 @@ var partitionTests = map[string]partition{
 		configName: "_china",
 		tests:      []string{testTypeKeyEc2Linux},
 		ami:        []string{"cloudwatch-agent-integration-test-aarch64-al2023*"},
+		excludedTestDirs: map[string]struct{}{
+			"./test/cloudwatchlogs_concurrency": {}, // IAM deny policy not deployed to China account
+		},
 		testConfigOverrides: map[string]testConfig{
 			"./test/metric_value_benchmark": {
 				// Exclude DiskIOInstanceStore and DiskIOEBS tests - custom AMI doesn't support NVMe instance store metrics
@@ -487,7 +503,7 @@ func main() {
 			if len(partition.tests) != 0 && !slices.Contains(partition.tests, testType) {
 				continue
 			}
-			testMatrix := genMatrix(testType, testConfigs, partition.ami, partition.testConfigOverrides)
+			testMatrix := genMatrix(testType, testConfigs, partition.ami, partition.testConfigOverrides, partition.excludedTestDirs)
 			writeTestMatrixFile(testType+partition.configName, testMatrix)
 		}
 	}
@@ -517,7 +533,7 @@ func generateTestName(testType string, test_directory string) string {
 
 	return strings.Join(cleaned, "_")
 }
-func genMatrix(testType string, testConfigs []testConfig, ami []string, overrides map[string]testConfig) []matrixRow {
+func genMatrix(testType string, testConfigs []testConfig, ami []string, overrides map[string]testConfig, excludedTestDirs map[string]struct{}) []matrixRow {
 	openTestMatrix, err := os.Open(fmt.Sprintf("generator/resources/%v_test_matrix.json", testType))
 
 	if err != nil {
@@ -537,6 +553,13 @@ func genMatrix(testType string, testConfigs []testConfig, ami []string, override
 	testMatrixComplete := make([]matrixRow, 0, len(testMatrix))
 	for _, test := range testMatrix {
 		for _, testConfig := range testConfigs {
+			// Skip excluded test directories
+			if excludedTestDirs != nil {
+				if _, excluded := excludedTestDirs[testConfig.testDir]; excluded {
+					continue
+				}
+			}
+
 			// Apply partition-specific overrides if available
 			if overrides != nil {
 				if override, ok := overrides[testConfig.testDir]; ok {

go.mod

@@ -20,8 +20,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.26.3
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2
 	github.com/aws/aws-sdk-go-v2/service/ecs v1.35.2
+	github.com/aws/aws-sdk-go-v2/service/iam v1.27.4
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2
 	github.com/aws/aws-sdk-go-v2/service/ssm v1.44.2
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.2
 	github.com/aws/aws-sdk-go-v2/service/xray v1.23.2
 	github.com/aws/aws-xray-sdk-go v1.8.3
 	github.com/cenkalti/backoff/v4 v4.2.1
@@ -63,7 +65,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.18.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.2 // indirect
 	github.com/aws/smithy-go v1.18.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect

go.sum

@@ -94,6 +94,8 @@ github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2 h1:e3Imv1oXz+W3Tfclflkh72t5TUP
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2/go.mod h1:d1hAqgLDOPaSO1Piy/0bBmj6oAplFwv6p0cquHntNHM=
 github.com/aws/aws-sdk-go-v2/service/ecs v1.35.2 h1:yIr1T8uPhZT2cKCBeO39utfzG/RKJn3SxbuBOdj18Nc=
 github.com/aws/aws-sdk-go-v2/service/ecs v1.35.2/go.mod h1:MvDz+yXfa2sSEfHB57rdf83deKJIeKEopqHFhVmaRlk=
+github.com/aws/aws-sdk-go-v2/service/iam v1.27.4 h1:W7aZ6WYk/R3kGhBbD6tAVwzYav8k0JQCGhEE+kXKl+k=
+github.com/aws/aws-sdk-go-v2/service/iam v1.27.4/go.mod h1:LklzfZoa7bL/NdhOzoaRtqSLGhu5j+GqE/9WoOQGFKY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 h1:xyfOAYV/ujzZOo01H9+OnyeiRKmTEp6EsITTsmq332Q=

terraform/ec2/cloudwatchlogs_concurrency/main.tf

@@ -0,0 +1,202 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+module "common" {
+  source = "../../common"
+}
+
+module "basic_components" {
+  source = "../../basic_components"
+
+  region = var.region
+}
+
+locals {
+  iam_role_name = "cwa-concurrency-${module.common.testing_id}"
+}
+
+#####################################################################
+# Generate EC2 Key Pair for log in access to EC2
+#####################################################################
+
+resource "tls_private_key" "ssh_key" {
+  count     = var.ssh_key_name == "" ? 1 : 0
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "aws_key_pair" "aws_ssh_key" {
+  count      = var.ssh_key_name == "" ? 1 : 0
+  key_name   = "ec2-key-pair-${module.common.testing_id}"
+  public_key = tls_private_key.ssh_key[0].public_key_openssh
+}
+
+locals {
+  ssh_key_name        = var.ssh_key_name != "" ? var.ssh_key_name : aws_key_pair.aws_ssh_key[0].key_name
+  private_key_content = var.ssh_key_name != "" ? var.ssh_key_value : tls_private_key.ssh_key[0].private_key_pem
+  binary_uri          = var.is_canary ? "${var.s3_bucket}/release/amazon_linux/${var.arc}/latest/${var.binary_name}" : "${var.s3_bucket}/integration-test/binary/${var.cwa_github_sha}/linux/${var.arc}/${var.binary_name}"
+}
+
+#####################################################################
+# Per-test IAM Role with self-modify permissions
+#####################################################################
+
+resource "aws_iam_role" "cwagent_role" {
+  name = local.iam_role_name
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Principal = { Service = "ec2.amazonaws.com" }
+      Action    = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_role_policy" "cwagent_policy" {
+  name = "${local.iam_role_name}-policy"
+  role = aws_iam_role.cwagent_role.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid      = "DenyRestrictedLogGroups"
+        Effect   = "Deny"
+        Action   = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"]
+        Resource = "arn:aws:logs:*:*:log-group:aws-restricted-log-group-name-*"
+      },
+      {
+        Effect = "Allow"
+        Action = [
+          "cloudwatch:PutMetricData",
+          "cloudwatch:GetMetricData",
+          "cloudwatch:ListMetrics",
+          "logs:PutLogEvents",
+          "logs:DescribeLogStreams",
+          "logs:DescribeLogGroups",
+          "logs:CreateLogStream",
+          "logs:CreateLogGroup",
+          "logs:DeleteLogGroup",
+          "logs:DeleteLogStream",
+          "logs:PutRetentionPolicy",
+          "logs:GetLogEvents",
+          "ec2:DescribeVolumes",
+          "ec2:DescribeTags",
+          "ec2:DescribeInstances",
+          "ssm:GetParameter",
+          "ssm:Describe*",
+          "ssm:Get*",
+          "ssm:List*",
+          "s3:GetObject",
+          "s3:GetObjectAcl",
+          "s3:ListBucket",
+        ]
+        Resource = "*"
+      },
+      {
+        Effect   = "Allow"
+        Action   = ["iam:PutRolePolicy", "iam:DeleteRolePolicy"]
+        Resource = aws_iam_role.cwagent_role.arn
+      }
+    ]
+  })
+}
+
+resource "aws_iam_instance_profile" "cwagent_instance_profile" {
+  name = "${local.iam_role_name}-profile"
+  role = aws_iam_role.cwagent_role.name
+}
+
+#####################################################################
+# Generate EC2 Instance
+#####################################################################
+
+resource "aws_instance" "cwagent" {
+  ami                                  = data.aws_ami.latest.id
+  instance_type                        = var.ec2_instance_type
+  key_name                             = local.ssh_key_name
+  iam_instance_profile                 = aws_iam_instance_profile.cwagent_instance_profile.name
+  vpc_security_group_ids               = [module.basic_components.security_group]
+  associate_public_ip_address          = true
+  instance_initiated_shutdown_behavior = "terminate"
+
+  metadata_options {
+    http_endpoint = "enabled"
+    http_tokens   = "required"
+  }
+
+  tags = {
+    Name = "cwagent-integ-test-ec2-${var.test_name}-${module.common.testing_id}"
+  }
+}
+
+data "aws_ami" "latest" {
+  most_recent = true
+  owners      = ["self", "amazon"]
+
+  filter {
+    name   = "name"
+    values = [var.ami]
+  }
+}
+
+#####################################################################
+# Test Setup and Execution
+#####################################################################
+
+resource "null_resource" "integration_test_setup" {
+  connection {
+    type        = "ssh"
+    user        = var.user
+    private_key = local.private_key_content
+    host        = aws_instance.cwagent.public_ip
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "echo sha ${var.cwa_github_sha}",
+      "sudo cloud-init status --wait",
+      "echo clone ${var.github_test_repo} branch ${var.github_test_repo_branch} and install agent",
+      "if [ ! -d amazon-cloudwatch-agent-test/vendor ]; then",
+      "echo 'Vendor directory not found, cloning...'",
+      "sudo rm -rf amazon-cloudwatch-agent-test",
+      "git clone --branch ${var.github_test_repo_branch} ${var.github_test_repo} -q",
+      "fi",
+      "cd amazon-cloudwatch-agent-test",
+      "git rev-parse --short HEAD",
+      "aws s3 cp --no-progress s3://${local.binary_uri} .",
+      "export PATH=$PATH:/snap/bin:/usr/local/go/bin",
+      var.install_agent,
+    ]
+  }
+
+  depends_on = [
+    aws_iam_role.cwagent_role,
+    aws_iam_role_policy.cwagent_policy
+  ]
+}
+
+resource "null_resource" "integration_test_run" {
+  connection {
+    type        = "ssh"
+    user        = var.user
+    private_key = local.private_key_content
+    host        = aws_instance.cwagent.public_ip
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "echo Preparing environment...",
+      "nohup bash -c 'while true; do sudo shutdown -c; sleep 30; done' >/dev/null 2>&1 &",
+      "export AWS_REGION=${var.region}",
+      "export PATH=$PATH:/snap/bin:/usr/local/go/bin",
+      "cd ~/amazon-cloudwatch-agent-test",
+      "echo run sanity test && go test ./test/sanity -p 1 -v",
+      "go test ${var.test_dir} -p 1 -timeout 1h -computeType=EC2 -bucket=${var.s3_bucket} -cwaCommitSha=${var.cwa_github_sha} -instanceId=${aws_instance.cwagent.id} -iamRoleName=${local.iam_role_name} -v"
+    ]
+  }
+
+  depends_on = [null_resource.integration_test_setup]
+}