pkg/destroy/aws: Delete security groups by VPC

Sometimes CI leaks untagged security groups.  Because we are allowed
to remove all resources from within a cluster-owned VPC, add a ByVPC
walker to remove these indirectly-owned groups.

The default name skip avoids errors like:

  time="2019-08-22T12:39:23-07:00" level=debug msg="deleting EC2 security group sg-07c2e6d7b620fb39c: CannotDelete: the specified group: \"sg-07c2e6d7b620fb39c\" name: \"default\" cannot be deleted by a user\n\tstatus code: 400, request id: c88fd74c-77c3-41fe-badb-c53e8022226d" arn="arn:aws:ec2:us-west-2:269733383066:vpc/vpc-0c9097bf5797f611b"

Without the name guard, hitting the error would cause an early exit
from deleteEC2SecurityGroupsByVPC, and mean we never progressed
further in deleteEC2VPC, leading a hung cluster teardown.

Author: wking

pkg/destroy/aws/aws.go

@@ -880,31 +880,45 @@ func deleteEC2SecurityGroup(client *ec2.EC2, id string, logger logrus.FieldLogge
 	}
 
 	for _, group := range response.SecurityGroups {
-		if len(group.IpPermissions) > 0 {
-			_, err := client.RevokeSecurityGroupIngress(&ec2.RevokeSecurityGroupIngressInput{
-				GroupId:       group.GroupId,
-				IpPermissions: group.IpPermissions,
-			})
-			if err != nil {
-				return errors.Wrap(err, "revoking ingress permissions")
-			}
-			logger.Debug("Revoked ingress permissions")
+		err = deleteEC2SecurityGroupObject(client, group, logger)
+		if err != nil {
+			return err
 		}
+	}
 
-		if len(group.IpPermissionsEgress) > 0 {
-			_, err := client.RevokeSecurityGroupEgress(&ec2.RevokeSecurityGroupEgressInput{
-				GroupId:       group.GroupId,
-				IpPermissions: group.IpPermissionsEgress,
-			})
-			if err != nil {
-				return errors.Wrap(err, "revoking egress permissions")
-			}
-			logger.Debug("Revoked egress permissions")
+	return nil
+}
+
+func deleteEC2SecurityGroupObject(client *ec2.EC2, group *ec2.SecurityGroup, logger logrus.FieldLogger) error {
+	if group.GroupName != nil && *group.GroupName == "default" {
+		logger.Debug("Skipping default security group")
+		return nil
+	}
+
+	if len(group.IpPermissions) > 0 {
+		_, err := client.RevokeSecurityGroupIngress(&ec2.RevokeSecurityGroupIngressInput{
+			GroupId:       group.GroupId,
+			IpPermissions: group.IpPermissions,
+		})
+		if err != nil {
+			return errors.Wrap(err, "revoking ingress permissions")
+		}
+		logger.Debug("Revoked ingress permissions")
+	}
+
+	if len(group.IpPermissionsEgress) > 0 {
+		_, err := client.RevokeSecurityGroupEgress(&ec2.RevokeSecurityGroupEgressInput{
+			GroupId:       group.GroupId,
+			IpPermissions: group.IpPermissionsEgress,
+		})
+		if err != nil {
+			return errors.Wrap(err, "revoking egress permissions")
 		}
+		logger.Debug("Revoked egress permissions")
 	}
 
-	_, err = client.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
-		GroupId: aws.String(id),
+	_, err := client.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
+		GroupId: group.GroupId,
 	})
 	if err != nil {
 		if err.(awserr.Error).Code() == "InvalidGroup.NotFound" {
@@ -917,6 +931,41 @@ func deleteEC2SecurityGroup(client *ec2.EC2, id string, logger logrus.FieldLogge
 	return nil
 }
 
+func deleteEC2SecurityGroupsByVPC(client *ec2.EC2, vpc string, failFast bool, logger logrus.FieldLogger) error {
+	var lastError error
+	err := client.DescribeSecurityGroupsPages(
+		&ec2.DescribeSecurityGroupsInput{
+			Filters: []*ec2.Filter{
+				{
+					Name:   aws.String("vpc-id"),
+					Values: []*string{&vpc},
+				},
+			},
+		},
+		func(results *ec2.DescribeSecurityGroupsOutput, lastPage bool) bool {
+			for _, group := range results.SecurityGroups {
+				err := deleteEC2SecurityGroupObject(client, group, logger.WithField("security group", *group.GroupId))
+				if err != nil {
+					if lastError != nil {
+						logger.Debug(err)
+					}
+					lastError = errors.Wrapf(err, "deleting EC2 security group %s", *group.GroupId)
+					if failFast {
+						return false
+					}
+				}
+			}
+
+			return !lastPage
+		},
+	)
+
+	if lastError != nil {
+		return lastError
+	}
+	return err
+}
+
 func deleteEC2Snapshot(client *ec2.EC2, id string, logger logrus.FieldLogger) error {
 	_, err := client.DeleteSnapshot(&ec2.DeleteSnapshotInput{
 		SnapshotId: &id,
@@ -1058,6 +1107,7 @@ func deleteEC2VPC(ec2Client *ec2.EC2, elbClient *elb.ELB, elbv2Client *elbv2.ELB
 		deleteEC2NATGatewaysByVPC,      // not always tagged
 		deleteEC2NetworkInterfaceByVPC, // not always tagged
 		deleteEC2RouteTablesByVPC,      // not always tagged
+		deleteEC2SecurityGroupsByVPC,   // not always tagged
 		deleteEC2SubnetsByVPC,          // not always tagged
 		deleteEC2VPCEndpointsByVPC,     // not taggable
 	} {