[1] Adding reaper thread to terminate orphaned resources

README.md

@@ -36,6 +36,7 @@ The -hubConfig configuration key and value are optional to specify Selenium Hub
 * ipAddress (required) - Resolvable host name or ip address of the hub for started nodes to connect to.  Note 'localhost' or '127.0.0.1' will not work as this will not be resolvable from the context of another machine
 * totalNodeCount - Maximum number of nodes that can connect to your hub.  Defaults to 150 if not specified
 * extraCapabilities - CSV list of capabilities you want to be considered if specified client side (e.g. adding 'target' to this list will require any capabilities coming in with the 'target' key present to match the value in the node config)
+* useReaperThread - Set this to 'false' if you do not want the reaper thread running.  The reaper thread will automatically terminate instances that are not registered with the hub to prevent unused instances from accumulating in AWS over time.  This can be useful to disable if you're trying to debug a node that the plugin started and killed the java process on it.
 
 More information can be found [here](http://docs.aws.amazon.com/AWSSecurityCredentials/1.0/AboutAWSCredentials.html) on AWS Access
 

pom.xml

@@ -192,19 +192,6 @@
                     </execution>
                 </executions>
             </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-source-plugin</artifactId>
-                <version>2.2.1</version>
-                <executions>
-                    <execution>
-                        <id>attach-sources</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>

src/main/java/com/rmn/qa/AutomationConstants.java

@@ -35,4 +35,5 @@ public interface AutomationConstants {
     String AWS_ACCESS_KEY="awsAccessKey";
     String AWS_PRIVATE_KEY="awsSecretKey";
     String AWS_DEFAULT_RESOURCE_NAME= "aws.properties.default";
+    String REAPER_THREAD_CONFIG = "useReaperThread";
 }

src/main/java/com/rmn/qa/aws/AwsTagReporter.java

@@ -122,6 +122,7 @@ private void setTagsForInstance(String instanceId) {
         }
         // Including a hard coded tag here so we can track which resources originate from this plugin
         Tag nodeTag = new Tag("LaunchSource","SeleniumGridScalerPlugin");
+        log.info("Adding hard-coded tag: " + nodeTag);
         tags.add(nodeTag);
         CreateTagsRequest ctr = new CreateTagsRequest(Arrays.asList(instanceId),tags);
         ec2Client.createTags(ctr);

src/main/java/com/rmn/qa/aws/AwsVmManager.java

@@ -14,9 +14,11 @@
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.services.ec2.AmazonEC2Client;
+import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
 import com.amazonaws.services.ec2.model.Instance;
 import com.amazonaws.services.ec2.model.InstanceState;
 import com.amazonaws.services.ec2.model.InstanceStateChange;
+import com.amazonaws.services.ec2.model.Reservation;
 import com.amazonaws.services.ec2.model.RunInstancesRequest;
 import com.amazonaws.services.ec2.model.RunInstancesResult;
 import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
@@ -302,6 +304,11 @@ public boolean terminateInstance(String instanceId) {
         return true;
     }
 
+    @Override
+    public List<Reservation> describeInstances(DescribeInstancesRequest describeInstancesRequest) {
+        return client.describeInstances(describeInstancesRequest).getReservations();
+    }
+
     /**
      * Returns a zip file containing the necessary user data for the images we're going to spin up
      * @param uuid UUID of the test run

src/main/java/com/rmn/qa/aws/VmManager.java

@@ -11,7 +11,9 @@
  */
 package com.rmn.qa.aws;
 
+import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
 import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.Reservation;
 import com.rmn.qa.NodesCouldNotBeStartedException;
 
 import java.util.List;
@@ -37,4 +39,11 @@ public interface VmManager {
      */
     // TODO Rename to be node or instance in the name
     boolean terminateInstance(String instanceId);
+
+    /**
+     * Returns a list of reservations as defined in the {@link com.amazonaws.services.ec2.model.DescribeInstancesRequest DescribeInstancesRequest}
+     * @param describeInstancesRequest
+     * @return
+     */
+    List<Reservation> describeInstances(DescribeInstancesRequest describeInstancesRequest);
 }

src/main/java/com/rmn/qa/servlet/AutomationTestRunServlet.java

@@ -26,6 +26,7 @@
 import com.rmn.qa.task.AutomationHubCleanupTask;
 import com.rmn.qa.task.AutomationNodeCleanupTask;
 import com.rmn.qa.task.AutomationNodeRegistryTask;
+import com.rmn.qa.task.AutomationReaperTask;
 import com.rmn.qa.task.AutomationRunCleanupTask;
 import org.openqa.grid.internal.ProxySet;
 import org.openqa.grid.internal.Registry;
@@ -111,6 +112,15 @@ private void initCleanupThreads() {
         } else {
             log.info("Hub is not a dynamic hub -- termination logic will not be started");
         }
+        String runReaperThread = System.getProperty(AutomationConstants.REAPER_THREAD_CONFIG);
+        // Reaper thread defaults to on unless specified not to run
+        if(!"false".equalsIgnoreCase(runReaperThread)) {
+            // Spin up a scheduled thread to terminate orphaned instances
+            Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(new AutomationReaperTask(this,ec2),
+                    AutomationTestRunServlet.HUB_TERMINATE_START_DELAY_IN_MINUTES,AutomationTestRunServlet.NODE_REGISTRATION_POLLING_TIME_IN_MINUTES, TimeUnit.MINUTES);
+        } else {
+            log.info("Reaper thread not running due to config flag.");
+        }
     }
 
     void setManageEc2(VmManager ec2) {

src/main/java/com/rmn/qa/task/AutomationReaperTask.java

@@ -0,0 +1,60 @@
+package com.rmn.qa.task;
+
+import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
+import com.amazonaws.services.ec2.model.Filter;
+import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.Reservation;
+import com.google.common.annotations.VisibleForTesting;
+import com.rmn.qa.AutomationContext;
+import com.rmn.qa.AutomationUtils;
+import com.rmn.qa.RegistryRetriever;
+import com.rmn.qa.aws.VmManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+
+public class AutomationReaperTask extends AbstractAutomationCleanupTask {
+
+    private static final Logger log = LoggerFactory.getLogger(AutomationReaperTask.class);
+    @VisibleForTesting static final String NAME = "VM Reaper Task";
+
+    private VmManager ec2;
+
+    /**
+     * Constructs a registry task with the specified context retrieval mechanism
+     * @param registryRetriever Represents the retrieval mechanism you wish to use
+     */
+    public AutomationReaperTask(RegistryRetriever registryRetriever,VmManager ec2) {
+        super(registryRetriever);
+        this.ec2 = ec2;
+    }
+    @Override
+    public void doWork() {
+        log.info("Running " + AutomationReaperTask.NAME);
+        DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest();
+        Filter filter = new Filter("tag:LaunchSource");
+        filter.withValues("SeleniumGridScalerPlugin");
+        describeInstancesRequest.withFilters(filter);
+        List<Reservation> reservations = ec2.describeInstances(describeInstancesRequest);
+        for(Reservation reservation : reservations) {
+            for(Instance instance : reservation.getInstances()) {
+                // Look for orphaned nodes
+                Date threshold = AutomationUtils.modifyDate(new Date(),-30, Calendar.MINUTE);
+                String instanceId = instance.getInstanceId();
+                // If we found a node old enough AND we're not internally tracking it, this means this is an orphaned node and we should terminate it
+                if(threshold.after(instance.getLaunchTime()) && !AutomationContext.getContext().nodeExists(instanceId)) {
+                    log.info("Terminating orphaned node: " + instanceId);
+                    ec2.terminateInstance(instanceId);
+                }
+            }
+        }
+    }
+
+    @Override
+    public String getDescription() {
+        return AutomationReaperTask.NAME;
+    }
+}

src/main/resources/logback.xml

@@ -63,6 +63,10 @@
         <level value="INFO"/>
         <appender-ref ref="FILE" />
     </logger>
+    <logger name="com.rmn.qa.task.AutomationReaperTask">
+        <level value="INFO"/>
+        <appender-ref ref="FILE" />
+    </logger>
     <logger name="com.rmn.qa.AutomationCapabilityMatcher">
         <level value="INFO"/>
         <appender-ref ref="FILE" />

src/test/java/com/rmn/qa/MockVmManager.java

@@ -12,7 +12,9 @@
 
 package com.rmn.qa;
 
+import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
 import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.Reservation;
 import com.rmn.qa.aws.VmManager;
 
 import java.util.ArrayList;
@@ -25,6 +27,7 @@ public class MockVmManager implements VmManager {
     private String browser;
     private boolean throwException = false;
     private boolean terminated = false;
+    private List<Reservation> reservations;
 
 
     @Override
@@ -48,6 +51,11 @@ public boolean terminateInstance(String instanceId) {
         return true;
     }
 
+    @Override
+    public List<Reservation> describeInstances(DescribeInstancesRequest describeInstancesRequest) {
+        return reservations;
+    }
+
     public boolean isNodesLaunched() {
         return nodesLaunched;
     }
@@ -67,4 +75,8 @@ public void setThrowException() {
     public boolean isTerminated() {
         return terminated;
     }
+
+    public void setReservations(List<Reservation> reservations) {
+        this.reservations = reservations;
+    }
 }

src/test/java/com/rmn/qa/aws/AwsTagReporterTest.java

@@ -25,7 +25,7 @@
 public class AwsTagReporterTest {
 
     @Test
-    public void testTagsAssociated() {
+         public void testTagsAssociated() {
         MockAmazonEc2Client client = new MockAmazonEc2Client(null);
         Collection<Instance> instances = Arrays.asList(new Instance());
         DescribeInstancesResult describeInstancesResult = new DescribeInstancesResult();
@@ -34,7 +34,24 @@ public void testTagsAssociated() {
         reservation.setInstances(instances);
         client.setDescribeInstances(describeInstancesResult);
         Properties properties = new Properties();
-        properties.setProperty("accounting_tag","foo");
+        properties.setProperty("tagAccounting","key,value");
+        properties.setProperty("function_tag","foo2");
+        properties.setProperty("product_tag","foo3");
+        AwsTagReporter reporter = new AwsTagReporter("testUuid",client,instances,properties);
+        reporter.run();
+    }
+
+    @Test
+    public void testExceptionCaught() {
+        MockAmazonEc2Client client = new MockAmazonEc2Client(null);
+        Collection<Instance> instances = Arrays.asList(new Instance());
+        DescribeInstancesResult describeInstancesResult = new DescribeInstancesResult();
+        Reservation reservation = new Reservation();
+        describeInstancesResult.setReservations(Arrays.asList(reservation));
+        reservation.setInstances(instances);
+        client.setDescribeInstances(describeInstancesResult);
+        Properties properties = new Properties();
+        properties.setProperty("tagAccounting","key");
         properties.setProperty("function_tag","foo2");
         properties.setProperty("product_tag","foo3");
         AwsTagReporter reporter = new AwsTagReporter("testUuid",client,instances,properties);

src/test/java/com/rmn/qa/task/AutomationReaperTaskTest.java

@@ -0,0 +1,72 @@
+package com.rmn.qa.task;
+
+import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.Reservation;
+import com.rmn.qa.AutomationContext;
+import com.rmn.qa.AutomationDynamicNode;
+import com.rmn.qa.AutomationUtils;
+import com.rmn.qa.MockVmManager;
+import junit.framework.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+
+public class AutomationReaperTaskTest {
+
+    @Test
+    public void testShutdown() {
+        MockVmManager ec2 = new MockVmManager();
+        Reservation reservation = new Reservation();
+        Instance instance = new Instance();
+        String instanceId = "foo";
+        instance.setInstanceId(instanceId);
+        instance.setLaunchTime(AutomationUtils.modifyDate(new Date(),-5,Calendar.HOUR));
+        reservation.setInstances(Arrays.asList(instance));
+        ec2.setReservations(Arrays.asList(reservation));
+        AutomationReaperTask task = new AutomationReaperTask(null,ec2);
+        task.run();
+        Assert.assertTrue("Node should be terminated as it was empty", ec2.isTerminated());
+    }
+
+    @Test
+    // Tests that a node that is not old enough is not terminated
+    public void testNoShutdownTooRecent() {
+        MockVmManager ec2 = new MockVmManager();
+        Reservation reservation = new Reservation();
+        Instance instance = new Instance();
+        String instanceId = "foo";
+        instance.setInstanceId(instanceId);
+        instance.setLaunchTime(AutomationUtils.modifyDate(new Date(),-15,Calendar.MINUTE));
+        reservation.setInstances(Arrays.asList(instance));
+        ec2.setReservations(Arrays.asList(reservation));
+        AutomationReaperTask task = new AutomationReaperTask(null,ec2);
+        task.run();
+        Assert.assertFalse("Node should NOT be terminated as it was not old", ec2.isTerminated());
+    }
+
+    @Test
+    // Tests that a node that is being tracked internally is not shut down
+    public void testNoShutdownNodeTracked() {
+        MockVmManager ec2 = new MockVmManager();
+        Reservation reservation = new Reservation();
+        Instance instance = new Instance();
+        String instanceId = "foo";
+        AutomationContext.getContext().addNode(new AutomationDynamicNode("faky",instanceId,null,null,new Date(),1));
+        instance.setInstanceId(instanceId);
+        instance.setLaunchTime(AutomationUtils.modifyDate(new Date(),-5,Calendar.HOUR));
+        reservation.setInstances(Arrays.asList(instance));
+        ec2.setReservations(Arrays.asList(reservation));
+        AutomationReaperTask task = new AutomationReaperTask(null,ec2);
+        task.run();
+        Assert.assertFalse("Node should NOT be terminated as it was tracked internally", ec2.isTerminated());
+    }
+
+    @Test
+    // Tests that the hardcoded name of the task is correct
+    public void testTaskName() {
+        AutomationReaperTask task = new AutomationReaperTask(null,null);
+        Assert.assertEquals("Name should be the same",AutomationReaperTask.NAME, task.getDescription()  );
+    }
+}