diff --git a/README.md b/README.md index b12157a..9dd4802 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ The -hubConfig configuration key and value are optional to specify Selenium Hub * ipAddress (required) - Resolvable host name or ip address of the hub for started nodes to connect to. Note 'localhost' or '127.0.0.1' will not work as this will not be resolvable from the context of another machine * totalNodeCount - Maximum number of nodes that can connect to your hub. Defaults to 150 if not specified * extraCapabilities - CSV list of capabilities you want to be considered if specified client side (e.g. adding 'target' to this list will require any capabilities coming in with the 'target' key present to match the value in the node config) +* useReaperThread - Set this to 'false' if you do not want the reaper thread running. The reaper thread will automatically terminate instances that are not registered with the hub to prevent unused instances from accumulating in AWS over time. This can be useful to disable if you're trying to debug a node that the plugin started and killed the java process on it. More information can be found [here](http://docs.aws.amazon.com/AWSSecurityCredentials/1.0/AboutAWSCredentials.html) on AWS Access diff --git a/pom.xml b/pom.xml index 4c23941..e1fcba2 100644 --- a/pom.xml +++ b/pom.xml @@ -192,19 +192,6 @@ - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - attach-sources - - jar - - - - org.apache.maven.plugins maven-javadoc-plugin diff --git a/src/main/java/com/rmn/qa/AutomationConstants.java b/src/main/java/com/rmn/qa/AutomationConstants.java index 7bbd286..7929751 100644 --- a/src/main/java/com/rmn/qa/AutomationConstants.java +++ b/src/main/java/com/rmn/qa/AutomationConstants.java @@ -35,4 +35,5 @@ public interface AutomationConstants { String AWS_ACCESS_KEY="awsAccessKey"; String AWS_PRIVATE_KEY="awsSecretKey"; String AWS_DEFAULT_RESOURCE_NAME= "aws.properties.default"; + String REAPER_THREAD_CONFIG = "useReaperThread"; } diff --git a/src/main/java/com/rmn/qa/aws/AwsTagReporter.java b/src/main/java/com/rmn/qa/aws/AwsTagReporter.java index dc69df3..ca126bb 100644 --- a/src/main/java/com/rmn/qa/aws/AwsTagReporter.java +++ b/src/main/java/com/rmn/qa/aws/AwsTagReporter.java @@ -122,6 +122,7 @@ private void setTagsForInstance(String instanceId) { } // Including a hard coded tag here so we can track which resources originate from this plugin Tag nodeTag = new Tag("LaunchSource","SeleniumGridScalerPlugin"); + log.info("Adding hard-coded tag: " + nodeTag); tags.add(nodeTag); CreateTagsRequest ctr = new CreateTagsRequest(Arrays.asList(instanceId),tags); ec2Client.createTags(ctr); diff --git a/src/main/java/com/rmn/qa/aws/AwsVmManager.java b/src/main/java/com/rmn/qa/aws/AwsVmManager.java index de45fe3..9051906 100644 --- a/src/main/java/com/rmn/qa/aws/AwsVmManager.java +++ b/src/main/java/com/rmn/qa/aws/AwsVmManager.java @@ -14,9 +14,11 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.Instance; import com.amazonaws.services.ec2.model.InstanceState; import com.amazonaws.services.ec2.model.InstanceStateChange; +import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.RunInstancesResult; import com.amazonaws.services.ec2.model.TerminateInstancesRequest; @@ -302,6 +304,11 @@ public boolean terminateInstance(String instanceId) { return true; } + @Override + public List describeInstances(DescribeInstancesRequest describeInstancesRequest) { + return client.describeInstances(describeInstancesRequest).getReservations(); + } + /** * Returns a zip file containing the necessary user data for the images we're going to spin up * @param uuid UUID of the test run diff --git a/src/main/java/com/rmn/qa/aws/VmManager.java b/src/main/java/com/rmn/qa/aws/VmManager.java index c65beeb..421e8e8 100644 --- a/src/main/java/com/rmn/qa/aws/VmManager.java +++ b/src/main/java/com/rmn/qa/aws/VmManager.java @@ -11,7 +11,9 @@ */ package com.rmn.qa.aws; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Reservation; import com.rmn.qa.NodesCouldNotBeStartedException; import java.util.List; @@ -37,4 +39,11 @@ public interface VmManager { */ // TODO Rename to be node or instance in the name boolean terminateInstance(String instanceId); + + /** + * Returns a list of reservations as defined in the {@link com.amazonaws.services.ec2.model.DescribeInstancesRequest DescribeInstancesRequest} + * @param describeInstancesRequest + * @return + */ + List describeInstances(DescribeInstancesRequest describeInstancesRequest); } diff --git a/src/main/java/com/rmn/qa/servlet/AutomationTestRunServlet.java b/src/main/java/com/rmn/qa/servlet/AutomationTestRunServlet.java index bbbbc61..df088d5 100644 --- a/src/main/java/com/rmn/qa/servlet/AutomationTestRunServlet.java +++ b/src/main/java/com/rmn/qa/servlet/AutomationTestRunServlet.java @@ -26,6 +26,7 @@ import com.rmn.qa.task.AutomationHubCleanupTask; import com.rmn.qa.task.AutomationNodeCleanupTask; import com.rmn.qa.task.AutomationNodeRegistryTask; +import com.rmn.qa.task.AutomationReaperTask; import com.rmn.qa.task.AutomationRunCleanupTask; import org.openqa.grid.internal.ProxySet; import org.openqa.grid.internal.Registry; @@ -111,6 +112,15 @@ private void initCleanupThreads() { } else { log.info("Hub is not a dynamic hub -- termination logic will not be started"); } + String runReaperThread = System.getProperty(AutomationConstants.REAPER_THREAD_CONFIG); + // Reaper thread defaults to on unless specified not to run + if(!"false".equalsIgnoreCase(runReaperThread)) { + // Spin up a scheduled thread to terminate orphaned instances + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(new AutomationReaperTask(this,ec2), + AutomationTestRunServlet.HUB_TERMINATE_START_DELAY_IN_MINUTES,AutomationTestRunServlet.NODE_REGISTRATION_POLLING_TIME_IN_MINUTES, TimeUnit.MINUTES); + } else { + log.info("Reaper thread not running due to config flag."); + } } void setManageEc2(VmManager ec2) { diff --git a/src/main/java/com/rmn/qa/task/AutomationReaperTask.java b/src/main/java/com/rmn/qa/task/AutomationReaperTask.java new file mode 100644 index 0000000..18c5b27 --- /dev/null +++ b/src/main/java/com/rmn/qa/task/AutomationReaperTask.java @@ -0,0 +1,60 @@ +package com.rmn.qa.task; + +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; +import com.amazonaws.services.ec2.model.Filter; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Reservation; +import com.google.common.annotations.VisibleForTesting; +import com.rmn.qa.AutomationContext; +import com.rmn.qa.AutomationUtils; +import com.rmn.qa.RegistryRetriever; +import com.rmn.qa.aws.VmManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +public class AutomationReaperTask extends AbstractAutomationCleanupTask { + + private static final Logger log = LoggerFactory.getLogger(AutomationReaperTask.class); + @VisibleForTesting static final String NAME = "VM Reaper Task"; + + private VmManager ec2; + + /** + * Constructs a registry task with the specified context retrieval mechanism + * @param registryRetriever Represents the retrieval mechanism you wish to use + */ + public AutomationReaperTask(RegistryRetriever registryRetriever,VmManager ec2) { + super(registryRetriever); + this.ec2 = ec2; + } + @Override + public void doWork() { + log.info("Running " + AutomationReaperTask.NAME); + DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest(); + Filter filter = new Filter("tag:LaunchSource"); + filter.withValues("SeleniumGridScalerPlugin"); + describeInstancesRequest.withFilters(filter); + List reservations = ec2.describeInstances(describeInstancesRequest); + for(Reservation reservation : reservations) { + for(Instance instance : reservation.getInstances()) { + // Look for orphaned nodes + Date threshold = AutomationUtils.modifyDate(new Date(),-30, Calendar.MINUTE); + String instanceId = instance.getInstanceId(); + // If we found a node old enough AND we're not internally tracking it, this means this is an orphaned node and we should terminate it + if(threshold.after(instance.getLaunchTime()) && !AutomationContext.getContext().nodeExists(instanceId)) { + log.info("Terminating orphaned node: " + instanceId); + ec2.terminateInstance(instanceId); + } + } + } + } + + @Override + public String getDescription() { + return AutomationReaperTask.NAME; + } +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index c230859..23e2a1d 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -63,6 +63,10 @@ + + + + diff --git a/src/test/java/com/rmn/qa/MockVmManager.java b/src/test/java/com/rmn/qa/MockVmManager.java index b285910..f4f76ae 100644 --- a/src/test/java/com/rmn/qa/MockVmManager.java +++ b/src/test/java/com/rmn/qa/MockVmManager.java @@ -12,7 +12,9 @@ package com.rmn.qa; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Reservation; import com.rmn.qa.aws.VmManager; import java.util.ArrayList; @@ -25,6 +27,7 @@ public class MockVmManager implements VmManager { private String browser; private boolean throwException = false; private boolean terminated = false; + private List reservations; @Override @@ -48,6 +51,11 @@ public boolean terminateInstance(String instanceId) { return true; } + @Override + public List describeInstances(DescribeInstancesRequest describeInstancesRequest) { + return reservations; + } + public boolean isNodesLaunched() { return nodesLaunched; } @@ -67,4 +75,8 @@ public void setThrowException() { public boolean isTerminated() { return terminated; } + + public void setReservations(List reservations) { + this.reservations = reservations; + } } diff --git a/src/test/java/com/rmn/qa/aws/AwsTagReporterTest.java b/src/test/java/com/rmn/qa/aws/AwsTagReporterTest.java index a006a0f..ad472e4 100644 --- a/src/test/java/com/rmn/qa/aws/AwsTagReporterTest.java +++ b/src/test/java/com/rmn/qa/aws/AwsTagReporterTest.java @@ -25,7 +25,7 @@ public class AwsTagReporterTest { @Test - public void testTagsAssociated() { + public void testTagsAssociated() { MockAmazonEc2Client client = new MockAmazonEc2Client(null); Collection instances = Arrays.asList(new Instance()); DescribeInstancesResult describeInstancesResult = new DescribeInstancesResult(); @@ -34,7 +34,24 @@ public void testTagsAssociated() { reservation.setInstances(instances); client.setDescribeInstances(describeInstancesResult); Properties properties = new Properties(); - properties.setProperty("accounting_tag","foo"); + properties.setProperty("tagAccounting","key,value"); + properties.setProperty("function_tag","foo2"); + properties.setProperty("product_tag","foo3"); + AwsTagReporter reporter = new AwsTagReporter("testUuid",client,instances,properties); + reporter.run(); + } + + @Test + public void testExceptionCaught() { + MockAmazonEc2Client client = new MockAmazonEc2Client(null); + Collection instances = Arrays.asList(new Instance()); + DescribeInstancesResult describeInstancesResult = new DescribeInstancesResult(); + Reservation reservation = new Reservation(); + describeInstancesResult.setReservations(Arrays.asList(reservation)); + reservation.setInstances(instances); + client.setDescribeInstances(describeInstancesResult); + Properties properties = new Properties(); + properties.setProperty("tagAccounting","key"); properties.setProperty("function_tag","foo2"); properties.setProperty("product_tag","foo3"); AwsTagReporter reporter = new AwsTagReporter("testUuid",client,instances,properties); diff --git a/src/test/java/com/rmn/qa/task/AutomationReaperTaskTest.java b/src/test/java/com/rmn/qa/task/AutomationReaperTaskTest.java new file mode 100644 index 0000000..2f548c9 --- /dev/null +++ b/src/test/java/com/rmn/qa/task/AutomationReaperTaskTest.java @@ -0,0 +1,72 @@ +package com.rmn.qa.task; + +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Reservation; +import com.rmn.qa.AutomationContext; +import com.rmn.qa.AutomationDynamicNode; +import com.rmn.qa.AutomationUtils; +import com.rmn.qa.MockVmManager; +import junit.framework.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; + +public class AutomationReaperTaskTest { + + @Test + public void testShutdown() { + MockVmManager ec2 = new MockVmManager(); + Reservation reservation = new Reservation(); + Instance instance = new Instance(); + String instanceId = "foo"; + instance.setInstanceId(instanceId); + instance.setLaunchTime(AutomationUtils.modifyDate(new Date(),-5,Calendar.HOUR)); + reservation.setInstances(Arrays.asList(instance)); + ec2.setReservations(Arrays.asList(reservation)); + AutomationReaperTask task = new AutomationReaperTask(null,ec2); + task.run(); + Assert.assertTrue("Node should be terminated as it was empty", ec2.isTerminated()); + } + + @Test + // Tests that a node that is not old enough is not terminated + public void testNoShutdownTooRecent() { + MockVmManager ec2 = new MockVmManager(); + Reservation reservation = new Reservation(); + Instance instance = new Instance(); + String instanceId = "foo"; + instance.setInstanceId(instanceId); + instance.setLaunchTime(AutomationUtils.modifyDate(new Date(),-15,Calendar.MINUTE)); + reservation.setInstances(Arrays.asList(instance)); + ec2.setReservations(Arrays.asList(reservation)); + AutomationReaperTask task = new AutomationReaperTask(null,ec2); + task.run(); + Assert.assertFalse("Node should NOT be terminated as it was not old", ec2.isTerminated()); + } + + @Test + // Tests that a node that is being tracked internally is not shut down + public void testNoShutdownNodeTracked() { + MockVmManager ec2 = new MockVmManager(); + Reservation reservation = new Reservation(); + Instance instance = new Instance(); + String instanceId = "foo"; + AutomationContext.getContext().addNode(new AutomationDynamicNode("faky",instanceId,null,null,new Date(),1)); + instance.setInstanceId(instanceId); + instance.setLaunchTime(AutomationUtils.modifyDate(new Date(),-5,Calendar.HOUR)); + reservation.setInstances(Arrays.asList(instance)); + ec2.setReservations(Arrays.asList(reservation)); + AutomationReaperTask task = new AutomationReaperTask(null,ec2); + task.run(); + Assert.assertFalse("Node should NOT be terminated as it was tracked internally", ec2.isTerminated()); + } + + @Test + // Tests that the hardcoded name of the task is correct + public void testTaskName() { + AutomationReaperTask task = new AutomationReaperTask(null,null); + Assert.assertEquals("Name should be the same",AutomationReaperTask.NAME, task.getDescription() ); + } +}