From be5313db3e915f5a73b20412f81cd7332d4b64d2 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Mon, 11 Feb 2019 15:05:45 -0500 Subject: [PATCH 01/30] AWS Support: Packer portion --- malboxes/config-example.js | 4 ++++ .../snippets/builder_virtualbox_windows.json | 3 +++ .../templates/snippets/postprocessor_aws.json | 17 +++++++++++++++++ malboxes/templates/win10_64_analyst.json | 4 +++- 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 malboxes/templates/snippets/postprocessor_aws.json diff --git a/malboxes/config-example.js b/malboxes/config-example.js index 4957655..cfa1c83 100644 --- a/malboxes/config-example.js +++ b/malboxes/config-example.js @@ -45,6 +45,10 @@ "vsphere_user": "", "vsphere_password": "", "vsphere_insecure": "true", + // If AWS, the following configuration options are mandatory + "aws_access_key": "", + "aws_secret_key": "", + "aws_s3_bucket": "", //"proxy": "company_proxy:3128", diff --git a/malboxes/templates/snippets/builder_virtualbox_windows.json b/malboxes/templates/snippets/builder_virtualbox_windows.json index b2cf75f..9437c83 100644 --- a/malboxes/templates/snippets/builder_virtualbox_windows.json +++ b/malboxes/templates/snippets/builder_virtualbox_windows.json @@ -13,4 +13,7 @@ ], "boot_wait": "10s", "disk_size": "{{ disk_size }}", + {% if hypervisor == "aws" %} + "format": "ova", + {% endif %} "output_directory": "builds" diff --git a/malboxes/templates/snippets/postprocessor_aws.json b/malboxes/templates/snippets/postprocessor_aws.json new file mode 100644 index 0000000..a74605a --- /dev/null +++ b/malboxes/templates/snippets/postprocessor_aws.json @@ -0,0 +1,17 @@ +"post-processors": [ + {# List of list used to make sure post-processors are not run in parallel #} + [{ + "type": "amazon-import", + "access_key": "{{ aws_access_key }}", + "secret_key": "{{ aws_secret_key }}", + "region": "us-east-1", + "ami_name": "malboxes", + "s3_bucket_name": "{{ aws_s3_bucket }}", + "license_type": "BYOL", + "tags": { "Description": "packer amazon import" } + }, + { + "type": "manifest", + "output": "{{ cache_dir }}/manifest.json" + }] +] \ No newline at end of file diff --git a/malboxes/templates/win10_64_analyst.json b/malboxes/templates/win10_64_analyst.json index 911e6fe..957be5b 100644 --- a/malboxes/templates/win10_64_analyst.json +++ b/malboxes/templates/win10_64_analyst.json @@ -1,7 +1,7 @@ { "builders": [{ - {% if hypervisor == "virtualbox" %} + {% if hypervisor == "virtualbox" or hypervisor == "aws" %} "guest_os_type": "Windows10_64", {% include 'snippets/builder_virtualbox_windows.json' %}, {% elif hypervisor == "vsphere" %} @@ -24,6 +24,8 @@ {% if hypervisor == 'virtualbox' %} {% include 'snippets/postprocessor_vagrant.json' %}, + {% elif hypervisor == 'aws' %} + {% include 'snippets/postprocessor_aws.json' %}, {% endif %} "provisioners": [ From 0b4db0dc1b206614a893d0e6a6605ded2c06cbaa Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Mon, 11 Feb 2019 17:13:08 -0500 Subject: [PATCH 02/30] AWS Support: Vagrant portion --- malboxes/config-example.js | 2 ++ malboxes/malboxes.py | 10 ++++++++ .../templates/snippets/postprocessor_aws.json | 6 ++--- malboxes/vagrantfiles/analyst_aws.rb | 23 +++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 malboxes/vagrantfiles/analyst_aws.rb diff --git a/malboxes/config-example.js b/malboxes/config-example.js index cfa1c83..897dd24 100644 --- a/malboxes/config-example.js +++ b/malboxes/config-example.js @@ -49,6 +49,8 @@ "aws_access_key": "", "aws_secret_key": "", "aws_s3_bucket": "", + "aws_security_group": "", + "aws_keypair": "", //"proxy": "company_proxy:3128", diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index c6b549c..1fd00d1 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -463,6 +463,16 @@ def spin(parser, args): elif config['hypervisor'] == 'vsphere': with open("Vagrantfile", 'w') as f: _prepare_vagrantfile(config, "analyst_vsphere.rb", f) + elif config['hypervisor'] == 'aws': + with open("Vagrantfile", 'w') as f: + manifest_filename = os.path.join(DIRS.user_cache_dir, + "manifest.json") + with open(manifest_filename, 'r') as _f: + _manifest = json.loads(_f.read()) + # AMI Id is like: [builds][0][artifact_id] == "us-east-1:ami-aabbccddeeff0011" + # and we keep after the colon + config['aws_ami_id'] = _manifest['builds'][0]['artifact_id'].split(':')[1] + _prepare_vagrantfile(config, "analyst_aws.rb", f) print("Vagrantfile generated. You can move it in your analysis directory " "and issue a `vagrant up` to get started with your VM.") diff --git a/malboxes/templates/snippets/postprocessor_aws.json b/malboxes/templates/snippets/postprocessor_aws.json index a74605a..6e6f108 100644 --- a/malboxes/templates/snippets/postprocessor_aws.json +++ b/malboxes/templates/snippets/postprocessor_aws.json @@ -6,12 +6,12 @@ "secret_key": "{{ aws_secret_key }}", "region": "us-east-1", "ami_name": "malboxes", - "s3_bucket_name": "{{ aws_s3_bucket }}", + "s3_bucket_name": "{{ aws_s3_bucket }}", "license_type": "BYOL", - "tags": { "Description": "packer amazon import" } + "tags": { "Description": "packer amazon import" } }, { "type": "manifest", "output": "{{ cache_dir }}/manifest.json" }] -] \ No newline at end of file +] diff --git a/malboxes/vagrantfiles/analyst_aws.rb b/malboxes/vagrantfiles/analyst_aws.rb new file mode 100644 index 0000000..0a5c31c --- /dev/null +++ b/malboxes/vagrantfiles/analyst_aws.rb @@ -0,0 +1,23 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure(2) do |config| + # Using dummy box with provider=aws metadata + config.vm.box = "dummy" + config.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box" + + config.vm.guest = :windows + config.vm.communicator = :winrm + config.winrm.username = "{{ username }}" + config.winrm.password = "{{ password }}" + + config.vm.provider :aws do |aws, override| + aws.access_key_id = "{{ aws_access_key }}" + aws.secret_access_key = "{{ aws_secret_key }}" + aws.instance_type = "m3.medium" + aws.security_groups = "{{ aws_security_group }}" + aws.keypair_name = "{{ aws_keypair }}" + # FIXME still experimenting with this one + aws.terminate_on_shutdown = true + aws.ami = "{{ aws_ami_id }}" + end +end \ No newline at end of file From a17f830386be1f63cfb018781df687de3d8042e3 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Tue, 12 Feb 2019 08:11:32 -0500 Subject: [PATCH 03/30] ISO download URL fixed for Windows10_64 Deprecated URL for Windows10_64 has been replaced by a working one. --- malboxes/templates/win10_64_analyst.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/malboxes/templates/win10_64_analyst.json b/malboxes/templates/win10_64_analyst.json index 957be5b..d8c1a5e 100644 --- a/malboxes/templates/win10_64_analyst.json +++ b/malboxes/templates/win10_64_analyst.json @@ -11,7 +11,7 @@ "iso_urls": [ "file://{{ iso_path }}/14393.0.160715-1616.RS1_RELEASE_CLIENTENTERPRISEEVAL_OEMRET_X64FRE_EN-US.ISO", - "http://care.dlservice.microsoft.com/dl/download/2/5/4/254230E8-AEA5-43C5-94F6-88CE222A5846/14393.0.160715-1616.RS1_RELEASE_CLIENTENTERPRISEEVAL_OEMRET_X64FRE_EN-US.ISO" + "http://download.microsoft.com/download/2/5/4/254230E8-AEA5-43C5-94F6-88CE222A5846/14393.0.160715-1616.RS1_RELEASE_CLIENTENTERPRISEEVAL_OEMRET_X64FRE_EN-US.ISO" ], "iso_checksum": "a86ae3d664553cd0ee9a6bcd83a5dbe92e3dc41a", "iso_checksum_type": "sha1", From 804f86483d75433b057b4bf0894a449c9e9ff6fc Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Tue, 12 Feb 2019 13:39:16 -0500 Subject: [PATCH 04/30] Build command for AWS The build command and the end message are now adapted for the cloud feature. --- malboxes/malboxes.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index 1fd00d1..a57aff6 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -416,7 +416,7 @@ def build(parser, args): print("Packer failed. Build failed. Exiting...") sys.exit(3) - if not args.skip_vagrant_box_add: + if not (args.skip_vagrant_box_add or config['hypervisor'] == 'aws'): ret = add_box(config, args) else: ret = 0 @@ -425,7 +425,25 @@ def build(parser, args): print("'vagrant box add' failed. Build failed. Exiting...") sys.exit(4) - if not args.skip_vagrant_box_add: + if config['hypervisor'] == 'aws': + print(textwrap.dedent(""" + =============================================================== + The AMI was successfully created on the Amazon Elastic Compute Cloud. + + The required box will be added to Vagrant, if it's not already there, + with the spin command. + + You should generate a Vagrantfile configuration in order to + launch an instance of the AMI: + + malboxes spin {} + + You can re-use this box several times by using `malboxes + spin`. Each EC2 instance will be independent of each other. + ===============================================================""") + .format(args.template, DIRS.user_cache_dir)) + + elif not args.skip_vagrant_box_add: print(textwrap.dedent(""" =============================================================== A base box was imported into your local Vagrant box repository. @@ -441,8 +459,7 @@ def build(parser, args): spin`. Each VM will be independent of each other. ===============================================================""") .format(args.template, DIRS.user_cache_dir)) - - + def spin(parser, args): """ Creates a Vagrantfile meant for user-interaction in the current directory. @@ -476,7 +493,6 @@ def spin(parser, args): print("Vagrantfile generated. You can move it in your analysis directory " "and issue a `vagrant up` to get started with your VM.") - def prepare_profile(template, config): """Converts the profile to a powershell script.""" From 079614ac86193389d93bab13e6e66fbc79fae834 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Tue, 12 Feb 2019 14:44:15 -0500 Subject: [PATCH 05/30] Updated description of the AMI The AMI description is no longer the default one and now gives information about the packer template used. --- malboxes/templates/snippets/postprocessor_aws.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/malboxes/templates/snippets/postprocessor_aws.json b/malboxes/templates/snippets/postprocessor_aws.json index 6e6f108..82ab63a 100644 --- a/malboxes/templates/snippets/postprocessor_aws.json +++ b/malboxes/templates/snippets/postprocessor_aws.json @@ -8,7 +8,7 @@ "ami_name": "malboxes", "s3_bucket_name": "{{ aws_s3_bucket }}", "license_type": "BYOL", - "tags": { "Description": "packer amazon import" } + "tags": { "Description": "This AMI was generated using Malboxes from the following packer template: {{ template_name }}" } }, { "type": "manifest", From 10ad46efa9b5cd341bc4fc840f829048f07c69cf Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Tue, 12 Feb 2019 16:10:48 -0500 Subject: [PATCH 06/30] Documentation: AWS part The code related to the AWS feature for Windows 10 is now documented, --- .../snippets/builder_virtualbox_windows.json | 2 ++ .../templates/snippets/postprocessor_aws.json | 16 +++++++++++----- malboxes/vagrantfiles/analyst_aws.rb | 7 +++++-- malboxes/vagrantfiles/box_win.rb | 1 + 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/malboxes/templates/snippets/builder_virtualbox_windows.json b/malboxes/templates/snippets/builder_virtualbox_windows.json index 9437c83..58933f3 100644 --- a/malboxes/templates/snippets/builder_virtualbox_windows.json +++ b/malboxes/templates/snippets/builder_virtualbox_windows.json @@ -13,6 +13,8 @@ ], "boot_wait": "10s", "disk_size": "{{ disk_size }}", + {# The ouput format of the virtual machine requires to be .OVA + for the amazon-import post-processor #} {% if hypervisor == "aws" %} "format": "ova", {% endif %} diff --git a/malboxes/templates/snippets/postprocessor_aws.json b/malboxes/templates/snippets/postprocessor_aws.json index 82ab63a..86f5b78 100644 --- a/malboxes/templates/snippets/postprocessor_aws.json +++ b/malboxes/templates/snippets/postprocessor_aws.json @@ -1,6 +1,10 @@ "post-processors": [ - {# List of list used to make sure post-processors are not run in parallel #} - [{ + {# List of lists used to make sure post-processors are not run in parallel #} + [ + {# Exports the .OVA created by the builder to the S3 bucket then converts the file to an AMI + (This can take a very long time). + Use this command to see the status of the import: aws ec2 describe-import-image-tasks #} + { "type": "amazon-import", "access_key": "{{ aws_access_key }}", "secret_key": "{{ aws_secret_key }}", @@ -9,9 +13,11 @@ "s3_bucket_name": "{{ aws_s3_bucket }}", "license_type": "BYOL", "tags": { "Description": "This AMI was generated using Malboxes from the following packer template: {{ template_name }}" } - }, - { + }, + {# The Vagrantfile will retrieve the AMI ID from the generated manifest #} + { "type": "manifest", "output": "{{ cache_dir }}/manifest.json" - }] + } + ] ] diff --git a/malboxes/vagrantfiles/analyst_aws.rb b/malboxes/vagrantfiles/analyst_aws.rb index 0a5c31c..cd108c9 100644 --- a/malboxes/vagrantfiles/analyst_aws.rb +++ b/malboxes/vagrantfiles/analyst_aws.rb @@ -10,14 +10,17 @@ config.winrm.username = "{{ username }}" config.winrm.password = "{{ password }}" + # Giving plenty of times for updates + config.vm.boot_timeout = 600 + config.vm.graceful_halt_timeout = 600 + + #Configuration information for AWS config.vm.provider :aws do |aws, override| aws.access_key_id = "{{ aws_access_key }}" aws.secret_access_key = "{{ aws_secret_key }}" aws.instance_type = "m3.medium" aws.security_groups = "{{ aws_security_group }}" aws.keypair_name = "{{ aws_keypair }}" - # FIXME still experimenting with this one - aws.terminate_on_shutdown = true aws.ami = "{{ aws_ami_id }}" end end \ No newline at end of file diff --git a/malboxes/vagrantfiles/box_win.rb b/malboxes/vagrantfiles/box_win.rb index 32f4bbe..1396768 100644 --- a/malboxes/vagrantfiles/box_win.rb +++ b/malboxes/vagrantfiles/box_win.rb @@ -7,6 +7,7 @@ # Giving plenty of times for updates config.vm.boot_timeout = 600 config.vm.graceful_halt_timeout = 600 + config.vm.provider "virtualbox" do |vb| vb.gui = true vb.customize ["modifyvm", :id, "--vram", "128"] From bfb7fce7e434eb75a08426e2ad6e7435ffef9797 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 13 Feb 2019 13:58:26 -0500 Subject: [PATCH 07/30] AWS Support: Windows 7 The AWS feature now supports Windows 7 64x --- malboxes/scripts/windows/do-nothing.ps1 | 1 + malboxes/templates/snippets/postprocessor_aws.json | 4 ++-- malboxes/templates/snippets/provision_powershell_win7.json | 3 ++- malboxes/templates/win7_64_analyst.json | 6 +++++- 4 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 malboxes/scripts/windows/do-nothing.ps1 diff --git a/malboxes/scripts/windows/do-nothing.ps1 b/malboxes/scripts/windows/do-nothing.ps1 new file mode 100644 index 0000000..e58a5de --- /dev/null +++ b/malboxes/scripts/windows/do-nothing.ps1 @@ -0,0 +1 @@ +#This is only to simplify the use of the comma in JSON files. \ No newline at end of file diff --git a/malboxes/templates/snippets/postprocessor_aws.json b/malboxes/templates/snippets/postprocessor_aws.json index 86f5b78..4db299f 100644 --- a/malboxes/templates/snippets/postprocessor_aws.json +++ b/malboxes/templates/snippets/postprocessor_aws.json @@ -9,10 +9,10 @@ "access_key": "{{ aws_access_key }}", "secret_key": "{{ aws_secret_key }}", "region": "us-east-1", - "ami_name": "malboxes", + "ami_name": "malboxes {{ template_name }}", "s3_bucket_name": "{{ aws_s3_bucket }}", "license_type": "BYOL", - "tags": { "Description": "This AMI was generated using Malboxes from the following packer template: {{ template_name }}" } + "tags": { "Description": "This AMI was generated using Malboxes" } }, {# The Vagrantfile will retrieve the AMI ID from the generated manifest #} { diff --git a/malboxes/templates/snippets/provision_powershell_win7.json b/malboxes/templates/snippets/provision_powershell_win7.json index dcdc747..dbb4b26 100644 --- a/malboxes/templates/snippets/provision_powershell_win7.json +++ b/malboxes/templates/snippets/provision_powershell_win7.json @@ -5,8 +5,9 @@ {% if not windows_updates == "true" %}"{{ dir }}/scripts/windows/disable_auto-updates.ps1",{% endif %} {% if not windows_defender == "true" %}"{{ dir }}/scripts/windows/disable_defender.ps1",{% endif %} {% if hypervisor == "virtualbox" %} - "{{ dir }}/scripts/windows/vmtools.ps1" + "{{ dir }}/scripts/windows/vmtools.ps1", {% endif %} + "{{ dir }}/scripts/windows/do-nothing.ps1" ] }, { diff --git a/malboxes/templates/win7_64_analyst.json b/malboxes/templates/win7_64_analyst.json index 97dd8e1..eeeaf33 100644 --- a/malboxes/templates/win7_64_analyst.json +++ b/malboxes/templates/win7_64_analyst.json @@ -23,7 +23,11 @@ ] }], - {% include 'snippets/postprocessor_vagrant.json' %}, + {% if hypervisor == 'aws' %} + {% include 'snippets/postprocessor_aws.json' %}, + {% else %} + {% include 'snippets/postprocessor_vagrant.json' %}, + {% endif %} "provisioners": [ From 171e51bd2daacc7e197a3d8e4bbe741e3c1a5ae3 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 13 Feb 2019 18:14:33 -0500 Subject: [PATCH 08/30] Manifest.json no longer needed The AMI ID is now retrieved from AWS by the library Boto 3 instead of a post-processor. Plus, an error message was added if you try to build more than once the same template. --- malboxes/malboxes.py | 46 ++++++++++++++----- .../templates/snippets/postprocessor_aws.json | 36 ++++++--------- 2 files changed, 48 insertions(+), 34 deletions(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index a57aff6..055724b 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -34,6 +34,8 @@ from jinja2 import Environment, FileSystemLoader from jsmin import jsmin +import boto3 + from malboxes._version import __version__ DIRS = AppDirs("malboxes") @@ -142,7 +144,7 @@ def prepare_packer_template(config, template_name): 'templates/{}.json'.format(template_name)) except FileNotFoundError: print("Template doesn't exist: {}".format(template_name)) - sys.exit(2) + sys.exit(1) filepath = resource_filename(__name__, 'templates/') env = Environment(loader=FileSystemLoader(filepath), autoescape=False, @@ -386,7 +388,7 @@ def default(parser, args): parser.print_help() print("\n") list_templates(parser, args) - sys.exit(1) + sys.exit(2) def list_templates(parser, args): @@ -398,15 +400,41 @@ def list_templates(parser, args): print(m.group(1)) print() +def get_AMI_ID_by_template(template): + boto3_connection = boto3.resource("ec2") + images = list(boto3_connection.images.filter(Owners=['self'],Filters=[{'Name': 'tag:Template', 'Values': [template]}])) + return images[0].image_id -def build(parser, args): +def template_already_AMI(template): + boto3_connection = boto3.resource("ec2") + images = list(boto3_connection.images.filter(Owners=['self'],Filters=[{'Name': 'tag:Template', 'Values': [template]}])) + if not images: + return False + else: + return True +def build(parser, args): print("Generating configuration files...") config, packer_tmpl = prepare_config(args) prepare_autounattend(config) _prepare_vagrantfile(config, "box_win.rb", create_cachefd('box_win.rb')) print("Configuration files are ready") + if config['hypervisor'] == 'aws' and template_already_AMI(args.template): + print(textwrap.dedent(""" + =============================================================== + This template has already been converted to an AMI. + + You should generate a Vagrantfile configuration in order to + launch an instance of this AMI: + + malboxes spin {} + + Exiting... + ===============================================================""") + .format(args.template, DIRS.user_cache_dir)) + sys.exit(4) + if not args.skip_packer_build: ret = run_packer(packer_tmpl, args) else: @@ -423,7 +451,7 @@ def build(parser, args): if ret != 0: print("'vagrant box add' failed. Build failed. Exiting...") - sys.exit(4) + sys.exit(5) if config['hypervisor'] == 'aws': print(textwrap.dedent(""" @@ -466,7 +494,7 @@ def spin(parser, args): """ if os.path.isfile('Vagrantfile'): print("Vagrantfile already exists. Please move it away. Exiting...") - sys.exit(5) + sys.exit(6) config, _ = prepare_config(args) @@ -482,13 +510,7 @@ def spin(parser, args): _prepare_vagrantfile(config, "analyst_vsphere.rb", f) elif config['hypervisor'] == 'aws': with open("Vagrantfile", 'w') as f: - manifest_filename = os.path.join(DIRS.user_cache_dir, - "manifest.json") - with open(manifest_filename, 'r') as _f: - _manifest = json.loads(_f.read()) - # AMI Id is like: [builds][0][artifact_id] == "us-east-1:ami-aabbccddeeff0011" - # and we keep after the colon - config['aws_ami_id'] = _manifest['builds'][0]['artifact_id'].split(':')[1] + config['aws_ami_id'] = get_AMI_ID_by_template(config['template_name']) _prepare_vagrantfile(config, "analyst_aws.rb", f) print("Vagrantfile generated. You can move it in your analysis directory " "and issue a `vagrant up` to get started with your VM.") diff --git a/malboxes/templates/snippets/postprocessor_aws.json b/malboxes/templates/snippets/postprocessor_aws.json index 4db299f..eb8be1b 100644 --- a/malboxes/templates/snippets/postprocessor_aws.json +++ b/malboxes/templates/snippets/postprocessor_aws.json @@ -1,23 +1,15 @@ -"post-processors": [ - {# List of lists used to make sure post-processors are not run in parallel #} - [ - {# Exports the .OVA created by the builder to the S3 bucket then converts the file to an AMI - (This can take a very long time). - Use this command to see the status of the import: aws ec2 describe-import-image-tasks #} - { - "type": "amazon-import", - "access_key": "{{ aws_access_key }}", - "secret_key": "{{ aws_secret_key }}", - "region": "us-east-1", - "ami_name": "malboxes {{ template_name }}", - "s3_bucket_name": "{{ aws_s3_bucket }}", - "license_type": "BYOL", - "tags": { "Description": "This AMI was generated using Malboxes" } - }, - {# The Vagrantfile will retrieve the AMI ID from the generated manifest #} - { - "type": "manifest", - "output": "{{ cache_dir }}/manifest.json" - } - ] +"post-processors": [ + {# Exports the .OVA created by the builder to the S3 bucket then converts the file to an AMI + (This can take a very long time). + Use this command to see the status of the import: aws ec2 describe-import-image-tasks #} + { + "type": "amazon-import", + "access_key": "{{ aws_access_key }}", + "secret_key": "{{ aws_secret_key }}", + "region": "us-east-1", + "ami_name": "malboxes", + "s3_bucket_name": "{{ aws_s3_bucket }}", + "license_type": "BYOL", + "tags": { "Description": "This AMI was generated using Malboxes.", "Template": "{{ template_name }}" } + } ] From 59104ed722785af34165c61e23ec7b726ff79615 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Mon, 18 Feb 2019 09:02:23 -0500 Subject: [PATCH 09/30] Boto3 now gets AWS creds from config.js Boto3 uses the AWS credentials from the config.js file instead of the ones stored in environmental variables. --- malboxes/malboxes.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index 055724b..53f7738 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -400,15 +400,33 @@ def list_templates(parser, args): print(m.group(1)) print() -def get_AMI_ID_by_template(template): - boto3_connection = boto3.resource("ec2") - images = list(boto3_connection.images.filter(Owners=['self'],Filters=[{'Name': 'tag:Template', 'Values': [template]}])) - return images[0].image_id - -def template_already_AMI(template): - boto3_connection = boto3.resource("ec2") - images = list(boto3_connection.images.filter(Owners=['self'],Filters=[{'Name': 'tag:Template', 'Values': [template]}])) - if not images: +def create_EC2_client(config): + """ + Creates a client to interact with Amazon Elastic Compute Cloud. + It's Currently only used to retrieve the AMI ID. + """ + return boto3.client( + 'ec2', + aws_access_key_id=config['aws_access_key'], + aws_secret_access_key=config['aws_secret_key'], + region_name='us-east-1', + ) + +def get_AMI_ID_by_template(config, template): + """ + Gets the ID of an AMI by the template tag on it. + """ + images = create_EC2_client(config).describe_images(Owners=['self'],Filters=[{'Name': 'tag:Template', 'Values': [template]}]) + return images['Images'][0]['ImageId'] + +def template_already_AMI(config, template): + """ + Verifies if there's already an AMI based on a template. + If so, returns True. + Otherwise, returns False + """ + images = create_EC2_client(config).describe_images(Owners=['self'],Filters=[{'Name': 'tag:Template', 'Values': [template]}]) + if not images['Images']: return False else: return True @@ -419,8 +437,7 @@ def build(parser, args): prepare_autounattend(config) _prepare_vagrantfile(config, "box_win.rb", create_cachefd('box_win.rb')) print("Configuration files are ready") - - if config['hypervisor'] == 'aws' and template_already_AMI(args.template): + if config['hypervisor'] == 'aws' and template_already_AMI(config, args.template): print(textwrap.dedent(""" =============================================================== This template has already been converted to an AMI. @@ -510,7 +527,7 @@ def spin(parser, args): _prepare_vagrantfile(config, "analyst_vsphere.rb", f) elif config['hypervisor'] == 'aws': with open("Vagrantfile", 'w') as f: - config['aws_ami_id'] = get_AMI_ID_by_template(config['template_name']) + config['aws_ami_id'] = get_AMI_ID_by_template(config, config['template']) _prepare_vagrantfile(config, "analyst_aws.rb", f) print("Vagrantfile generated. You can move it in your analysis directory " "and issue a `vagrant up` to get started with your VM.") From e5091acb6aa75796620d6808b6dfa90e9a2f4c7a Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Mon, 18 Feb 2019 13:04:07 -0500 Subject: [PATCH 10/30] AWS region is now in config The region can now be set in config.js --- malboxes/config-example.js | 1 + malboxes/malboxes.py | 2 +- malboxes/templates/snippets/postprocessor_aws.json | 5 ++--- malboxes/vagrantfiles/analyst_aws.rb | 4 ++++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/malboxes/config-example.js b/malboxes/config-example.js index 897dd24..f316354 100644 --- a/malboxes/config-example.js +++ b/malboxes/config-example.js @@ -51,6 +51,7 @@ "aws_s3_bucket": "", "aws_security_group": "", "aws_keypair": "", + "aws_region": "", //"proxy": "company_proxy:3128", diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index 53f7738..dfdc1d5 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -409,7 +409,7 @@ def create_EC2_client(config): 'ec2', aws_access_key_id=config['aws_access_key'], aws_secret_access_key=config['aws_secret_key'], - region_name='us-east-1', + region_name=config['aws_region'], ) def get_AMI_ID_by_template(config, template): diff --git a/malboxes/templates/snippets/postprocessor_aws.json b/malboxes/templates/snippets/postprocessor_aws.json index eb8be1b..af32027 100644 --- a/malboxes/templates/snippets/postprocessor_aws.json +++ b/malboxes/templates/snippets/postprocessor_aws.json @@ -6,10 +6,9 @@ "type": "amazon-import", "access_key": "{{ aws_access_key }}", "secret_key": "{{ aws_secret_key }}", - "region": "us-east-1", - "ami_name": "malboxes", + "region": "{{ aws_region }}", "s3_bucket_name": "{{ aws_s3_bucket }}", "license_type": "BYOL", - "tags": { "Description": "This AMI was generated using Malboxes.", "Template": "{{ template_name }}" } + "tags": {"Name" : "Malboxes", "Template": "{{ template_name }}"} } ] diff --git a/malboxes/vagrantfiles/analyst_aws.rb b/malboxes/vagrantfiles/analyst_aws.rb index cd108c9..40eface 100644 --- a/malboxes/vagrantfiles/analyst_aws.rb +++ b/malboxes/vagrantfiles/analyst_aws.rb @@ -22,5 +22,9 @@ aws.security_groups = "{{ aws_security_group }}" aws.keypair_name = "{{ aws_keypair }}" aws.ami = "{{ aws_ami_id }}" + aws.tags = { + 'Name' => "Malboxes" + 'Template' => "{{ template_name }}" + } end end \ No newline at end of file From e720a135587bc6c52c3ed04115139f45afa5cb10 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Mon, 18 Feb 2019 13:39:04 -0500 Subject: [PATCH 11/30] Fix WinRM inbound rules WinRM is now working even if the network profile is configured as public --- malboxes/scripts/windows/allow-WinRM-public.ps1 | 2 ++ malboxes/scripts/windows/do-nothing.ps1 | 1 - malboxes/templates/snippets/provision_powershell.json | 3 ++- malboxes/templates/snippets/provision_powershell_win7.json | 2 +- malboxes/vagrantfiles/analyst_aws.rb | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 malboxes/scripts/windows/allow-WinRM-public.ps1 delete mode 100644 malboxes/scripts/windows/do-nothing.ps1 diff --git a/malboxes/scripts/windows/allow-WinRM-public.ps1 b/malboxes/scripts/windows/allow-WinRM-public.ps1 new file mode 100644 index 0000000..f3180f0 --- /dev/null +++ b/malboxes/scripts/windows/allow-WinRM-public.ps1 @@ -0,0 +1,2 @@ +# Change the network types of the WinRM rules for public +netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new profile=public \ No newline at end of file diff --git a/malboxes/scripts/windows/do-nothing.ps1 b/malboxes/scripts/windows/do-nothing.ps1 deleted file mode 100644 index e58a5de..0000000 --- a/malboxes/scripts/windows/do-nothing.ps1 +++ /dev/null @@ -1 +0,0 @@ -#This is only to simplify the use of the comma in JSON files. \ No newline at end of file diff --git a/malboxes/templates/snippets/provision_powershell.json b/malboxes/templates/snippets/provision_powershell.json index c1d0106..1786ce2 100644 --- a/malboxes/templates/snippets/provision_powershell.json +++ b/malboxes/templates/snippets/provision_powershell.json @@ -8,7 +8,8 @@ {% endif %} "{{ dir }}/scripts/windows/installtools.ps1", {% if profile is defined %}"{{ cache_dir }}/profile-{{ profile }}.ps1",{% endif %} - "{{ dir }}/scripts/windows/malware_analysis.ps1" + "{{ dir }}/scripts/windows/malware_analysis.ps1", + "{{ dir }}/scripts/windows/allow-WinRM-public.ps1" ] } {% if choco_packages %}, diff --git a/malboxes/templates/snippets/provision_powershell_win7.json b/malboxes/templates/snippets/provision_powershell_win7.json index dbb4b26..0dad13f 100644 --- a/malboxes/templates/snippets/provision_powershell_win7.json +++ b/malboxes/templates/snippets/provision_powershell_win7.json @@ -7,7 +7,7 @@ {% if hypervisor == "virtualbox" %} "{{ dir }}/scripts/windows/vmtools.ps1", {% endif %} - "{{ dir }}/scripts/windows/do-nothing.ps1" + "{{ dir }}/scripts/windows/allow-WinRM-public.ps1" ] }, { diff --git a/malboxes/vagrantfiles/analyst_aws.rb b/malboxes/vagrantfiles/analyst_aws.rb index 40eface..3c12b64 100644 --- a/malboxes/vagrantfiles/analyst_aws.rb +++ b/malboxes/vagrantfiles/analyst_aws.rb @@ -23,7 +23,7 @@ aws.keypair_name = "{{ aws_keypair }}" aws.ami = "{{ aws_ami_id }}" aws.tags = { - 'Name' => "Malboxes" + 'Name' => "Malboxes", 'Template' => "{{ template_name }}" } end From a136aea3cf7754d9757de6e6f770849d386768f9 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Tue, 19 Feb 2019 07:56:17 -0500 Subject: [PATCH 12/30] Updated requirements for installation Boto3 is now in the requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index a91552e..65c72cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ appdirs Jinja2>=2.9 jsmin +boto3 + From c5e29c83daf3104a3221b5205dbe131b2e3525cd Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Tue, 19 Feb 2019 10:03:49 -0500 Subject: [PATCH 13/30] Instance type and group added to config The instance type and the security group can now be set by the user in the config.js --- malboxes/config-example.js | 8 +++++--- malboxes/vagrantfiles/analyst_aws.rb | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/malboxes/config-example.js b/malboxes/config-example.js index f316354..25d9226 100644 --- a/malboxes/config-example.js +++ b/malboxes/config-example.js @@ -34,7 +34,7 @@ // Provision settings // Which Hypervisor for privisoning and deployment? (Options are: "virtualbox" and "vsphere") Default is "virtualbox" "hypervisor": "virtualbox", - //If vsphere, the following configuration options are mandatory + // If vsphere, the following configuration options are mandatory "remote_host": "", "remote_datastore": "", "remote_username": "", @@ -49,9 +49,11 @@ "aws_access_key": "", "aws_secret_key": "", "aws_s3_bucket": "", - "aws_security_group": "", "aws_keypair": "", - "aws_region": "", + // Optional + "aws_security_group": "default", + "aws_region": "us-east-1", + "aws_instance_type" : "m3.medium", //"proxy": "company_proxy:3128", diff --git a/malboxes/vagrantfiles/analyst_aws.rb b/malboxes/vagrantfiles/analyst_aws.rb index 3c12b64..2c67ac8 100644 --- a/malboxes/vagrantfiles/analyst_aws.rb +++ b/malboxes/vagrantfiles/analyst_aws.rb @@ -18,7 +18,7 @@ config.vm.provider :aws do |aws, override| aws.access_key_id = "{{ aws_access_key }}" aws.secret_access_key = "{{ aws_secret_key }}" - aws.instance_type = "m3.medium" + aws.instance_type = "{{ aws_instance_type }}" aws.security_groups = "{{ aws_security_group }}" aws.keypair_name = "{{ aws_keypair }}" aws.ami = "{{ aws_ami_id }}" From e182f88c494562f9e835d5957202d99e79fa1cb3 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 20 Feb 2019 12:54:57 -0500 Subject: [PATCH 14/30] WinRM firewall rule added As a workaround for the public network connectivity problem, a new rule has been created with a public network type. --- malboxes/scripts/windows/allow-WinRM-public.ps1 | 4 ++-- malboxes/templates/snippets/provision_powershell.json | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/malboxes/scripts/windows/allow-WinRM-public.ps1 b/malboxes/scripts/windows/allow-WinRM-public.ps1 index f3180f0..74b7901 100644 --- a/malboxes/scripts/windows/allow-WinRM-public.ps1 +++ b/malboxes/scripts/windows/allow-WinRM-public.ps1 @@ -1,2 +1,2 @@ -# Change the network types of the WinRM rules for public -netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new profile=public \ No newline at end of file +# Allow WinRM to communicate on public network +netsh advfirewall firewall add rule name="Public network WinRM" dir=in action=allow protocol=TCP localport=5985 profile=public \ No newline at end of file diff --git a/malboxes/templates/snippets/provision_powershell.json b/malboxes/templates/snippets/provision_powershell.json index 1786ce2..aef8966 100644 --- a/malboxes/templates/snippets/provision_powershell.json +++ b/malboxes/templates/snippets/provision_powershell.json @@ -1,6 +1,7 @@ { "type": "powershell", "scripts": [ + "{{ dir }}/scripts/windows/allow-WinRM-public.ps1", {% if not windows_updates == "true" %}"{{ dir }}/scripts/windows/disable_auto-updates.ps1",{% endif %} {% if not windows_defender == "true" %}"{{ dir }}/scripts/windows/disable_defender.ps1",{% endif %} {% if hypervisor == "virtualbox" %} @@ -8,8 +9,8 @@ {% endif %} "{{ dir }}/scripts/windows/installtools.ps1", {% if profile is defined %}"{{ cache_dir }}/profile-{{ profile }}.ps1",{% endif %} - "{{ dir }}/scripts/windows/malware_analysis.ps1", - "{{ dir }}/scripts/windows/allow-WinRM-public.ps1" + "{{ dir }}/scripts/windows/malware_analysis.ps1" + ] } {% if choco_packages %}, From 3421419e514dd1c9756ab3abd637406dcded5fe7 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 20 Feb 2019 14:34:11 -0500 Subject: [PATCH 15/30] AWS support for win7 32-bit Windows 7 32-bit can now be run on AWS --- malboxes/templates/win7_32_analyst.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/malboxes/templates/win7_32_analyst.json b/malboxes/templates/win7_32_analyst.json index b5b2d79..31e7ed1 100644 --- a/malboxes/templates/win7_32_analyst.json +++ b/malboxes/templates/win7_32_analyst.json @@ -23,7 +23,11 @@ ] }], - {% include 'snippets/postprocessor_vagrant.json' %}, + {% if hypervisor == 'virtualbox' %} + {% include 'snippets/postprocessor_vagrant.json' %}, + {% elif hypervisor == 'aws' %} + {% include 'snippets/postprocessor_aws.json' %}, + {% endif %} "provisioners": [ From ffde7814409cb47118ed4cbbf28eaa7542b2aa3f Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 20 Feb 2019 14:35:55 -0500 Subject: [PATCH 16/30] README and minor corrections It is now explained how to use the AWS feature of Malboxes in the README. --- README.adoc | 54 +++++++++++++++++++ malboxes/config-example.js | 6 +-- malboxes/scripts/windows/malware_analysis.ps1 | 2 +- malboxes/templates/win10_32_analyst.json | 2 - 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/README.adoc b/README.adoc index 00a4f69..0a4082e 100644 --- a/README.adoc +++ b/README.adoc @@ -26,6 +26,7 @@ https://github.com/gosecure/malboxes * vagrant: https://www.vagrantup.com/downloads.html * https://www.virtualbox.org/wiki/Downloads[VirtualBox] or an vSphere / ESXi server + === Minimum specs for the build machine * At least 5 GB of RAM @@ -40,6 +41,12 @@ https://github.com/gosecure/malboxes apt install vagrant git python3-pip +=== AWS +Run this command after normal installation: + + vagrant plugin install vagrant-aws + +NOTE: The AWS feature has only been tested on Linux for the moment and EC2 does not support 32-bit desktop version of Windows 10. == Installation @@ -136,6 +143,50 @@ For example: malboxes spin win7_32_analyst 20160519.cryptolocker.xyz +=== AWS + +Malboxes can upload and interact with a VM on the Amazon Web serivces. To do so, follow these steps: + +. Malboxes will need a S3 bucket on AWS to upload the VM before converting it to an AMI (Amazon Machine Image). If you don't have one, +link:https://docs.aws.amazon.com/quickstarts/latest/s3backup/step-1-create-bucket.html[create one now.] + +. Your instance also requires a link:https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#CreatingSecurityGroups[security group] with at least a rule allowing inbound connections for WinRM (Type: WinRM-HTTP, Protocol: TCP, Port Range: 5985, Source: host's public IP). + +. If the default config is used, change the hypervisor to aws and fill the mandatory options related. Otherwise, be sure to add all the options about AWS to your custom config. + +. Finally, you can follow the same steps described in the <> and the <> sections to launch your instance! + +NOTE: The AMI import can take a very long time (about an hour), however you can verify the status of the task by doing <>. At the moment, only one AMI can be build per template. + +==== AMI import status +Install awscli using pip: + + pip install awscli + +link:https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration[Configure] awscli with: + + aws configure + +Then run: + + aws ec2 describe-import-image-tasks + +==== RDP + +To connect to an instance on the cloud using RDP, run this command at the same location of your `+Vagrantfile+`: + + vagrant rdp -- /cert-ignore + +For this to work, the instance will require a security group allowing RDP inbound connections (Type: RDP, Protocol: TCP, Port Range: 3389, Source: host's public IP). + +NOTE: You can safely ignore the following error because rsync is not yet implemented: << No host IP was given to the Vagrant core NFS helper. This is an internal error that should be reported as a bug. >> + + +==== Stopping an Instance + +To stop an instance on the cloud, run this command at the same location of your `+Vagrantfile+`: + + vagrant halt == Configuration @@ -167,6 +218,9 @@ link:malboxes/profile-example.js[profile-example.js] for an example configuration. This new capacity is experimental and subject to change as we experiment with it. +=== AWS security groups + +Currently, Malboxes does not support the automatic creation of the security groups, so you'll have to use the AWS console to create yours. However, using the library link:https://boto3.amazonaws.com/v1/documentation/api/latest/index.html[Boto3] there should be a way to implement this. == More information diff --git a/malboxes/config-example.js b/malboxes/config-example.js index 25d9226..3732f13 100644 --- a/malboxes/config-example.js +++ b/malboxes/config-example.js @@ -32,7 +32,7 @@ //"input_locale": "fr-FR", // Provision settings - // Which Hypervisor for privisoning and deployment? (Options are: "virtualbox" and "vsphere") Default is "virtualbox" + // Which Hypervisor for privisoning and deployment? (Options are: "virtualbox", "vsphere" and "aws") Default is "virtualbox" "hypervisor": "virtualbox", // If vsphere, the following configuration options are mandatory "remote_host": "", @@ -45,13 +45,13 @@ "vsphere_user": "", "vsphere_password": "", "vsphere_insecure": "true", - // If AWS, the following configuration options are mandatory + // If AWS, the following configuration options are mandatory "aws_access_key": "", "aws_secret_key": "", "aws_s3_bucket": "", "aws_keypair": "", + "aws_security_group": "", // See Usage/AWS in doc to understand why you need to create one. // Optional - "aws_security_group": "default", "aws_region": "us-east-1", "aws_instance_type" : "m3.medium", diff --git a/malboxes/scripts/windows/malware_analysis.ps1 b/malboxes/scripts/windows/malware_analysis.ps1 index 4bcaa1d..6b3343b 100644 --- a/malboxes/scripts/windows/malware_analysis.ps1 +++ b/malboxes/scripts/windows/malware_analysis.ps1 @@ -5,4 +5,4 @@ Set-ItemProperty -Path "HKLM:\System\CurrentControlSet\Control\Terminal Server" netsh advfirewall firewall set rule group="remote desktop" new enable=Yes # IDA Remote Debugging -netsh advfirewall firewall add rule name="IDA Remote Debugging" dir=in action=allow protocol=TCP localport=23946 +netsh advfirewall firewall add rule name="IDA Remote Debugging" dir=in action=allow protocol=TCP localport=23946 \ No newline at end of file diff --git a/malboxes/templates/win10_32_analyst.json b/malboxes/templates/win10_32_analyst.json index 689b936..5b79623 100644 --- a/malboxes/templates/win10_32_analyst.json +++ b/malboxes/templates/win10_32_analyst.json @@ -22,8 +22,6 @@ ] }], - {% include 'snippets/postprocessor_vagrant.json' %}, - {% if hypervisor == 'virtualbox' %} {% include 'snippets/postprocessor_vagrant.json' %}, {% endif %} From e795a5fef042a6cdc19bb2f30b9de568a5821378 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 11:04:23 -0500 Subject: [PATCH 17/30] Correction of README General corrections of the README requested by obilodeau. --- README.adoc | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/README.adoc b/README.adoc index 0a4082e..697e19d 100644 --- a/README.adoc +++ b/README.adoc @@ -41,13 +41,6 @@ https://github.com/gosecure/malboxes apt install vagrant git python3-pip -=== AWS -Run this command after normal installation: - - vagrant plugin install vagrant-aws - -NOTE: The AWS feature has only been tested on Linux for the moment and EC2 does not support 32-bit desktop version of Windows 10. - == Installation === Linux/Unix @@ -58,7 +51,6 @@ NOTE: The AWS feature has only been tested on Linux for the moment and EC2 does + sudo pip3 install git+https://github.com/GoSecure/malboxes.git#egg=malboxes - === Windows NOTE: Starting with Windows 10 Hyper-V is always running below the operating @@ -100,6 +92,12 @@ installed. Otherwise, follow the <>. pip3 install setuptools pip3 install -U git+https://github.com/GoSecure/malboxes.git#egg=malboxes +=== To deploy on AWS (optional) +Run this command after normal installation: + + vagrant plugin install vagrant-aws + +NOTE: The AWS feature has only been tested on Linux for the moment and EC2 does not support 32-bit desktop version of Windows 10. == Usage @@ -143,7 +141,7 @@ For example: malboxes spin win7_32_analyst 20160519.cryptolocker.xyz -=== AWS +=== To deploy on AWS (optional) Malboxes can upload and interact with a VM on the Amazon Web serivces. To do so, follow these steps: @@ -152,7 +150,7 @@ link:https://docs.aws.amazon.com/quickstarts/latest/s3backup/step-1-create-bucke . Your instance also requires a link:https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#CreatingSecurityGroups[security group] with at least a rule allowing inbound connections for WinRM (Type: WinRM-HTTP, Protocol: TCP, Port Range: 5985, Source: host's public IP). -. If the default config is used, change the hypervisor to aws and fill the mandatory options related. Otherwise, be sure to add all the options about AWS to your custom config. +. If the <<_configuration,default config>> is used, change the hypervisor to aws and fill the mandatory options related. Otherwise, be sure to add all the options about AWS to your custom config. . Finally, you can follow the same steps described in the <> and the <> sections to launch your instance! @@ -173,18 +171,18 @@ Then run: ==== RDP -To connect to an instance on the cloud using RDP, run this command at the same location of your `+Vagrantfile+`: +To connect to an instance on the cloud using RDP, run this command at the same location of your `Vagrantfile`: vagrant rdp -- /cert-ignore For this to work, the instance will require a security group allowing RDP inbound connections (Type: RDP, Protocol: TCP, Port Range: 3389, Source: host's public IP). -NOTE: You can safely ignore the following error because rsync is not yet implemented: << No host IP was given to the Vagrant core NFS helper. This is an internal error that should be reported as a bug. >> +NOTE: You can safely ignore the following error because rsync is not yet implemented: `No host IP was given to the Vagrant core NFS helper. This is an internal error that should be reported as a bug.` ==== Stopping an Instance -To stop an instance on the cloud, run this command at the same location of your `+Vagrantfile+`: +To stop an instance on the cloud, run this command at the same location of your `Vagrantfile`: vagrant halt From a114e2a46215ed1588a3f9b7129a7bd5ecd718ed Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 11:09:59 -0500 Subject: [PATCH 18/30] =?UTF-8?q?Imports=E2=80=99=20order=20adjusted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The imports now respect the PEP 8. --- malboxes/malboxes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index dfdc1d5..d5e16f0 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -31,11 +31,10 @@ import textwrap from appdirs import AppDirs +import boto3 from jinja2 import Environment, FileSystemLoader from jsmin import jsmin -import boto3 - from malboxes._version import __version__ DIRS = AppDirs("malboxes") From fad539e6aef15cbce8aada59b6112d38f1db227e Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 11:14:00 -0500 Subject: [PATCH 19/30] Constants for exit status codes Constants were added to describe the exit status codes. --- malboxes/malboxes.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index d5e16f0..b5c4d07 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -39,6 +39,12 @@ DIRS = AppDirs("malboxes") DEBUG = False +EXIT_WITHOUT_ERROR = 1 +EXIT_TEMPLATE_NOT_FOUND = 2 +EXIT_PACKER_FAILED = 3 +EXIT_VAGRANT_BOX_ADD_FAILED = 4 +EXIT_VAGRANTFILE_ALREADY_EXISTS = 5 +EXIT_TEMPLATE_ALREADY_AMI = 6 def initialize(): # create appdata directories if they don't exist @@ -143,7 +149,7 @@ def prepare_packer_template(config, template_name): 'templates/{}.json'.format(template_name)) except FileNotFoundError: print("Template doesn't exist: {}".format(template_name)) - sys.exit(1) + sys.exit(EXIT_TEMPLATE_NOT_FOUND) filepath = resource_filename(__name__, 'templates/') env = Environment(loader=FileSystemLoader(filepath), autoescape=False, @@ -387,7 +393,7 @@ def default(parser, args): parser.print_help() print("\n") list_templates(parser, args) - sys.exit(2) + sys.exit(EXIT_WITHOUT_ERROR) def list_templates(parser, args): @@ -449,7 +455,7 @@ def build(parser, args): Exiting... ===============================================================""") .format(args.template, DIRS.user_cache_dir)) - sys.exit(4) + sys.exit(EXIT_TEMPLATE_ALREADY_AMI) if not args.skip_packer_build: ret = run_packer(packer_tmpl, args) @@ -458,7 +464,7 @@ def build(parser, args): if ret != 0: print("Packer failed. Build failed. Exiting...") - sys.exit(3) + sys.exit(EXIT_PACKER_FAILED) if not (args.skip_vagrant_box_add or config['hypervisor'] == 'aws'): ret = add_box(config, args) @@ -467,7 +473,7 @@ def build(parser, args): if ret != 0: print("'vagrant box add' failed. Build failed. Exiting...") - sys.exit(5) + sys.exit(EXIT_VAGRANT_BOX_ADD_FAILED) if config['hypervisor'] == 'aws': print(textwrap.dedent(""" @@ -510,7 +516,7 @@ def spin(parser, args): """ if os.path.isfile('Vagrantfile'): print("Vagrantfile already exists. Please move it away. Exiting...") - sys.exit(6) + sys.exit(VAGRANTFILE_ALREADY_EXISTS) config, _ = prepare_config(args) From d69d31841bf227e99e4855e1b76f56db720c71ad Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 11:21:01 -0500 Subject: [PATCH 20/30] Refactoring of is_template_already_ami The function now follows the DRY principle and has a more precise name. --- malboxes/malboxes.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index b5c4d07..8c0de76 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -425,16 +425,17 @@ def get_AMI_ID_by_template(config, template): return images['Images'][0]['ImageId'] def template_already_AMI(config, template): +def is_template_already_AMI(config, template): """ Verifies if there's already an AMI based on a template. If so, returns True. - Otherwise, returns False + Otherwise, returns False. """ - images = create_EC2_client(config).describe_images(Owners=['self'],Filters=[{'Name': 'tag:Template', 'Values': [template]}]) - if not images['Images']: + try: + get_AMI_ID_by_template(config, template) + except IndexError: return False - else: - return True + return True def build(parser, args): print("Generating configuration files...") From 2f0ccaba202fb610024f169f8a3db9712d7240d0 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 11:34:31 -0500 Subject: [PATCH 21/30] Length of lines is now below 80 chars Each line of the code related to AWS has now 80 chars or less. --- malboxes/malboxes.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index 8c0de76..31aca02 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -421,10 +421,11 @@ def get_AMI_ID_by_template(config, template): """ Gets the ID of an AMI by the template tag on it. """ - images = create_EC2_client(config).describe_images(Owners=['self'],Filters=[{'Name': 'tag:Template', 'Values': [template]}]) + images = create_EC2_client(config).describe_images(Owners=['self'], + Filters=[{'Name': 'tag:Template', 'Values': [template]}]) return images['Images'][0]['ImageId'] -def template_already_AMI(config, template): + def is_template_already_AMI(config, template): """ Verifies if there's already an AMI based on a template. @@ -443,7 +444,9 @@ def build(parser, args): prepare_autounattend(config) _prepare_vagrantfile(config, "box_win.rb", create_cachefd('box_win.rb')) print("Configuration files are ready") - if config['hypervisor'] == 'aws' and template_already_AMI(config, args.template): + if ( config['hypervisor'] == 'aws' and + is_template_already_AMI(config, args.template) + ): print(textwrap.dedent(""" =============================================================== This template has already been converted to an AMI. @@ -533,7 +536,8 @@ def spin(parser, args): _prepare_vagrantfile(config, "analyst_vsphere.rb", f) elif config['hypervisor'] == 'aws': with open("Vagrantfile", 'w') as f: - config['aws_ami_id'] = get_AMI_ID_by_template(config, config['template']) + config['aws_ami_id'] = get_AMI_ID_by_template(config, + config['template']) _prepare_vagrantfile(config, "analyst_aws.rb", f) print("Vagrantfile generated. You can move it in your analysis directory " "and issue a `vagrant up` to get started with your VM.") From 03d84a91226cca555b20be0640e9a4419179948f Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 11:37:39 -0500 Subject: [PATCH 22/30] AWS simpler build message Details about the vagrant box were removed in the build message because the user does not need to know them. --- malboxes/malboxes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index 31aca02..5a9cdba 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -484,9 +484,6 @@ def build(parser, args): =============================================================== The AMI was successfully created on the Amazon Elastic Compute Cloud. - The required box will be added to Vagrant, if it's not already there, - with the spin command. - You should generate a Vagrantfile configuration in order to launch an instance of the AMI: From 95a802dda37d6a1b804b708de35f4cd1af8f1393 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 11:45:08 -0500 Subject: [PATCH 23/30] New lines between functions Now there's two new lines between functions to respect the PEP 8. --- malboxes/malboxes.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index 5a9cdba..41fb437 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -66,6 +66,7 @@ def initialize(): return init_parser() + def init_parser(): parser = argparse.ArgumentParser( description="Vagrant box builder " @@ -405,6 +406,7 @@ def list_templates(parser, args): print(m.group(1)) print() + def create_EC2_client(config): """ Creates a client to interact with Amazon Elastic Compute Cloud. @@ -417,6 +419,7 @@ def create_EC2_client(config): region_name=config['aws_region'], ) + def get_AMI_ID_by_template(config, template): """ Gets the ID of an AMI by the template tag on it. @@ -438,6 +441,7 @@ def is_template_already_AMI(config, template): return False return True + def build(parser, args): print("Generating configuration files...") config, packer_tmpl = prepare_config(args) @@ -510,7 +514,8 @@ def build(parser, args): spin`. Each VM will be independent of each other. ===============================================================""") .format(args.template, DIRS.user_cache_dir)) - + + def spin(parser, args): """ Creates a Vagrantfile meant for user-interaction in the current directory. @@ -539,6 +544,7 @@ def spin(parser, args): print("Vagrantfile generated. You can move it in your analysis directory " "and issue a `vagrant up` to get started with your VM.") + def prepare_profile(template, config): """Converts the profile to a powershell script.""" @@ -664,6 +670,7 @@ def document(profile_name, modtype, docpath, fd): fd.write(line) + def shortcut_function(fd): """ Add shortcut function to the profile """ filename = resource_filename(__name__, "scripts/windows/add-shortcut.ps1") @@ -671,6 +678,7 @@ def shortcut_function(fd): fd.write(add_shortcut_file.read()) add_shortcut_file.close(); + def shortcut(dest, target, arguments, fd): """ Create shortcut on Desktop """ if arguments is None: @@ -681,6 +689,7 @@ def shortcut(dest, target, arguments, fd): print("Adding shortcut {}: {} with arguments {}".format(dest, target, arguments)) fd.write(line) + def main(): global DEBUG try: From 13460aa0249e4ac5aeb77da72a81cca818bd78f7 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 13:35:50 -0500 Subject: [PATCH 24/30] Fix to not affect ESXI The postprocessor_vagrant is now included for all the hypervisors except aws. --- malboxes/templates/win10_32_analyst.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/malboxes/templates/win10_32_analyst.json b/malboxes/templates/win10_32_analyst.json index 5b79623..920c2d6 100644 --- a/malboxes/templates/win10_32_analyst.json +++ b/malboxes/templates/win10_32_analyst.json @@ -22,7 +22,7 @@ ] }], - {% if hypervisor == 'virtualbox' %} + {% if hypervisor != 'aws' %} {% include 'snippets/postprocessor_vagrant.json' %}, {% endif %} From f287550a46e544486a642db07dde49eaec13c3df Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Wed, 27 Feb 2019 14:30:09 -0500 Subject: [PATCH 25/30] Fix forgotten name for an exit status VAGRANT_FILE_ALREADY_EXISTS is now EXIT_VAGRANT_FILE_ALREADY_EXISTS --- malboxes/malboxes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/malboxes/malboxes.py b/malboxes/malboxes.py index 41fb437..9e73b0a 100644 --- a/malboxes/malboxes.py +++ b/malboxes/malboxes.py @@ -522,7 +522,7 @@ def spin(parser, args): """ if os.path.isfile('Vagrantfile'): print("Vagrantfile already exists. Please move it away. Exiting...") - sys.exit(VAGRANTFILE_ALREADY_EXISTS) + sys.exit(EXIT_VAGRANTFILE_ALREADY_EXISTS) config, _ = prepare_config(args) From 8077beee2446c19b56c39c0c68f25032ebcf8186 Mon Sep 17 00:00:00 2001 From: Etienne Lacroix Date: Thu, 28 Feb 2019 11:21:29 -0500 Subject: [PATCH 26/30] ESXI correction for windows10 32bits Finally, the postprocessor_vagrant.json was not required at all by the vsphere hypervisor. --- malboxes/templates/win10_32_analyst.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/malboxes/templates/win10_32_analyst.json b/malboxes/templates/win10_32_analyst.json index 920c2d6..5b79623 100644 --- a/malboxes/templates/win10_32_analyst.json +++ b/malboxes/templates/win10_32_analyst.json @@ -22,7 +22,7 @@ ] }], - {% if hypervisor != 'aws' %} + {% if hypervisor == 'virtualbox' %} {% include 'snippets/postprocessor_vagrant.json' %}, {% endif %} From fa71f3367aa6217926c12041c2a4fdea440035ee Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Wed, 14 Aug 2019 17:26:18 -0400 Subject: [PATCH 27/30] Added boto3 requirement to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 42dca75..6b774a6 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,7 @@ def _teardown(): # your project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html - install_requires=['appdirs', 'Jinja2', 'jsmin'], + install_requires=['appdirs', 'Jinja2', 'jsmin', 'boto3'], # List additional groups of dependencies here (e.g. development # dependencies). You can install these using the following syntax, From 2911d153b9d72017225e4b6bdc2a15ad4068bde4 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Thu, 22 Aug 2019 14:33:40 -0400 Subject: [PATCH 28/30] doc: added a missing step in the guide --- README.adoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.adoc b/README.adoc index 697e19d..05eef4e 100644 --- a/README.adoc +++ b/README.adoc @@ -150,6 +150,10 @@ link:https://docs.aws.amazon.com/quickstarts/latest/s3backup/step-1-create-bucke . Your instance also requires a link:https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#CreatingSecurityGroups[security group] with at least a rule allowing inbound connections for WinRM (Type: WinRM-HTTP, Protocol: TCP, Port Range: 5985, Source: host's public IP). +. Next, you need a `vmimport` service role configured. + Follow the section named _VM Import Service Role_ of https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html[this guide]. + These steps must be performed with an account that has `iam:CreateRole` and `iam:PutRolePolicy` permissions. + . If the <<_configuration,default config>> is used, change the hypervisor to aws and fill the mandatory options related. Otherwise, be sure to add all the options about AWS to your custom config. . Finally, you can follow the same steps described in the <> and the <> sections to launch your instance! From a40b989015c1fb52b4e7d1daea482bd7f32e3c45 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Tue, 27 Aug 2019 16:09:25 -0400 Subject: [PATCH 29/30] aws: prevent rsync failures on "vagrant up" --- malboxes/vagrantfiles/analyst_aws.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/malboxes/vagrantfiles/analyst_aws.rb b/malboxes/vagrantfiles/analyst_aws.rb index 2c67ac8..697da9f 100644 --- a/malboxes/vagrantfiles/analyst_aws.rb +++ b/malboxes/vagrantfiles/analyst_aws.rb @@ -27,4 +27,7 @@ 'Template' => "{{ template_name }}" } end + + # Disable the default synced folder (vagrant-aws only supports rsync and our images don't) + config.vm.synced_folder ".", "/vagrant", disabled: true end \ No newline at end of file From aafec654eada423dde0cabff2ec42336f4d40ee0 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Tue, 27 Aug 2019 16:29:16 -0400 Subject: [PATCH 30/30] docs: some rough notes on AWS support --- docs/aws.adoc | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docs/aws.adoc diff --git a/docs/aws.adoc b/docs/aws.adoc new file mode 100644 index 0000000..8a610d1 --- /dev/null +++ b/docs/aws.adoc @@ -0,0 +1,32 @@ += Amazon Web Services (AWS) Helpers + +== Setup + +Install the `aws` command-line tool (pip or your OS' version) + +Configure it + + aws configure + +You are good to go! + + +== Built Templates + +They are available as images or AMIs in AWS' language. +You can find them under EC2 -> Images -> AMIs. + +=== List malboxes images hosted available to you + + aws ec2 describe-images --filters "Name=tag:Name,Values=Malboxes" + + +== Operations + +=== Map a drive + +Connecting to the instance with the following command will map the current +working directory as a drive letter on the Windows box using RDP: + + vagrant rdp -- /cert-ignore /a:drive,home,./ +