Slim handler

README.md

@@ -480,6 +480,7 @@ to change Zappa's behavior. Use these at your own risk!
         "remote_env": "s3://my-project-config-files/filename.json", // optional file in s3 bucket containing a flat json object which will be used to set custom environment variables.
         "role_name": "MyLambdaRole", // Name of Zappa execution role. Default ZappaExecutionRole. To use a different, pre-existing policy, you must also set manage_roles to false.
         "s3_bucket": "dev-bucket", // Zappa zip bucket,
+        "slim_handler": false, // Useful if project >50M. Set true to just upload a small handler to Lambda and load actual project from S3 at runtime.
         "settings_file": "~/Projects/MyApp/settings/dev_settings.py", // Server side settings file location,
         "timeout_seconds": 30, // Maximum lifespan for the Lambda function (default 30, max 300.)
         "touch": false, // GET the production URL upon initial deployment (default True)
@@ -543,6 +544,11 @@ The easiest way to enable CORS (Cross-Origin Resource Sharing) for in your Zappa
 
 You can also simply handle CORS directly in your application. If you do this, you'll need to add `Access-Control-Allow-Origin`, `Access-Control-Allow-Headers`, and `Access-Control-Allow-Methods` to the `method_header_types` key in your `zappa_settings.json`. See further [discussion here](https://github.com/Miserlou/Zappa/issues/41).
 
+#### Large Projects
+
+AWS currently limits Lambda zip sizes to 50 meg. If the project is larger than that, set the `slim_handler=true` in the zappa_settings.json. This just gives Lambda a small handler file. The handler file then pulls the large project down from S3 at run time. The initial load of the large project may add to runtime, but the difference should be minimal on a warm lambda function.
+
+
 #### Enabling Bash Completion
 
 Bash completion can be enabled by adding the following to your .bashrc:

test_settings.json

@@ -58,6 +58,11 @@
        "extends": "extendo",
        "s3_bucket": "lmbda2",
     },
+    "slim_handler": {
+       "extends": "ttt888",
+       "slim_handler": true,
+       "delete_local_zip": true
+    },
     "depricated_remote_env": {
        "s3_bucket": "lmbda",
        "remote_env_bucket": "lmbda-env",

tests/tests.py

@@ -1376,5 +1376,16 @@ def test_flask_logging_bug(self):
         finally:
             sys.stderr = old_stderr
 
+    def test_slim_handler(self):
+        zappa_cli = ZappaCLI()
+        zappa_cli.api_stage = 'slim_handler'
+        zappa_cli.load_settings('test_settings.json')
+        zappa_cli.create_package()
+
+        self.assertTrue(os.path.isfile(zappa_cli.handler_path))
+        self.assertTrue(os.path.isfile(zappa_cli.zip_path))
+
+        zappa_cli.remove_local_zip()
+
 if __name__ == '__main__':
     unittest.main()

zappa/cli.py

@@ -96,6 +96,7 @@ class ZappaCLI(object):
     s3_bucket_name = None
     settings_file = None
     zip_path = None
+    handler_path = None
     vpc_config = None
     memory_size = None
     use_apigateway = None
@@ -540,6 +541,24 @@ def deploy(self):
         if not success: # pragma: no cover
             raise ClickException("Unable to upload to S3. Quitting.")
 
+        # If using a slim handler, upload it to S3 and tell lambda to use this slim handler zip
+        if self.stage_config.get('slim_handler', False):
+            # https://github.com/Miserlou/Zappa/issues/510
+            success = self.zappa.upload_to_s3(self.handler_path, self.s3_bucket_name)
+            if not success:  # pragma: no cover
+                raise ClickException("Unable to upload handler to S3. Quitting.")
+
+            # Copy the project zip to the current project zip
+            current_project_name = '{0!s}_current_project.zip'.format(self.project_name)
+            success = self.zappa.copy_on_s3(src_file_name=self.zip_path, dst_file_name=current_project_name,
+                                            bucket_name=self.s3_bucket_name)
+            if not success:  # pragma: no cover
+                raise ClickException("Unable to copy the zip to be the current project. Quitting.")
+
+            handler_file = self.handler_path
+        else:
+            handler_file = self.zip_path
+
 
         # Fixes https://github.com/Miserlou/Zappa/issues/613
         try:
@@ -550,7 +569,7 @@ def deploy(self):
             # You'll also need to define the path to your lambda_handler code.
             self.lambda_arn = self.zappa.create_lambda_function(
                 bucket=self.s3_bucket_name,
-                s3_key=self.zip_path,
+                s3_key=handler_file,
                 function_name=self.lambda_name,
                 handler=self.lambda_handler,
                 description=self.lambda_description,
@@ -597,9 +616,8 @@ def deploy(self):
         if self.stage_config.get('delete_local_zip', True):
             self.remove_local_zip()
 
-        # Remove the uploaded zip from S3, because it is now registered..
-        if self.stage_config.get('delete_s3_zip', True):
-            self.zappa.remove_from_s3(self.zip_path, self.s3_bucket_name)
+        # Remove the project zip from S3.
+        self.remove_uploaded_zip()
 
         self.callback('post')
 
@@ -650,18 +668,34 @@ def update(self):
 
         # Upload it to S3
         success = self.zappa.upload_to_s3(self.zip_path, self.s3_bucket_name)
-        if not success: # pragma: no cover
-            print("Unable to upload to S3. Quitting.")
-            sys.exit(-1)
+        if not success:  # pragma: no cover
+            raise ClickException("Unable to upload project to S3. Quitting.")
+
+        # If using a slim handler, upload it to S3 and tell lambda to use this slim handler zip
+        if self.stage_config.get('slim_handler', False):
+            # https://github.com/Miserlou/Zappa/issues/510
+            success = self.zappa.upload_to_s3(self.handler_path, self.s3_bucket_name)
+            if not success:  # pragma: no cover
+                raise ClickException("Unable to upload handler to S3. Quitting.")
+
+            # Copy the project zip to the current project zip
+            current_project_name = '{0!s}_current_project.zip'.format(self.project_name)
+            success = self.zappa.copy_on_s3(src_file_name=self.zip_path, dst_file_name=current_project_name,
+                                            bucket_name=self.s3_bucket_name)
+            if not success:  # pragma: no cover
+                raise ClickException("Unable to copy the zip to be the current project. Quitting.")
+
+            handler_file = self.handler_path
+        else:
+            handler_file = self.zip_path
 
         # Register the Lambda function with that zip as the source
         # You'll also need to define the path to your lambda_handler code.
         self.lambda_arn = self.zappa.update_lambda_function(
-            self.s3_bucket_name, self.zip_path, self.lambda_name)
+            self.s3_bucket_name, handler_file, self.lambda_name)
 
         # Remove the uploaded zip from S3, because it is now registered..
-        if self.stage_config.get('delete_s3_zip', True):
-            self.zappa.remove_from_s3(self.zip_path, self.s3_bucket_name)
+        self.remove_uploaded_zip()
 
         # Update the configuration, in case there are changes.
         self.lambda_arn = self.zappa.update_lambda_configuration(lambda_arn=self.lambda_arn,
@@ -1426,8 +1460,8 @@ def callback(self, position):
                 raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style(
                     "find {position} callback ".format(position=position), bold=True) + 'function: "{cb_func_name}" '.format(
                     cb_func_name=click.style(cb_func_name, bold=True)) + 'in module "{mod_path}"'.format(mod_path=mod_path))
-            
-            
+
+
             cb_func = getattr(module_, cb_func_name)
             cb_func(self) # Call the function passing self
 
@@ -1621,9 +1655,31 @@ def create_package(self):
             inspect.getfile(inspect.currentframe())))
         handler_file = os.sep.join(current_file.split(os.sep)[0:]) + os.sep + 'handler.py'
 
-        # Create the zip file
-        self.zip_path = self.zappa.create_lambda_zip(
-                self.lambda_name,
+        # Create the zip file(s)
+        if self.stage_config.get('slim_handler', False):
+            # Create two zips. One with the application and the other with just the handler.
+            # https://github.com/Miserlou/Zappa/issues/510
+            self.zip_path = self.zappa.create_lambda_zip(
+                prefix=self.lambda_name,
+                use_precompiled_packages=self.stage_config.get('use_precompiled_packages', True),
+                exclude=self.stage_config.get('exclude', [])
+            )
+
+            # Make sure the normal venv is not included in the handler's zip
+            exclude = self.stage_config.get('exclude', [])
+            cur_venv = self.zappa.get_current_venv()
+            exclude.append(cur_venv.split('/')[-1])
+            self.handler_path = self.zappa.create_lambda_zip(
+                prefix='handler_{0!s}'.format(self.lambda_name),
+                venv=self.zappa.create_handler_venv(),
+                handler_file=handler_file,
+                slim_handler=True,
+                exclude=exclude
+            )
+        else:
+            # Create a single zip that has the handler and application
+            self.zip_path = self.zappa.create_lambda_zip(
+                prefix=self.lambda_name,
                 handler_file=handler_file,
                 use_precompiled_packages=self.stage_config.get('use_precompiled_packages', True),
                 exclude=self.stage_config.get(
@@ -1632,9 +1688,19 @@ def create_package(self):
                     # https://github.com/Miserlou/Zappa/issues/556
                     ["boto3", "dateutil", "botocore", "s3transfer", "six.py", "jmespath", "concurrent"])
             )
+            # Warn if this is too large for Lambda.
+            file_stats = os.stat(self.zip_path)
+            if file_stats.st_size > 52428800:  # pragma: no cover
+                print('\n\nWarning: Application zip package is likely to be too large for AWS Lambda.'
+                      'Try setting "slim_handler"=true in the zappa_settings.json.\n\n')
+
+        # Throw custom setings into the zip that handles requests
+        if self.stage_config.get('slim_handler', False):
+            handler_zip = self.handler_path
+        else:
+            handler_zip = self.zip_path
 
-        # Throw custom setings into the zip file
-        with zipfile.ZipFile(self.zip_path, 'a') as lambda_zip:
+        with zipfile.ZipFile(handler_zip, 'a') as lambda_zip:
 
             settings_s = "# Generated by Zappa\n"
 
@@ -1706,6 +1772,10 @@ def create_package(self):
             else:
                 settings_s = settings_s + "DJANGO_SETTINGS=None\n"
 
+            # If slim handler, path to project zip
+            if self.stage_config.get('slim_handler', False):
+                settings_s += "ZIP_PATH='s3://{0!s}/{1!s}_current_project.zip'\n".format(self.s3_bucket_name, self.project_name)
+
             # AWS Events function mapping
             event_mapping = {}
             events = self.stage_config.get('events', [])
@@ -1746,6 +1816,8 @@ def remove_local_zip(self):
             try:
                 if os.path.isfile(self.zip_path):
                     os.remove(self.zip_path)
+                if self.handler_path and os.path.isfile(self.handler_path):
+                    os.remove(self.handler_path)
             except Exception as e: # pragma: no cover
                 sys.exit(-1)
 
@@ -1757,6 +1829,10 @@ def remove_uploaded_zip(self):
         # Remove the uploaded zip from S3, because it is now registered..
         if self.stage_config.get('delete_s3_zip', True):
             self.zappa.remove_from_s3(self.zip_path, self.s3_bucket_name)
+            if self.stage_config.get('slim_handler', False):
+                # Need to keep the project zip as the slim handler uses it.
+                self.zappa.remove_from_s3(self.handler_path, self.s3_bucket_name)
+
 
     def on_exit(self):
         """
@@ -1916,7 +1992,7 @@ def execute_prebuild_script(self):
                 "find prebuild script ", bold=True) + 'function: "{pb_func}" '.format(
                 pb_func=click.style(pb_func, bold=True)) + 'in module "{pb_mod_path}"'.format(
                 pb_mod_path=pb_mod_path))
-        
+
         prebuild_function = getattr(module_, pb_func)
         prebuild_function()  # Call the function
 

zappa/handler.py

@@ -8,6 +8,7 @@
 import json
 import inspect
 import collections
+import zipfile
 
 import boto3
 import sys
@@ -112,6 +113,11 @@ def __init__(self, settings_name="zappa_settings", session=None):
             for key in self.settings.ENVIRONMENT_VARIABLES.keys():
                 os.environ[str(key)] = self.settings.ENVIRONMENT_VARIABLES[key]
 
+            # Pulling from S3 if given a zip path
+            project_zip_path = getattr(self.settings, 'ZIP_PATH', None)
+            if project_zip_path:
+                self.load_remote_project_zip(project_zip_path)
+
             # Django gets special treatment.
             if not self.settings.DJANGO_SETTINGS:
                 # The app module
@@ -133,6 +139,33 @@ def __init__(self, settings_name="zappa_settings", session=None):
 
             self.wsgi_app = ZappaWSGIMiddleware(wsgi_app_function)
 
+    def load_remote_project_zip(self, project_zip_path):
+        """
+        Puts the project files from S3 in /tmp and adds to path
+        """
+        project_folder = '/tmp/{0!s}'.format(self.settings.PROJECT_NAME)
+        if not os.path.isdir(project_folder):
+            # The project folder doesn't exist in this cold lambda, get it from S3
+            if not self.session:
+                boto_session = boto3.Session()
+            else:
+                boto_session = self.session
+
+            # Download the zip
+            remote_bucket, remote_file = project_zip_path.lstrip('s3://').split('/', 1)
+            s3 = boto_session.resource('s3')
+
+            zip_path = '/tmp/{0!s}'.format(remote_file)
+            s3.Object(remote_bucket, remote_file).download_file(zip_path)
+
+            # Unzip contents to project folder
+            with zipfile.ZipFile(zip_path, 'r') as z:
+                z.extractall(path=project_folder)
+
+        # Add to project path
+        sys.path.insert(0, project_folder)
+        return True
+
     def load_remote_settings(self, remote_bucket, remote_file):
         """
         Attempt to read a file from s3 containing a flat json object. Adds each