<commit_before>from __future__ import print_function

import boto3
import botocore
import glob
import json
import logging
import os
import random
import requests
import shutil
import string
import subprocess
import tarfile
import tempfile
import time
import troposphere
import troposphere.apigateway
import zipfile

from builtins import int, bytes
from botocore.exceptions import ClientError
from distutils.dir_util import copy_tree
from io import BytesIO, open
from lambda_packages import lambda_packages
from setuptools import find_packages
from tqdm import tqdm

# Zappa imports
from .utilities import (copytree,
                    add_event_source,
                    remove_event_source,
                    human_size,
                    get_topic_name,
                    contains_python_files_or_subdirs,
                    get_venv_from_python_version)

##
# Logging Config
##

logging.basicConfig(format='%(levelname)s:%(message)s')
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


##
# Policies And Template Mappings
##

ASSUME_POLICY = """{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Principal": {
        "Service": [
          "apigateway.amazonaws.com",
          "lambda.amazonaws.com",
          "events.amazonaws.com"
        ]
      },
      "Action": "sts:AssumeRole"
    }
  ]
}"""

ATTACH_POLICY = """{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "logs:*"
            ],
            "Resource": "arn:aws:logs:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "lambda:InvokeFunction"
            ],
            "Resource": [
                "*"
            ]
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:AttachNetworkInterface",
                "ec2:CreateNetworkInterface",
                "ec2:DeleteNetworkInterface",
                "ec2:DescribeInstances",
                "ec2:DescribeNetworkInterfaces",
                "ec2:DetachNetworkInterface",
                "ec2:ModifyNetworkInterfaceAttribute",
                "ec2:ResetNetworkInterfaceAttribute"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:*"
            ],
            "Resource": "arn:aws:s3:::*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "kinesis:*"
            ],
            "Resource": "arn:aws:kinesis:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "sns:*"
            ],
            "Resource": "arn:aws:sns:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "sqs:*"
            ],
            "Resource": "arn:aws:sqs:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "dynamodb:*"
            ],
            "Resource": "arn:aws:dynamodb:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "route53:*"
            ],
            "Resource": "*"
        }
    ]
}"""

# Latest list: https://docs.aws.amazon.com/general/latest/gr/rande.html#apigateway_region
API_GATEWAY_REGIONS = ['us-east-1', 'us-east-2',
                       'us-west-1', 'us-west-2',
                       'eu-central-1',
                       'eu-west-1', 'eu-west-2',
                       'ap-northeast-1', 'ap-northeast-2',
                       'ap-southeast-1', 'ap-southeast-2']

# Latest list: https://docs.aws.amazon.com/general/latest/gr/rande.html#lambda_region
LAMBDA_REGIONS = ['us-east-1', 'us-east-2',
                  'us-west-1', 'us-west-2',
                  'eu-central-1',
                  'eu-west-1', 'eu-west-2',
                  'ap-northeast-1', 'ap-northeast-2',
                  'ap-southeast-1', 'ap-southeast-2']

# We never need to include these.
# Related: https://github.com/Miserlou/Zappa/pull/56
# Related: https://github.com/Miserlou/Zappa/pull/581
ZIP_EXCLUDES = [
    '*.exe', '*.DS_Store', '*.Python', '*.git', '.git/*', '*.zip', '*.tar.gz',
    '*.hg', '*.egg-info', 'pip', 'docutils*', 'setuputils*'
]

##
# Classes
##

class Zappa(object):
    """
    Zappa!

    Makes it easy to run Python web applications on AWS Lambda/API Gateway.

    """

    ##
    # Configurables
    ##

    http_methods = [
        'ANY'
    ]
    role_name = "ZappaLambdaExecution"
    extra_permissions = None
    assume_policy = ASSUME_POLICY
    attach_policy = ATTACH_POLICY
    cloudwatch_log_levels = ['OFF', 'ERROR', 'INFO']

    ##
    # Credentials
    ##

    boto_session = None
    credentials_arn = None

    def __init__(self,
            boto_session=None,
            profile_name=None,
            aws_region=None,
            load_credentials=True,
            desired_role_name=None,
            runtime='python2.7'
        ):
        # Set aws_region to None to use the system's region instead
        if aws_region is None:
            # https://github.com/Miserlou/Zappa/issues/413
            self.aws_region = boto3.Session().region_name
            logger.debug("Set region from boto: %s", self.aws_region)
        else:
            self.aws_region = aws_region

        if desired_role_name:
            self.role_name = desired_role_name

        self.runtime = runtime

        if self.runtime == 'python2.7':
            self.manylinux_wheel_file_suffix = 'cp27mu-manylinux1_x86_64.whl'
        else:
            self.manylinux_wheel_file_suffix = 'cp36m-manylinux1_x86_64.whl'


        # Some common invokations, such as DB migrations,
        # can take longer than the default.

        # Note that this is set to 300s, but if connected to
        # APIGW, Lambda will max out at 30s.
        # Related: https://github.com/Miserlou/Zappa/issues/205
        long_config_dict = {
            'region_name': aws_region,
            'connect_timeout': 5,
            'read_timeout': 300
        }
        long_config = botocore.client.Config(**long_config_dict)

        if load_credentials:
            self.load_credentials(boto_session, profile_name)

        self.s3_client = self.boto_session.client('s3')
        self.lambda_client = self.boto_session.client('lambda', config=long_config)
        self.events_client = self.boto_session.client('events')
        self.apigateway_client = self.boto_session.client('apigateway')
        self.logs_client = self.boto_session.client('logs')
        self.iam_client = self.boto_session.client('iam')
        self.iam = self.boto_session.resource('iam')
        self.cloudwatch = self.boto_session.client('cloudwatch')
        self.route53 = self.boto_session.client('route53')
        self.sns_client = self.boto_session.client('sns')
        self.cf_client = self.boto_session.client('cloudformation')
        self.cf_template = troposphere.Template()
        self.cf_api_resources = []
        self.cf_parameters = {}

    def cache_param(self, value):
        '''Returns a troposphere Ref to a value cached as a parameter.'''

        if value not in self.cf_parameters:
            keyname = chr(ord('A') + len(self.cf_parameters))
            param = self.cf_template.add_parameter(troposphere.Parameter(
                keyname, Type="String", Default=value
            ))

            self.cf_parameters[value] = param

        return troposphere.Ref(self.cf_parameters[value])

    ##
    # Packaging
    ##
    def copy_editable_packages(self, egg_links, temp_package_path):
        """ """
        for egg_link in egg_links:
            with open(egg_link, 'rb') as df:
                egg_path = df.read().decode('utf-8').splitlines()[0].strip()
                pkgs = set([x.split(".")[0] for x in find_packages(egg_path, exclude=['test', 'tests'])])
                for pkg in pkgs:
                    copytree(os.path.join(egg_path, pkg), os.path.join(temp_package_path, pkg), symlinks=False)

        if temp_package_path:
            # now remove any egg-links as they will cause issues if they still exist
            for link in glob.glob(os.path.join(temp_package_path, "*.egg-link")):
                os.remove(link)

    def get_deps_list(self, pkg_name, installed_distros=None):
        """
        For a given package, returns a list of required packages. Recursive.
        """
        import pip
        deps = []
        if not installed_distros:
            installed_distros = pip.get_installed_distributions()
        for package in installed_distros:
            if package.project_name.lower() == pkg_name.lower():
                deps = [(package.project_name, package.version)]
                for req in package.requires():
                    deps += self.get_deps_list(pkg_name=req.project_name, installed_distros=installed_distros)
        return list(set(deps))  # de-dupe before returning

    def create_handler_venv(self):
        """
        Takes the installed zappa and brings it into a fresh virtualenv-like folder. All dependencies are then downloaded.
        """
        import pip

        # We will need the currenv venv to pull Zappa from
        current_venv = self.get_current_venv()

        # Make a new folder for the handler packages
        ve_path = os.path.join(os.getcwd(), 'handler_venv')

        if os.sys.platform == 'win32':
            current_site_packages_dir = os.path.join(current_venv, 'Lib', 'site-packages')
            venv_site_packages_dir = os.path.join(ve_path, 'Lib', 'site-packages')
        else:
            current_site_packages_dir = os.path.join(current_venv, 'lib', get_venv_from_python_version(), 'site-packages')
            venv_site_packages_dir = os.path.join(ve_path, 'lib', get_venv_from_python_version(), 'site-packages')

        if not os.path.isdir(venv_site_packages_dir):
            os.makedirs(venv_site_packages_dir)

        # Copy zappa* to the new virtualenv
        zappa_things = [z for z in os.listdir(current_site_packages_dir) if z.lower()[:5] == 'zappa']
        for z in zappa_things:
            copytree(os.path.join(current_site_packages_dir, z), os.path.join(venv_site_packages_dir, z))

        # Use pip to download zappa's dependencies. Copying from current venv causes issues with things like PyYAML that installs as yaml
        zappa_deps = self.get_deps_list('zappa')
        pkg_list = ['{0!s}=={1!s}'.format(dep, version) for dep, version in zappa_deps]

        # Need to manually add setuptools
        pkg_list.append('setuptools')
        pip.main(["install", "--quiet", "--target", venv_site_packages_dir] + pkg_list)

        return ve_path

    # staticmethod as per https://github.com/Miserlou/Zappa/issues/780
    @staticmethod
    def get_current_venv():
        """
        Returns the path to the current virtualenv
        """
        if 'VIRTUAL_ENV' in os.environ:
            venv = os.environ['VIRTUAL_ENV']
        elif os.path.exists('.python-version'):  # pragma: no cover
            try:
                subprocess.check_output('pyenv help', stderr=subprocess.STDOUT)
            except OSError:
                print("This directory seems to have pyenv's local venv, "
                      "but pyenv executable was not found.")
            with open('.python-version', 'r') as f:
                env_name = f.read()[:-1]
            bin_path = subprocess.check_output(['pyenv', 'which', 'python']).decode('utf-8')
            venv = bin_path[:bin_path.rfind(env_name)] + env_name
        else:  # pragma: no cover
            return None
        return venv

    def create_lambda_zip(  self,
                            prefix='lambda_package',
                            handler_file=None,
                            slim_handler=False,
                            minify=True,
                            exclude=None,
                            use_precompiled_packages=True,
                            include=None,
                            venv=None,
                            output=None
                        ):
        """
        Create a Lambda-ready zip file of the current virtualenvironment and working directory.

        Returns path to that file.

        """
        # Pip is a weird package.
        # Calling this function in some environments without this can cause.. funkiness.
        import pip

        if not venv:
            venv = self.get_current_venv()

        cwd = os.getcwd()
        if not output:
            zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
        else:
            zip_fname = output
        zip_path = os.path.join(cwd, zip_fname)

        # Files that should be excluded from the zip
        if exclude is None:
            exclude = list()

        # Exclude the zip itself
        exclude.append(zip_path)

        # Make sure that 'concurrent' is always forbidden.
        # https://github.com/Miserlou/Zappa/issues/827
        if not 'concurrent' in exclude:
            exclude.append('concurrent')

        def splitpath(path):
            parts = []
            (path, tail) = os.path.split(path)
            while path and tail:
                parts.append(tail)
                (path, tail) = os.path.split(path)
            parts.append(os.path.join(path, tail))
            return list(map(os.path.normpath, parts))[::-1]
        split_venv = splitpath(venv)
        split_cwd = splitpath(cwd)

        # Ideally this should be avoided automatically,
        # but this serves as an okay stop-gap measure.
        if split_venv[-1] == split_cwd[-1]:  # pragma: no cover
            print(
                "Warning! Your project and virtualenv have the same name! You may want "
                "to re-create your venv with a new name, or explicitly define a "
                "'project_name', as this may cause errors."
            )

        # First, do the project..
        temp_project_path = os.path.join(tempfile.gettempdir(), str(int(time.time())))

        os.makedirs(temp_project_path)
        if not slim_handler:
            # Slim handler does not take the project files.
            if minify:
                # Related: https://github.com/Miserlou/Zappa/issues/744
                excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
                copytree(cwd, temp_project_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
            else:
                copytree(cwd, temp_project_path, symlinks=False)

        # If a handler_file is supplied, copy that to the root of the package,
        # because that's where AWS Lambda looks for it. It can't be inside a package.
        if handler_file:
            filename = handler_file.split(os.sep)[-1]
            shutil.copy(handler_file, os.path.join(temp_project_path, filename))

        # Then, do site site-packages..
        egg_links = []
        temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1)))
        if os.sys.platform == 'win32':
            site_packages = os.path.join(venv, 'Lib', 'site-packages')
        else:
            site_packages = os.path.join(venv, 'lib', get_venv_from_python_version(), 'site-packages')
        egg_links.extend(glob.glob(os.path.join(site_packages, '*.egg-link')))

        if minify:
            excludes = ZIP_EXCLUDES + exclude
            copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))

        else:
            copytree(site_packages, temp_package_path, symlinks=False)

        # We may have 64-bin specific packages too.
        site_packages_64 = os.path.join(venv, 'lib64', get_venv_from_python_version(), 'site-packages')
        if os.path.exists(site_packages_64):
            egg_links.extend(glob.glob(os.path.join(site_packages_64, '*.egg-link')))
            if minify:
                excludes = ZIP_EXCLUDES + exclude
                copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
            else:
                copytree(site_packages_64, temp_package_path, symlinks=False)

        if egg_links:
            self.copy_editable_packages(egg_links, temp_package_path)

        copy_tree(temp_package_path, temp_project_path, update=True)

        # Then the pre-compiled packages..
        if use_precompiled_packages:
            print("Downloading and installing dependencies..")
            installed_packages = self.get_installed_packages(site_packages, site_packages_64)

            try:
                for installed_package_name, installed_package_version in installed_packages.items():

                    if self.have_correct_lambda_package_version(installed_package_name, installed_package_version):
                        print("Using lambda_packages binary for %s %s" % (installed_package_name, installed_package_version,))
                        self.extract_lambda_package(installed_package_name, temp_project_path)
                    else:
                        cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version)
                        if cached_wheel_path:
                            # Otherwise try to use manylinux packages from PyPi..
                            # Related: https://github.com/Miserlou/Zappa/issues/398
                            shutil.rmtree(os.path.join(temp_project_path, installed_package_name), ignore_errors=True)
                            with zipfile.ZipFile(cached_wheel_path) as zfile:
                                zfile.extractall(temp_project_path)

                        elif self.have_any_lambda_package_version(installed_package_name):
                            # Finally see if we may have at least one version of the package in lambda packages
                            # Related: https://github.com/Miserlou/Zappa/issues/855
                            lambda_version = lambda_packages[installed_package_name][self.runtime]['version']
                            print("Warning! You require pre-compiled %s version %s but will use %s that is in "
                                  "lambda_packages. " % (installed_package_name, installed_package_version, lambda_version, ))
                            self.extract_lambda_package(installed_package_name, temp_project_path)

            except Exception as e:
                print(e)
                # XXX - What should we do here?

        # Then zip it all up..
        print("Packaging project as zip..")
        try:
            # import zlib
            compression_method = zipfile.ZIP_DEFLATED
        except ImportError:  # pragma: no cover
            compression_method = zipfile.ZIP_STORED

        zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
        for root, dirs, files in os.walk(temp_project_path):

            for filename in files:

                # Skip .pyc files for Django migrations
                # https://github.com/Miserlou/Zappa/issues/436
                # https://github.com/Miserlou/Zappa/issues/464
                if filename[-4:] == '.pyc' and root[-10:] == 'migrations':
                    continue

                # If there is a .pyc file in this package,
                # we can skip the python source code as we'll just
                # use the compiled bytecode anyway..
                if filename[-3:] == '.py' and root[-10:] != 'migrations':
                    abs_filname = os.path.join(root, filename)
                    abs_pyc_filename = abs_filname + 'c'
                    if os.path.isfile(abs_pyc_filename):

                        # but only if the pyc is older than the py,
                        # otherwise we'll deploy outdated code!
                        py_time = os.stat(abs_filname).st_mtime
                        pyc_time = os.stat(abs_pyc_filename).st_mtime

                        if pyc_time > py_time:
                            continue

                # Make sure that the files are all correctly chmodded
                # Related: https://github.com/Miserlou/Zappa/issues/484
                # Related: https://github.com/Miserlou/Zappa/issues/682
                os.chmod(os.path.join(root, filename),  0o755)

                # Actually put the file into the proper place in the zip
                # Related: https://github.com/Miserlou/Zappa/pull/716
                zipi = zipfile.ZipInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
                zipi.create_system = 3
                zipi.external_attr = 0o755 << int(16) # Is this P2/P3 functional?
                with open(os.path.join(root, filename), 'rb') as f:
                    zipf.writestr(zipi, f.read(), compression_method)

            # Create python init file if it does not exist
            # Only do that if there are sub folders or python files
            # Related: https://github.com/Miserlou/Zappa/issues/766
            if '__init__.py' not in files and contains_python_files_or_subdirs(dirs, files):
                tmp_init = os.path.join(temp_project_path, '__init__.py')
                open(tmp_init, 'a').close()
                os.chmod(tmp_init,  0o755)
                zipf.write(tmp_init,
                           os.path.join(root.replace(temp_project_path, ''),
                                        os.path.join(root.replace(temp_project_path, ''), '__init__.py')))

        # And, we're done!
        zipf.close()

        # Trash the temp directory
        shutil.rmtree(temp_project_path)
        shutil.rmtree(temp_package_path)
        if os.path.isdir(venv) and slim_handler:
            # Remove the temporary handler venv folder
            shutil.rmtree(venv)

        return zip_fname

    def extract_lambda_package(self, package_name, path):
        """
        Extracts the lambda package into a given path. Assumes the package exists in lambda packages.
        """
        lambda_package = lambda_packages[package_name][self.runtime]

        # Trash the local version to help with package space saving
        shutil.rmtree(os.path.join(path, package_name), ignore_errors=True)

        tar = tarfile.open(lambda_package['path'], mode="r:gz")
        for member in tar.getmembers():
            tar.extract(member, path)

    @staticmethod
    def get_installed_packages(site_packages, site_packages_64):
        """
        Returns a dict of installed packages that Zappa cares about.
        """
        import pip  # this is to avoid 'funkiness' with global import
        package_to_keep = []
        if os.path.isdir(site_packages):
            package_to_keep += os.listdir(site_packages)
        if os.path.isdir(site_packages_64):
            package_to_keep += os.listdir(site_packages_64)

        installed_packages = {package.project_name.lower(): package.version for package in
                              pip.get_installed_distributions() if package.project_name in package_to_keep or
                              package.location in [site_packages, site_packages_64]}

        return installed_packages

    def have_correct_lambda_package_version(self, package_name, package_version):
        """
        Checks if a given package version binary should be copied over from lambda packages.
        """
        lambda_package_details = lambda_packages.get(package_name, {}).get(self.runtime)

        if lambda_package_details is None:
            return False

        # Binaries can be compiled for different package versions
        # Related: https://github.com/Miserlou/Zappa/issues/800
        if package_version != lambda_package_details['version']:
            return False

        return True

    def have_any_lambda_package_version(self, package_name):
        """
        Checks if a given package has any lambda package version. We can try and use it with a warning.
        """
        return lambda_packages.get(package_name, {}).get(self.runtime) is not None

    @staticmethod
    def download_url_with_progress(url, stream):
        """
        Downloads a given url in chunks and writes to the provided stream (can be any io stream).<commit_msg>fixes #865: match lower-cased get_installed_packages in lambda_packages dict for cases like MySQL-Python and Pillow<commit_after>from __future__ import print_function

import boto3
import botocore
import glob
import json
import logging
import os
import random
import requests
import shutil
import string
import subprocess
import tarfile
import tempfile
import time
import troposphere
import troposphere.apigateway
import zipfile

from builtins import int, bytes
from botocore.exceptions import ClientError
from distutils.dir_util import copy_tree
from io import BytesIO, open
from lambda_packages import lambda_packages as lambda_packages_orig
from setuptools import find_packages
from tqdm import tqdm

# Zappa imports
from .utilities import (copytree,
                    add_event_source,
                    remove_event_source,
                    human_size,
                    get_topic_name,
                    contains_python_files_or_subdirs,
                    get_venv_from_python_version)

##
# Logging Config
##

logging.basicConfig(format='%(levelname)s:%(message)s')
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


# We lower-case lambda package keys to match lower-cased keys in get_installed_packages()
lambda_packages = {package_name.lower():val
                   for package_name,val in lambda_packages_orig.items()}

##
# Policies And Template Mappings
##

ASSUME_POLICY = """{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Principal": {
        "Service": [
          "apigateway.amazonaws.com",
          "lambda.amazonaws.com",
          "events.amazonaws.com"
        ]
      },
      "Action": "sts:AssumeRole"
    }
  ]
}"""

ATTACH_POLICY = """{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "logs:*"
            ],
            "Resource": "arn:aws:logs:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "lambda:InvokeFunction"
            ],
            "Resource": [
                "*"
            ]
        },
        {
            "Effect": "Allow",
            "Action": [
                "ec2:AttachNetworkInterface",
                "ec2:CreateNetworkInterface",
                "ec2:DeleteNetworkInterface",
                "ec2:DescribeInstances",
                "ec2:DescribeNetworkInterfaces",
                "ec2:DetachNetworkInterface",
                "ec2:ModifyNetworkInterfaceAttribute",
                "ec2:ResetNetworkInterfaceAttribute"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:*"
            ],
            "Resource": "arn:aws:s3:::*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "kinesis:*"
            ],
            "Resource": "arn:aws:kinesis:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "sns:*"
            ],
            "Resource": "arn:aws:sns:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "sqs:*"
            ],
            "Resource": "arn:aws:sqs:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "dynamodb:*"
            ],
            "Resource": "arn:aws:dynamodb:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "route53:*"
            ],
            "Resource": "*"
        }
    ]
}"""

# Latest list: https://docs.aws.amazon.com/general/latest/gr/rande.html#apigateway_region
API_GATEWAY_REGIONS = ['us-east-1', 'us-east-2',
                       'us-west-1', 'us-west-2',
                       'eu-central-1',
                       'eu-west-1', 'eu-west-2',
                       'ap-northeast-1', 'ap-northeast-2',
                       'ap-southeast-1', 'ap-southeast-2']

# Latest list: https://docs.aws.amazon.com/general/latest/gr/rande.html#lambda_region
LAMBDA_REGIONS = ['us-east-1', 'us-east-2',
                  'us-west-1', 'us-west-2',
                  'eu-central-1',
                  'eu-west-1', 'eu-west-2',
                  'ap-northeast-1', 'ap-northeast-2',
                  'ap-southeast-1', 'ap-southeast-2']

# We never need to include these.
# Related: https://github.com/Miserlou/Zappa/pull/56
# Related: https://github.com/Miserlou/Zappa/pull/581
ZIP_EXCLUDES = [
    '*.exe', '*.DS_Store', '*.Python', '*.git', '.git/*', '*.zip', '*.tar.gz',
    '*.hg', '*.egg-info', 'pip', 'docutils*', 'setuputils*'
]

##
# Classes
##

class Zappa(object):
    """
    Zappa!

    Makes it easy to run Python web applications on AWS Lambda/API Gateway.

    """

    ##
    # Configurables
    ##

    http_methods = [
        'ANY'
    ]
    role_name = "ZappaLambdaExecution"
    extra_permissions = None
    assume_policy = ASSUME_POLICY
    attach_policy = ATTACH_POLICY
    cloudwatch_log_levels = ['OFF', 'ERROR', 'INFO']

    ##
    # Credentials
    ##

    boto_session = None
    credentials_arn = None

    def __init__(self,
            boto_session=None,
            profile_name=None,
            aws_region=None,
            load_credentials=True,
            desired_role_name=None,
            runtime='python2.7'
        ):
        # Set aws_region to None to use the system's region instead
        if aws_region is None:
            # https://github.com/Miserlou/Zappa/issues/413
            self.aws_region = boto3.Session().region_name
            logger.debug("Set region from boto: %s", self.aws_region)
        else:
            self.aws_region = aws_region

        if desired_role_name:
            self.role_name = desired_role_name

        self.runtime = runtime

        if self.runtime == 'python2.7':
            self.manylinux_wheel_file_suffix = 'cp27mu-manylinux1_x86_64.whl'
        else:
            self.manylinux_wheel_file_suffix = 'cp36m-manylinux1_x86_64.whl'


        # Some common invokations, such as DB migrations,
        # can take longer than the default.

        # Note that this is set to 300s, but if connected to
        # APIGW, Lambda will max out at 30s.
        # Related: https://github.com/Miserlou/Zappa/issues/205
        long_config_dict = {
            'region_name': aws_region,
            'connect_timeout': 5,
            'read_timeout': 300
        }
        long_config = botocore.client.Config(**long_config_dict)

        if load_credentials:
            self.load_credentials(boto_session, profile_name)

        self.s3_client = self.boto_session.client('s3')
        self.lambda_client = self.boto_session.client('lambda', config=long_config)
        self.events_client = self.boto_session.client('events')
        self.apigateway_client = self.boto_session.client('apigateway')
        self.logs_client = self.boto_session.client('logs')
        self.iam_client = self.boto_session.client('iam')
        self.iam = self.boto_session.resource('iam')
        self.cloudwatch = self.boto_session.client('cloudwatch')
        self.route53 = self.boto_session.client('route53')
        self.sns_client = self.boto_session.client('sns')
        self.cf_client = self.boto_session.client('cloudformation')
        self.cf_template = troposphere.Template()
        self.cf_api_resources = []
        self.cf_parameters = {}

    def cache_param(self, value):
        '''Returns a troposphere Ref to a value cached as a parameter.'''

        if value not in self.cf_parameters:
            keyname = chr(ord('A') + len(self.cf_parameters))
            param = self.cf_template.add_parameter(troposphere.Parameter(
                keyname, Type="String", Default=value
            ))

            self.cf_parameters[value] = param

        return troposphere.Ref(self.cf_parameters[value])

    ##
    # Packaging
    ##
    def copy_editable_packages(self, egg_links, temp_package_path):
        """ """
        for egg_link in egg_links:
            with open(egg_link, 'rb') as df:
                egg_path = df.read().decode('utf-8').splitlines()[0].strip()
                pkgs = set([x.split(".")[0] for x in find_packages(egg_path, exclude=['test', 'tests'])])
                for pkg in pkgs:
                    copytree(os.path.join(egg_path, pkg), os.path.join(temp_package_path, pkg), symlinks=False)

        if temp_package_path:
            # now remove any egg-links as they will cause issues if they still exist
            for link in glob.glob(os.path.join(temp_package_path, "*.egg-link")):
                os.remove(link)

    def get_deps_list(self, pkg_name, installed_distros=None):
        """
        For a given package, returns a list of required packages. Recursive.
        """
        import pip
        deps = []
        if not installed_distros:
            installed_distros = pip.get_installed_distributions()
        for package in installed_distros:
            if package.project_name.lower() == pkg_name.lower():
                deps = [(package.project_name, package.version)]
                for req in package.requires():
                    deps += self.get_deps_list(pkg_name=req.project_name, installed_distros=installed_distros)
        return list(set(deps))  # de-dupe before returning

    def create_handler_venv(self):
        """
        Takes the installed zappa and brings it into a fresh virtualenv-like folder. All dependencies are then downloaded.
        """
        import pip

        # We will need the currenv venv to pull Zappa from
        current_venv = self.get_current_venv()

        # Make a new folder for the handler packages
        ve_path = os.path.join(os.getcwd(), 'handler_venv')

        if os.sys.platform == 'win32':
            current_site_packages_dir = os.path.join(current_venv, 'Lib', 'site-packages')
            venv_site_packages_dir = os.path.join(ve_path, 'Lib', 'site-packages')
        else:
            current_site_packages_dir = os.path.join(current_venv, 'lib', get_venv_from_python_version(), 'site-packages')
            venv_site_packages_dir = os.path.join(ve_path, 'lib', get_venv_from_python_version(), 'site-packages')

        if not os.path.isdir(venv_site_packages_dir):
            os.makedirs(venv_site_packages_dir)

        # Copy zappa* to the new virtualenv
        zappa_things = [z for z in os.listdir(current_site_packages_dir) if z.lower()[:5] == 'zappa']
        for z in zappa_things:
            copytree(os.path.join(current_site_packages_dir, z), os.path.join(venv_site_packages_dir, z))

        # Use pip to download zappa's dependencies. Copying from current venv causes issues with things like PyYAML that installs as yaml
        zappa_deps = self.get_deps_list('zappa')
        pkg_list = ['{0!s}=={1!s}'.format(dep, version) for dep, version in zappa_deps]

        # Need to manually add setuptools
        pkg_list.append('setuptools')
        pip.main(["install", "--quiet", "--target", venv_site_packages_dir] + pkg_list)

        return ve_path

    # staticmethod as per https://github.com/Miserlou/Zappa/issues/780
    @staticmethod
    def get_current_venv():
        """
        Returns the path to the current virtualenv
        """
        if 'VIRTUAL_ENV' in os.environ:
            venv = os.environ['VIRTUAL_ENV']
        elif os.path.exists('.python-version'):  # pragma: no cover
            try:
                subprocess.check_output('pyenv help', stderr=subprocess.STDOUT)
            except OSError:
                print("This directory seems to have pyenv's local venv, "
                      "but pyenv executable was not found.")
            with open('.python-version', 'r') as f:
                env_name = f.read()[:-1]
            bin_path = subprocess.check_output(['pyenv', 'which', 'python']).decode('utf-8')
            venv = bin_path[:bin_path.rfind(env_name)] + env_name
        else:  # pragma: no cover
            return None
        return venv

    def create_lambda_zip(  self,
                            prefix='lambda_package',
                            handler_file=None,
                            slim_handler=False,
                            minify=True,
                            exclude=None,
                            use_precompiled_packages=True,
                            include=None,
                            venv=None,
                            output=None
                        ):
        """
        Create a Lambda-ready zip file of the current virtualenvironment and working directory.

        Returns path to that file.

        """
        # Pip is a weird package.
        # Calling this function in some environments without this can cause.. funkiness.
        import pip

        if not venv:
            venv = self.get_current_venv()

        cwd = os.getcwd()
        if not output:
            zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
        else:
            zip_fname = output
        zip_path = os.path.join(cwd, zip_fname)

        # Files that should be excluded from the zip
        if exclude is None:
            exclude = list()

        # Exclude the zip itself
        exclude.append(zip_path)

        # Make sure that 'concurrent' is always forbidden.
        # https://github.com/Miserlou/Zappa/issues/827
        if not 'concurrent' in exclude:
            exclude.append('concurrent')

        def splitpath(path):
            parts = []
            (path, tail) = os.path.split(path)
            while path and tail:
                parts.append(tail)
                (path, tail) = os.path.split(path)
            parts.append(os.path.join(path, tail))
            return list(map(os.path.normpath, parts))[::-1]
        split_venv = splitpath(venv)
        split_cwd = splitpath(cwd)

        # Ideally this should be avoided automatically,
        # but this serves as an okay stop-gap measure.
        if split_venv[-1] == split_cwd[-1]:  # pragma: no cover
            print(
                "Warning! Your project and virtualenv have the same name! You may want "
                "to re-create your venv with a new name, or explicitly define a "
                "'project_name', as this may cause errors."
            )

        # First, do the project..
        temp_project_path = os.path.join(tempfile.gettempdir(), str(int(time.time())))

        os.makedirs(temp_project_path)
        if not slim_handler:
            # Slim handler does not take the project files.
            if minify:
                # Related: https://github.com/Miserlou/Zappa/issues/744
                excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
                copytree(cwd, temp_project_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
            else:
                copytree(cwd, temp_project_path, symlinks=False)

        # If a handler_file is supplied, copy that to the root of the package,
        # because that's where AWS Lambda looks for it. It can't be inside a package.
        if handler_file:
            filename = handler_file.split(os.sep)[-1]
            shutil.copy(handler_file, os.path.join(temp_project_path, filename))

        # Then, do site site-packages..
        egg_links = []
        temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1)))
        if os.sys.platform == 'win32':
            site_packages = os.path.join(venv, 'Lib', 'site-packages')
        else:
            site_packages = os.path.join(venv, 'lib', get_venv_from_python_version(), 'site-packages')
        egg_links.extend(glob.glob(os.path.join(site_packages, '*.egg-link')))

        if minify:
            excludes = ZIP_EXCLUDES + exclude
            copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))

        else:
            copytree(site_packages, temp_package_path, symlinks=False)

        # We may have 64-bin specific packages too.
        site_packages_64 = os.path.join(venv, 'lib64', get_venv_from_python_version(), 'site-packages')
        if os.path.exists(site_packages_64):
            egg_links.extend(glob.glob(os.path.join(site_packages_64, '*.egg-link')))
            if minify:
                excludes = ZIP_EXCLUDES + exclude
                copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
            else:
                copytree(site_packages_64, temp_package_path, symlinks=False)

        if egg_links:
            self.copy_editable_packages(egg_links, temp_package_path)

        copy_tree(temp_package_path, temp_project_path, update=True)

        # Then the pre-compiled packages..
        if use_precompiled_packages:
            print("Downloading and installing dependencies..")
            installed_packages = self.get_installed_packages(site_packages, site_packages_64)

            try:
                for installed_package_name, installed_package_version in installed_packages.items():
                    if self.have_correct_lambda_package_version(installed_package_name, installed_package_version):
                        print("Using lambda_packages binary for %s %s" % (installed_package_name, installed_package_version,))
                        self.extract_lambda_package(installed_package_name, temp_project_path)
                    else:
                        cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version)
                        if cached_wheel_path:
                            # Otherwise try to use manylinux packages from PyPi..
                            # Related: https://github.com/Miserlou/Zappa/issues/398
                            shutil.rmtree(os.path.join(temp_project_path, installed_package_name), ignore_errors=True)
                            with zipfile.ZipFile(cached_wheel_path) as zfile:
                                zfile.extractall(temp_project_path)

                        elif self.have_any_lambda_package_version(installed_package_name):
                            # Finally see if we may have at least one version of the package in lambda packages
                            # Related: https://github.com/Miserlou/Zappa/issues/855
                            lambda_version = lambda_packages[installed_package_name][self.runtime]['version']
                            print("Warning! You require pre-compiled %s version %s but will use %s that is in "
                                  "lambda_packages. " % (installed_package_name, installed_package_version, lambda_version, ))
                            self.extract_lambda_package(installed_package_name, temp_project_path)

            except Exception as e:
                print(e)
                # XXX - What should we do here?

        # Then zip it all up..
        print("Packaging project as zip..")
        try:
            # import zlib
            compression_method = zipfile.ZIP_DEFLATED
        except ImportError:  # pragma: no cover
            compression_method = zipfile.ZIP_STORED

        zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
        for root, dirs, files in os.walk(temp_project_path):

            for filename in files:

                # Skip .pyc files for Django migrations
                # https://github.com/Miserlou/Zappa/issues/436
                # https://github.com/Miserlou/Zappa/issues/464
                if filename[-4:] == '.pyc' and root[-10:] == 'migrations':
                    continue

                # If there is a .pyc file in this package,
                # we can skip the python source code as we'll just
                # use the compiled bytecode anyway..
                if filename[-3:] == '.py' and root[-10:] != 'migrations':
                    abs_filname = os.path.join(root, filename)
                    abs_pyc_filename = abs_filname + 'c'
                    if os.path.isfile(abs_pyc_filename):

                        # but only if the pyc is older than the py,
                        # otherwise we'll deploy outdated code!
                        py_time = os.stat(abs_filname).st_mtime
                        pyc_time = os.stat(abs_pyc_filename).st_mtime

                        if pyc_time > py_time:
                            continue

                # Make sure that the files are all correctly chmodded
                # Related: https://github.com/Miserlou/Zappa/issues/484
                # Related: https://github.com/Miserlou/Zappa/issues/682
                os.chmod(os.path.join(root, filename),  0o755)

                # Actually put the file into the proper place in the zip
                # Related: https://github.com/Miserlou/Zappa/pull/716
                zipi = zipfile.ZipInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
                zipi.create_system = 3
                zipi.external_attr = 0o755 << int(16) # Is this P2/P3 functional?
                with open(os.path.join(root, filename), 'rb') as f:
                    zipf.writestr(zipi, f.read(), compression_method)

            # Create python init file if it does not exist
            # Only do that if there are sub folders or python files
            # Related: https://github.com/Miserlou/Zappa/issues/766
            if '__init__.py' not in files and contains_python_files_or_subdirs(dirs, files):
                tmp_init = os.path.join(temp_project_path, '__init__.py')
                open(tmp_init, 'a').close()
                os.chmod(tmp_init,  0o755)
                zipf.write(tmp_init,
                           os.path.join(root.replace(temp_project_path, ''),
                                        os.path.join(root.replace(temp_project_path, ''), '__init__.py')))

        # And, we're done!
        zipf.close()

        # Trash the temp directory
        shutil.rmtree(temp_project_path)
        shutil.rmtree(temp_package_path)
        if os.path.isdir(venv) and slim_handler:
            # Remove the temporary handler venv folder
            shutil.rmtree(venv)

        return zip_fname

    def extract_lambda_package(self, package_name, path):
        """
        Extracts the lambda package into a given path. Assumes the package exists in lambda packages.
        """
        lambda_package = lambda_packages[package_name][self.runtime]

        # Trash the local version to help with package space saving
        shutil.rmtree(os.path.join(path, package_name), ignore_errors=True)

        tar = tarfile.open(lambda_package['path'], mode="r:gz")
        for member in tar.getmembers():
            tar.extract(member, path)

    @staticmethod
    def get_installed_packages(site_packages, site_packages_64):
        """
        Returns a dict of installed packages that Zappa cares about.
        """
        import pip  # this is to avoid 'funkiness' with global import
        package_to_keep = []
        if os.path.isdir(site_packages):
            package_to_keep += os.listdir(site_packages)
        if os.path.isdir(site_packages_64):
            package_to_keep += os.listdir(site_packages_64)

        installed_packages = {package.project_name.lower(): package.version for package in
                              pip.get_installed_distributions() if package.project_name in package_to_keep or
                              package.location in [site_packages, site_packages_64]}

        return installed_packages

    def have_correct_lambda_package_version(self, package_name, package_version):
        """
        Checks if a given package version binary should be copied over from lambda packages.
        package_name should be lower-cased version of package name.
        """
        lambda_package_details = lambda_packages.get(package_name, {}).get(self.runtime)

        if lambda_package_details is None:
            return False

        # Binaries can be compiled for different package versions
        # Related: https://github.com/Miserlou/Zappa/issues/800
        if package_version != lambda_package_details['version']:
            return False

        return True

    def have_any_lambda_package_version(self, package_name):
        """
        Checks if a given package has any lambda package version. We can try and use it with a warning.
        package_name should be lower-cased version of package name.
        """
        return lambda_packages.get(package_name, {}).get(self.runtime) is not None

    @staticmethod
    def download_url_with_progress(url, stream):
        """
        Downloads a given url in chunks and writes to the provided stream (can be any io stream).
{'fraction_non_alphanumeric': 0.1023848936300656, 'fraction_numerical': 0.008113134285655898, 'mean_word_length': 1.9432214603632865, 'pattern_counts': {'":': 72, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 36, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5123772', 'n_tokens_mistral': 14534, 'n_tokens_neox': 13896, 'n_words': 3141}