<commit_before>from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from boto.beanstalk import connect_to_region
from boto.s3.key import Key

from datetime import datetime
from time import time, sleep
import zipfile
import os
import sys
import yaml
import re

def out(message):
    """
    print alias
    """
    sys.stdout.write(message+"\n")
    sys.stdout.flush()

def merge_dict(dict1, dict2):
    ret = dict(dict2)
    for key, val in dict1.items():
        val2 = dict2.get(key)
        if val2 is None:
            ret[key] = val
        elif isinstance(val, dict) and isinstance(val2, dict):
            ret[key] = merge_dict(val, val2)
        elif isinstance(val, (list)) and isinstance(val2, (list)):
            ret[key] = val + val2
        else:
            ret[key] = val2
    return ret

def get(vals, key, default_val=None):
    """
    Returns a dictionary value
    """
    val = vals
    for part in key.split('.'):
        if isinstance(val, dict):
            val = val.get(part, None)
            if val is None:
                return default_val
        else:
            return default_val
    return val

def parse_option_settings(option_settings):
    """
    Parses option_settings as they are defined in the configuration file
    """
    ret = []
    for namespace, params in option_settings.items():
        for key, value in params.items():
            ret.append((namespace, key, value))
    return ret

def parse_env_config(config, env_name):
    """
    Parses an environment config
    """
    all_env = get(config, 'app.all_environments', {})
    env = get(config, 'app.environments.'+env_name, {})
    return merge_dict(all_env, env)

def upload_application_archive(helper, env_config, archive=None, directory=None, version_label=None):
    if version_label is None:
        version_label = datetime.now().strftime('%Y%m%d_%H%M%S')
    if not archive:
        if not directory:
            directory = "."
        includes = get(env_config, 'archive.includes', [])
        excludes = get(env_config, 'archive.excludes', [])
        archive_files =  get(env_config, 'archive.files', [])
        def _predicate(f):
            for exclude in excludes:
                if re.match(exclude, f):
                    return False
            if len(includes)>0:
                for include in includes:
                    if re.match(include, f):
                        return True
                return False
            return True
        archive = create_archive(directory, version_label+".zip", config=archive_files, ignore_predicate=_predicate)
    helper.upload_archive(archive, version_label+".zip")
    helper.create_application_version(version_label, version_label+".zip")
    return version_label

def create_archive(directory, filename, config={}, ignore_predicate=None, ignored_files=['.git', '.svn']):
    """
    Creates an archive from a directory and returns
    the file that was created.
    """
    zip = zipfile.ZipFile(filename, 'w', compression=zipfile.ZIP_DEFLATED)
    root_len = len(os.path.abspath(directory))

    # create it
    out("Creating archive: "+filename)
    for root, dirs, files in os.walk(directory, followlinks=True):
        archive_root = os.path.abspath(root)[root_len+1:]
        for f in files:<commit_msg>Added support for invoking an external command to create the application archive<commit_after>from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from boto.beanstalk import connect_to_region
from boto.s3.key import Key

from datetime import datetime
from time import time, sleep
import zipfile
import os
import subprocess
import sys
import yaml
import re

def out(message):
    """
    print alias
    """
    sys.stdout.write(message+"\n")
    sys.stdout.flush()

def merge_dict(dict1, dict2):
    ret = dict(dict2)
    for key, val in dict1.items():
        val2 = dict2.get(key)
        if val2 is None:
            ret[key] = val
        elif isinstance(val, dict) and isinstance(val2, dict):
            ret[key] = merge_dict(val, val2)
        elif isinstance(val, (list)) and isinstance(val2, (list)):
            ret[key] = val + val2
        else:
            ret[key] = val2
    return ret

def get(vals, key, default_val=None):
    """
    Returns a dictionary value
    """
    val = vals
    for part in key.split('.'):
        if isinstance(val, dict):
            val = val.get(part, None)
            if val is None:
                return default_val
        else:
            return default_val
    return val

def parse_option_settings(option_settings):
    """
    Parses option_settings as they are defined in the configuration file
    """
    ret = []
    for namespace, params in option_settings.items():
        for key, value in params.items():
            ret.append((namespace, key, value))
    return ret

def parse_env_config(config, env_name):
    """
    Parses an environment config
    """
    all_env = get(config, 'app.all_environments', {})
    env = get(config, 'app.environments.'+env_name, {})
    return merge_dict(all_env, env)

def upload_application_archive(helper, env_config, archive=None, directory=None, version_label=None):
    if version_label is None:
        version_label = datetime.now().strftime('%Y%m%d_%H%M%S')
    archive_file_name = None

    # generate the archive externally
    if get(env_config, 'archive.generate'):
        cmd = get(env_config, 'archive.generate.cmd')
        output_file = get(env_config, 'archive.generate.output_file')
        use_shell = get(env_config, 'archive.generate.use_shell', True)
        exit_code = get(env_config, 'archive.generate.exit_code', 0)
        if not cmd or not output_file:
            raise Exception('Archive generation requires cmd and output_file at a minimum')
        output_regex = None
        try:
            output_regex = re.compile(output_file)
        except:
            pass
        result = subprocess.call(cmd, shell=use_shell)
        if result != exit_code:
            raise Exception('Generate command execited with code %s (expected %s)' % (result, exit_code))

        if output_file and os.path.exists(output_file):
            archive_file_name = os.path.basename(output_file)
        else:
            for root, dirs, files in os.walk(".", followlinks=True):
                for f in files:
                    fullpath = os.path.join(root, f)
                    if fullpath.endswith(output_file):
                        archive = fullpath
                        archive_file_name = os.path.basename(fullpath)
                        break
                    elif output_regex and output_regex.match(fullpath):
                        archive = fullpath
                        archive_file_name = os.path.basename(fullpath)
                        break
                if archive:
                    break
            if not archive or not archive_file_name:
                raise Exception('Unable to find expected output file matching: %s' % (output_file))

    # create the archive
    elif not archive:
        if not directory:
            directory = "."
        includes = get(env_config, 'archive.includes', [])
        excludes = get(env_config, 'archive.excludes', [])
        archive_files =  get(env_config, 'archive.files', [])
        def _predicate(f):
            for exclude in excludes:
                if re.match(exclude, f):
                    return False
            if len(includes)>0:
                for include in includes:
                    if re.match(include, f):
                        return True
                return False
            return True
        archive = create_archive(directory, version_label+".zip", config=archive_files, ignore_predicate=_predicate)
        archive_file_name = version_label+".zip"

    helper.upload_archive(archive, archive_file_name)
    helper.create_application_version(version_label, archive_file_name)
    return version_label

def create_archive(directory, filename, config={}, ignore_predicate=None, ignored_files=['.git', '.svn']):
    """
    Creates an archive from a directory and returns
    the file that was created.
    """
    zip = zipfile.ZipFile(filename, 'w', compression=zipfile.ZIP_DEFLATED)
    root_len = len(os.path.abspath(directory))

    # create it
    out("Creating archive: "+filename)
    for root, dirs, files in os.walk(directory, followlinks=True):
        archive_root = os.path.abspath(root)[root_len+1:]
        for f in files:
{'fraction_non_alphanumeric': 0.09929742388758782, 'fraction_numerical': 0.004332552693208431, 'mean_word_length': 2.3946740858505566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7112275', 'n_tokens_mistral': 2459, 'n_tokens_neox': 2408, 'n_words': 668}