<commit_before>#!/usr/bin/env python
import os
import sys
import platform
import tempfile
import urllib2
import shutil
import boto

from boto.exception import BotoClientError

def merge_dicts(a, b):
    """
Merge two dictionaries. If there is a key collision, `b` overrides `a`.
    :param a: Dictionary of default settings
    :param b: Dictionary of override settings
    :rtype : dict
    """
    
    try:
        a.update(b)
    except Exception as exc:
        #TODO: Update `except` logic
        raise SystemError('Failed to merge dictionaries. Dictionary A:\n\n'
                          '{0}\n\n'
                          'Dictionary B:\n\n'
                          '{1}\n\n'
                          'Exception: {2}'
                          .format(a, b, exc))
    
    return a


def get_scripts_to_execute(system, workingdir, **scriptparams):
    """
Returns an array of hashtables. Each hashtable has two keys: 'ScriptUrl' and 'Parameters'.
'ScriptSource' is the path to the script to be executed. Only supports http/s sources currently.
'Parameters' is a hashtable of parameters to pass to the script.
Use `merge_dicts({yourdict}, scriptparams)` to merge command line parameters with a set of default parameters.
    :param system: str, the system type as returned from `platform.system`
    :param workingdir: str, the working directory where content should be saved
    :param scriptparams: dict, parameters passed to the master script which should be relayed to the content scripts
    :rtype : dict
    """

    if 'Linux' in system:
        scriptstoexecute = (
            {
                'ScriptSource': "https://systemprep.s3.amazonaws.com/ContentScripts/systemprep-linuxyumrepoinstall.py",
                'Parameters': merge_dicts({
                    'yumrepomap': [
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-amzn-packages.repo',
                            'dist': 'amazon',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-centos-packages.repo',
                            'dist': 'centos',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-rhel-packages.repo',
                            'dist': 'redhat',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-epel6-packages.repo',
                            'dist': 'all',
                            'epel_version': '6',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-epel7-packages.repo',
                            'dist': 'all',
                            'epel_version': '7',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-salt-epel6-packages.repo',
                            'dist': 'all',
                            'epel_version': '6',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-salt-epel7-packages.repo',
                            'dist': 'all',
                            'epel_version': '7',
                        },
                    ],
                }, scriptparams)
            },
            { 
                'ScriptSource': "https://systemprep.s3.amazonaws.com/ContentScripts/SystemPrep-LinuxSaltInstall.py",
                'Parameters': merge_dicts({
                    'saltinstallmethod': 'yum',
                    'saltcontentsource': "https://systemprep-content.s3.amazonaws.com/linux/salt/salt-content.zip",
                    'formulastoinclude': [
                        "https://salt-formulas.s3.amazonaws.com/systemprep-formula-master.zip",
                        "https://salt-formulas.s3.amazonaws.com/ash-linux-formula-master.zip",
                        "https://salt-formulas.s3.amazonaws.com/join-domain-formula-master.zip",
                    ],
                    'formulaterminationstrings': [
                        "-master",
                        "-latest",
                    ],
                    'saltstates': 'Highstate',
                    'salt_results_log': '/var/log/saltcall.results.log',
                    'salt_debug_log': '/var/log/saltcall.debug.log',
                    'sourceiss3bucket': 'True',
                }, scriptparams)
            },
        )
    elif 'Windows' in system:
        scriptstoexecute = (
            {
                'ScriptSource': "https://systemprep.s3.amazonaws.com/SystemContent/Windows/Salt/SystemPrep-WindowsSaltInstall.ps1",
                'Parameters': merge_dicts({
                    'saltworkingdir': '{0}\\SystemContent\\Windows\\Salt'.format(workingdir),
                    'saltcontentsource': "https://systemprep.s3.amazonaws.com/SystemContent/Windows/Salt/salt-content.zip",
                    'formulastoinclude': [
                        "https://salt-formulas.s3.amazonaws.com/systemprep-formula-master.zip",
                        "https://salt-formulas.s3.amazonaws.com/ash-windows-formula-master.zip",
                    ],
                    'formulaterminationstrings': [
                        "-latest",
                    ],
                    'ashrole': "MemberServer",
                    'netbannerstring': "Unclass",
                    'saltstates': "Highstate",
                }, scriptparams)
            },
        )
    else:
        #TODO: Update `except` logic
        raise SystemError('System, {0}, is not recognized?'.format(system))
    
    return scriptstoexecute


def create_working_dir(basedir, dirprefix):
    """
Creates a directory in `basedir` with a prefix of `dirprefix`.
The directory will have a random 5 character string appended to `dirprefix`.
Returns the path to the working directory.
    :rtype : str
    :param basedir: str, the directory in which to create the working directory
    :param dirprefix: str, prefix to prepend to the working directory
    """
    workingdir = None
    try:
        workingdir = tempfile.mkdtemp(prefix=dirprefix, dir=basedir)
    except Exception as exc:
        #TODO: Update `except` logic
        raise SystemError('Could not create workingdir in {0}.\n'
                          'Exception: {1}'.format(basedir, exc))

    return workingdir

<commit_msg>Add `EntEnv` to linux masterscript<commit_after>#!/usr/bin/env python
import os
import sys
import platform
import tempfile
import urllib2
import shutil
import boto

from boto.exception import BotoClientError

def merge_dicts(a, b):
    """
Merge two dictionaries. If there is a key collision, `b` overrides `a`.
    :param a: Dictionary of default settings
    :param b: Dictionary of override settings
    :rtype : dict
    """
    
    try:
        a.update(b)
    except Exception as exc:
        #TODO: Update `except` logic
        raise SystemError('Failed to merge dictionaries. Dictionary A:\n\n'
                          '{0}\n\n'
                          'Dictionary B:\n\n'
                          '{1}\n\n'
                          'Exception: {2}'
                          .format(a, b, exc))
    
    return a


def get_scripts_to_execute(system, workingdir, **scriptparams):
    """
Returns an array of hashtables. Each hashtable has two keys: 'ScriptUrl' and 'Parameters'.
'ScriptSource' is the path to the script to be executed. Only supports http/s sources currently.
'Parameters' is a hashtable of parameters to pass to the script.
Use `merge_dicts({yourdict}, scriptparams)` to merge command line parameters with a set of default parameters.
    :param system: str, the system type as returned from `platform.system`
    :param workingdir: str, the working directory where content should be saved
    :param scriptparams: dict, parameters passed to the master script which should be relayed to the content scripts
    :rtype : dict
    """

    if 'Linux' in system:
        scriptstoexecute = (
            {
                'ScriptSource': "https://systemprep.s3.amazonaws.com/ContentScripts/systemprep-linuxyumrepoinstall.py",
                'Parameters': merge_dicts({
                    'yumrepomap': [
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-amzn-packages.repo',
                            'dist': 'amazon',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-centos-packages.repo',
                            'dist': 'centos',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-rhel-packages.repo',
                            'dist': 'redhat',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-epel6-packages.repo',
                            'dist': 'all',
                            'epel_version': '6',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-epel7-packages.repo',
                            'dist': 'all',
                            'epel_version': '7',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-salt-epel6-packages.repo',
                            'dist': 'all',
                            'epel_version': '6',
                        },
                        {
                            'url': 'https://s3.amazonaws.com/systemprep-repo/linux/yum.repos/systemprep-repo-salt-epel7-packages.repo',
                            'dist': 'all',
                            'epel_version': '7',
                        },
                    ],
                }, scriptparams)
            },
            { 
                'ScriptSource': "https://systemprep.s3.amazonaws.com/ContentScripts/SystemPrep-LinuxSaltInstall.py",
                'Parameters': merge_dicts({
                    'saltinstallmethod': 'yum',
                    'saltcontentsource': "https://systemprep-content.s3.amazonaws.com/linux/salt/salt-content.zip",
                    'formulastoinclude': [
                        "https://salt-formulas.s3.amazonaws.com/systemprep-formula-master.zip",
                        "https://salt-formulas.s3.amazonaws.com/ash-linux-formula-master.zip",
                        "https://salt-formulas.s3.amazonaws.com/join-domain-formula-master.zip",
                    ],
                    'formulaterminationstrings': [
                        "-master",
                        "-latest",
                    ],
                    'saltstates': 'Highstate',
                    'entenv': 'False',
                    'salt_results_log': '/var/log/saltcall.results.log',
                    'salt_debug_log': '/var/log/saltcall.debug.log',
                    'sourceiss3bucket': 'True',
                }, scriptparams)
            },
        )
    elif 'Windows' in system:
        scriptstoexecute = (
            {
                'ScriptSource': "https://systemprep.s3.amazonaws.com/SystemContent/Windows/Salt/SystemPrep-WindowsSaltInstall.ps1",
                'Parameters': merge_dicts({
                    'saltworkingdir': '{0}\\SystemContent\\Windows\\Salt'.format(workingdir),
                    'saltcontentsource': "https://systemprep.s3.amazonaws.com/SystemContent/Windows/Salt/salt-content.zip",
                    'formulastoinclude': [
                        "https://salt-formulas.s3.amazonaws.com/systemprep-formula-master.zip",
                        "https://salt-formulas.s3.amazonaws.com/ash-windows-formula-master.zip",
                    ],
                    'formulaterminationstrings': [
                        "-latest",
                    ],
                    'ashrole': "MemberServer",
                    'entenv': 'False',
                    'saltstates': "Highstate",
                }, scriptparams)
            },
        )
    else:
        #TODO: Update `except` logic
        raise SystemError('System, {0}, is not recognized?'.format(system))
    
    return scriptstoexecute


def create_working_dir(basedir, dirprefix):
    """
Creates a directory in `basedir` with a prefix of `dirprefix`.
The directory will have a random 5 character string appended to `dirprefix`.
Returns the path to the working directory.
    :rtype : str
    :param basedir: str, the directory in which to create the working directory
    :param dirprefix: str, prefix to prepend to the working directory
    """
    workingdir = None
    try:
        workingdir = tempfile.mkdtemp(prefix=dirprefix, dir=basedir)
    except Exception as exc:
        #TODO: Update `except` logic
        raise SystemError('Could not create workingdir in {0}.\n'
                          'Exception: {1}'.format(basedir, exc))

    return workingdir


{'fraction_non_alphanumeric': 0.11251384274640089, 'fraction_numerical': 0.0053156146179402, 'mean_word_length': 1.4732517801716267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 34, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3001887', 'n_tokens_mistral': 3689, 'n_tokens_neox': 3469, 'n_words': 736}