<commit_before>#!/usr/bin/env python
import os
import sys
import platform
import tempfile


def mergeDicts(a, b):
    """
Merge two dictionaries. If there is a key collision, 'b' overrides 'a'.
    """
    
    try:
        a_copy = a.copy()
        a_copy.update(b)
    except:
        SystemError('Failed to merge dictionaries. Dictionary A:\n\n' + a + '\n\nDictionary B:\n\n' + b)

    return a_copy


def getScriptsToExecute(system, workingdir, **scriptparams):
    """
Returns an array of hashtables. Each hashtable has two keys: 'ScriptUrl' and 'Parameters'.
'ScriptUrl' is the url of the script to be downloaded and execute.
'Parameters' is a hashtable of parameters to pass to the script.
    """

    if 'Linux' in system:
        scriptstoexecute = (
            { 
                'ScriptUrl'  : "https://systemprep.s3.amazonaws.com/SystemContent/Linux/Salt/SystemPrep-LinuxSaltInstall.py",
                'Parameters' : mergeDicts({ 
                                  'SaltWorkingPath' : workingdir + '/systemcontent/linux',
                                  'SaltContentUrl' : "https://systemprep.s3.amazonaws.com/SystemContent/Linux/Salt/salt-content.zip" ,
                                  'FormulasToInclude' : (
                                                            "https://salt-formulas.s3.amazonaws.com/ash-linux-formula-latest.zip", 
                                                        ),
                                  'FormulaTerminationStrings' : ( "-latest" ),
                                  'SaltStates' : 'Highstate',
                               }, scriptparams)
            },
        )
    #TODO: Add Windows parameters
    else:
        raise SystemError('System, ' + system + ', is not recognized?')

    return scriptstoexecute


def createWorkingDir(basedir, dirprefix):
    """
Creates a directory in 'basedir' with a prefix of 'dirprefix'.
The directory will have a random 5 character string appended to 'dirprefix'.
Returns the path to the working directory.
    """
    
    try:
        workingdir = tempfile.mkdtemp(prefix=dirprefix, dir=basedir)
    except:
        SystemError('Could not create workingdir in ' + str('basedir'))

    return workingdir


def getSystemParams(system):
    """
Returns a dictionary of OS platform-specific parameters.
    """
    
    dict_a = {}
    workingdirprefix = 'systemprep'
    if 'Linux' in system:
        tempdir = '/usr/tmp/'
        dict_a['pathseparator'] = '/'
        dict_a['readyfile'] = '/var/run/system-is-ready'
        # dict_a['restart'] = 
    #TODO: Add Windows parameters
    elif 'Windows' in system:
        systemroot = os.environ['SYSTEMROOT']
        systemdrive = os.environ['SYSTEMDRIVE']
        tempdir = os.environ['TEMP']
        dict_a['pathseparator'] = '\\'
        dict_a['readyfile'] = systemdrive + '\system-is-ready'
        dict_a['restart'] = systemroot + '\system32\shutdown.exe/r /t 30 /d p:2:4 /c "SystemPrep complete. Rebooting computer."'
    else:
        raise SystemError('System, ' + system + ', is not recognized?')

    dict_a['workingdir'] = createWorkingDir(tempdir, workingdirprefix)

    return dict_a


def downloadFile(url, filename):
    import urllib.request
    import shutil

    # Download the file from `url` and save it locally under `filename`:
    try:
        with urllib.request.urlopen(url) as response, open(file_name, 'wb') as out_file:
            shutil.copyfileobj(response, out_file)
    except:
        raise SystemError('Unable to download and save file. Url =\n\n    ' + url + '\n\nfilename = \n\n    ' + filename)
        
    print('Downloaded file -- ' + url)


def cleanup(workingdir):
    print('+-' * 40)
    print('Cleanup Time...')
    print('Removing temporary data...')
    try:
        os.removedirs(workingdir)
    except:
        raise SystemError('Cleanup Failed!')


def main(**kwargs):
    """
Master Script that calls subscripts to be deployed to new Linux systems
    """

    scriptname = __file__

    print('+' * 80)
    print('Entering script -- ' + scriptname)
    print('Printing parameters...')
    for key, value in kwargs.items():
        print('    ' + str(key) + ' = ' + str(value))

    system = platform.system()
    systemparams = getSystemParams(system)
    scriptstoexecute = getScriptsToExecute(system, workingdir, **kwargs)

    # #Loop through each 'script' in scriptstoexecute
    # for script in scriptstoexecute:
        # filename = script['ScriptUrl'].split('/')[-1]
        # fullfilepath = systemparams['workingdir'] + systemparams['pathseparator'] + filename
        # #Download each script, script['ScriptUrl']
        # downloadFile(script['ScriptUrl'], fullfilepath)
        # #Execute each script, passing it the parameters in script['Parameters']
        # os.system('python ' + fullfilepath + script['Parameters']) ## likely a dirty hack, probably want to code the python sub-script with an importable module instead

    # cleanup(systemparams['workingdir'])

    # if 'True' == kwargs['NoReboot']:
        # print('Detected NoReboot switch. System will not be rebooted.')
    # else:
        # print('Reboot scheduled. System will reboot in 30 seconds')
        # os.system(systemparams['restart'])

    print(str(scriptname) + 'complete!')
    print('-' * 80)
    raw_input("\n\nPress the enter key to exit.")


if __name__ == "__main__":
    kwargs = dict(x.split('=', 1) for x in sys.argv[1:])
    main(**kwargs)<commit_msg>Fixed up some syntactical errors and added TODOs<commit_after>#!/usr/bin/env python
import os
import sys
import platform
import tempfile
import urllib2
import shutil


def mergeDicts(a, b):
    """
Merge two dictionaries. If there is a key collision, `b` overrides `a`.
    """
    
    try:
        a.update(b)
    except:
        SystemError('Failed to merge dictionaries. Dictionary A:\n\n' + a + '\n\nDictionary B:\n\n' + b)
    
    return a


def getScriptsToExecute(system, workingdir, **scriptparams):
    """
Returns an array of hashtables. Each hashtable has two keys: 'ScriptUrl' and 'Parameters'.
'ScriptUrl' is the url of the script to be downloaded and execute.
'Parameters' is a hashtable of parameters to pass to the script.
    """

    if 'Linux' in system:
        scriptstoexecute = (
            { 
                'ScriptUrl'  : "https://systemprep.s3.amazonaws.com/SystemContent/Linux/Salt/SystemPrep-LinuxSaltInstall.py",
                'Parameters' : mergeDicts({ 
                                  'SaltWorkingPath' : workingdir + '/systemcontent/linux',
                                  'SaltContentUrl' : "https://systemprep.s3.amazonaws.com/SystemContent/Linux/Salt/salt-content.zip" ,
                                  'FormulasToInclude' : (
                                                            "https://salt-formulas.s3.amazonaws.com/ash-linux-formula-latest.zip", 
                                                        ),
                                  'FormulaTerminationStrings' : ( "-latest" ),
                                  'SaltStates' : 'Highstate',
                               }, scriptparams)
            },
        )
    if 'Windows' in system:
        scriptstoexecute = (
            {
                'ScriptUrl'  : "https://systemprep.s3.amazonaws.com/SystemContent/Windows/Salt/SystemPrep-WindowsSaltInstall.ps1",
                'Parameters' : mergeDicts({ 
                                              'SaltWorkingDir' : workingdir + '\\SystemContent\\Windows\\Salt',
                                              'SaltContentUrl' : "https://systemprep.s3.amazonaws.com/SystemContent/Windows/Salt/salt-content.zip",
                                              'FormulasToInclude' : (
                                                                        "https://salt-formulas.s3.amazonaws.com/ash-windows-formula-latest.zip",
                                                                    ),
                                              'FormulaTerminationStrings' : ( "-latest" ),
                                              'AshRole' : "MemberServer",
                                              'NetBannerString' : "Unclass",
                                              'SaltStates' : "Highstate",
                                            }, scriptparams)
            },
        )
    else:
        raise SystemError('System, ' + system + ', is not recognized?')
    
    return scriptstoexecute


def createWorkingDir(basedir, dirprefix):
    """
Creates a directory in `basedir` with a prefix of `dirprefix`.
The directory will have a random 5 character string appended to `dirprefix`.
Returns the path to the working directory.
    """
    
    try:
        workingdir = tempfile.mkdtemp(prefix=dirprefix, dir=basedir)
    except:
        SystemError('Could not create workingdir in ' + str('basedir'))

    return workingdir


def getSystemParams(system):
    """
Returns a dictionary of OS platform-specific parameters.
    """
    
    a = {}
    workingdirprefix = 'systemprep-'
    if 'Linux' in system:
        tempdir = '/usr/tmp/'
        a['pathseparator'] = '/'
        a['readyfile'] = '/var/run/system-is-ready'
        #TODO: figure out how to restart a linux system with a 30 second delay
        # a['restart'] = 
    #TODO: Add and test more Windows parameters/functionality
    elif 'Windows' in system:
        systemroot = os.environ['SYSTEMROOT']
        systemdrive = os.environ['SYSTEMDRIVE']
        tempdir = os.environ['TEMP']
        a['pathseparator'] = '\\'
        a['readyfile'] = systemdrive + '\system-is-ready'
        a['restart'] = systemroot + '\system32\shutdown.exe/r /t 30 /d p:2:4 /c "SystemPrep complete. Rebooting computer."'
    else:
        raise SystemError('System, ' + system + ', is not recognized?')

    a['workingdir'] = createWorkingDir(tempdir, workingdirprefix)

    return a


def downloadFile(url, filename):
    """
Download the file from `url` and save it locally under `filename`:
    """
    try:
        response = urllib2.urlopen(url)
    except:
        raise SystemError('Unable to open connection to web server. \nurl =\n    ' + url)
    
    try:
        with open(filename, 'wb') as outfile:
            shutil.copyfileobj(response, outfile)
    except:
        raise SystemError('Unable to save file. \nfilename = \n    ' + filename)
    
    print('Downloaded file -- \n    url      = ' + url + '\n    filename = ' + filename)


def cleanup(workingdir):
    print('+-' * 40)
    print('Cleanup Time...')
    print('Removing temporary data...')
    try:
        shutil.rmtree(workingdir)
    except:
        raise SystemError('Cleanup Failed!')


def main(**kwargs):
    """
Master Script that calls subscripts to be deployed to new Linux systems
    """

    scriptname = __file__

    print('+' * 80)
    print('Entering script -- ' + scriptname)
    print('Printing parameters --')
    for key, value in kwargs.items():
        print('    ' + str(key) + ' = ' + str(value))
    
    system = platform.system()
    systemparams = getSystemParams(system)
    scriptstoexecute = getScriptsToExecute(system, systemparams['workingdir'], **kwargs)
    
    #Loop through each 'script' in scriptstoexecute
    for script in scriptstoexecute:
        filename = script['ScriptUrl'].split('/')[-1]
        fullfilepath = systemparams['workingdir'] + systemparams['pathseparator'] + filename
        #Download each script, script['ScriptUrl']
        downloadFile(script['ScriptUrl'], fullfilepath)
        #Execute each script, passing it the parameters in script['Parameters']
        #TODO: figure out a better way to call and execute the script
        #os.system('python ' + fullfilepath + script['Parameters']) ## likely a dirty hack, probably want to code the python sub-script with an importable module instead

    cleanup(systemparams['workingdir'])
    
    #TODO: uncomment this when linux has a value for systemparams['restart']
    # if 'True' == kwargs['NoReboot']:
        # print('Detected NoReboot switch. System will not be rebooted.')
    # else:
        # print('Reboot scheduled. System will reboot in 30 seconds')
        # os.system(systemparams['restart'])

    print(str(scriptname) + ' complete!')
    print('-' * 80)


if __name__ == "__main__":
    kwargs = dict(x.split('=', 1) for x in sys.argv[1:])
    main(**kwargs)
{'fraction_non_alphanumeric': 0.11815608572583906, 'fraction_numerical': 0.0040436716538617065, 'mean_word_length': 2.107816034179442, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 9, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2634875', 'n_tokens_mistral': 3488, 'n_tokens_neox': 3319, 'n_words': 906}