<commit_before>package testeng

import org.yaml.snakeyaml.Yaml
import static org.edx.jenkins.dsl.JenkinsPublicConstants.JENKINS_PUBLIC_HIPCHAT

Map config = [:]
Binding bindings = getBinding()
config.putAll(bindings.getVariables())
PrintStream out = config['out']

try {
    out.println('Parsing secret YAML file')
    /* Parse k:v pairs from the secret file referenced by secretFileVariable */
    String contents = new File("${BACKUP_JENKINS_SECRET}").text
    Yaml yaml = new Yaml()
    secretMap = yaml.load(contents)
    out.println('Successfully parsed secret YAML file')
}
catch (any) {
    out.println('Jenkins DSL: Error parsing secret YAML file')
    out.println('Exiting with error code 1')
    return 1
}

/*
Example secret YAML file used by this script
Config:
    jenkinsInstance : test
    volumeId : vol-123
    region : us-west.1
    hipchat : hipchat token
    email : <EMAIL>
*/

/* Iterate over the job configurations */
secretMap.each { jobConfigs ->

    Map jobConfig = jobConfigs.getValue()

    assert jobConfig.containsKey('jenkinsInstance')
    assert jobConfig.containsKey('volumeId')
    assert jobConfig.containsKey('region')
    assert jobConfig.containsKey('hipchat')
    assert jobConfig.containsKey('email')

    job("backup-${jobConfig['jenkinsInstance']}-jenkins") {
        
        // private job
        authorization {
            blocksInheritance(true)
            permissionAll('edx')
        }

        description('A regularly run job for creating snapshots of build jenkins master ' +
                    ' and uploading them to Amazon s3')
        // Keep logs longer than normal jenkins jobs
        logRotator {
            numToKeep(50)
        }
        concurrentBuild(false)
        label("backup-runner")

        // Configure the Exclusive Execution plugin, to reduce the amount of things in memory
        // during snapshotting
        configure { project ->
            project / buildWrappers << 'hudson.plugins.execution.exclusive.ExclusiveBuildWrapper' {
                skipWaitOnRunningJobs false
            }
        }

        // Run snapshotting script once a day, at 1:00 AM
        triggers {
            cron('0 1 * * *')
        }

        wrappers {
            timeout {
                absolute(20)
                abortBuild()
            }
            timestamps()
            colorizeOutput('xterm')
        }

        environmentVariables {
            env('AWS_DEFAULT_REGION', jobConfig['region'])
        }
        
        // Sync currently paged files to disk
        String script = "set -o pipefail\n"
        script += "sync\n"
        // This might seem overkill, but in case the pip requirements change, read them from
        // the requirements file in the workspace
        readFileFromWorkspace('testeng/resources/requirements.txt').split("\n").each { line ->
            script += "pip install --exists-action w ${line}\n"
        }
        script += "aws ec2 create-snapshot --volume-id ${jobConfig['volumeId']} --description 'Automatic ${jobConfig['jenkinsInstance']} jenkins snapshot' |tee \${WORKSPACE}/snapshot-out"
        steps {
            virtualenv {
                clear()
                name('venv')
                nature('shell')
                command(script)
            }
        }
<commit_msg>tweak script to use keys instead of roles<commit_after>package testeng

import org.yaml.snakeyaml.Yaml
import static org.edx.jenkins.dsl.JenkinsPublicConstants.JENKINS_PUBLIC_HIPCHAT

Map config = [:]
Binding bindings = getBinding()
config.putAll(bindings.getVariables())
PrintStream out = config['out']

try {
    out.println('Parsing secret YAML file')
    /* Parse k:v pairs from the secret file referenced by secretFileVariable */
    String contents = new File("${BACKUP_JENKINS_SECRET}").text
    Yaml yaml = new Yaml()
    secretMap = yaml.load(contents)
    out.println('Successfully parsed secret YAML file')
}
catch (any) {
    out.println('Jenkins DSL: Error parsing secret YAML file')
    out.println('Exiting with error code 1')
    return 1
}

/*
Example secret YAML file used by this script
Config:
    jenkinsInstance : test
    volumeId : vol-123
    region : us-west.1
    accessKey : 123
    secretAccessKey: 123
    hipchat : hipchat token
    email : <EMAIL>
*/

/* Iterate over the job configurations */
secretMap.each { jobConfigs ->

    Map jobConfig = jobConfigs.getValue()

    assert jobConfig.containsKey('jenkinsInstance')
    assert jobConfig.containsKey('volumeId')
    assert jobConfig.containsKey('region')
    assert jobConfig.containsKey('accessKey')
    assert jobConfig.containsKey('secretAccessKey')
    assert jobConfig.containsKey('hipchat')
    assert jobConfig.containsKey('email')

    job("backup-${jobConfig['jenkinsInstance']}-jenkins") {
        
        // private job
        authorization {
            blocksInheritance(true)
            permissionAll('edx')
        }

        description('A regularly run job for creating snapshots of build jenkins master ' +
                    ' and uploading them to Amazon s3')
        // Keep logs longer than normal jenkins jobs
        logRotator {
            numToKeep(50)
        }
        concurrentBuild(false)
        label("backup-runner")

        // Configure the Exclusive Execution plugin, to reduce the amount of things in memory
        // during snapshotting
        configure { project ->
            project / buildWrappers << 'hudson.plugins.execution.exclusive.ExclusiveBuildWrapper' {
                skipWaitOnRunningJobs false
            }
        }

        // Run snapshotting script once a day, when there is usually no Jenkins activity
        triggers {
            if (jobConfig['jenkinsInstance'] == 'build') {
                cron('0 1 * * *')
            }
            // test jenkins
            else {
                cron('0 2 * * *')
            }
        }

        wrappers {
            timeout {
                absolute(20)
                abortBuild()
            }
            timestamps()
            colorizeOutput('xterm')
        }

        // load env vars for executing awscli commands
        environmentVariables {
            env('AWS_ACCESS_KEY_ID', jobConfig['accessKey'])
            env('AWS_SECRET_ACCESS_KEY', jobConfig['secretAccessKey'])
            env('AWS_DEFAULT_REGION', jobConfig['region'])
        }
        
        // Sync currently paged files to disk
        String script = "sync\n"
        // This might seem overkill, but in case the pip requirements change, read them from
        // the requirements file in the workspace
        readFileFromWorkspace('testeng/resources/requirements.txt').split("\n").each { line ->
            script += "pip install --exists-action w ${line}\n"
        }
        script += "aws ec2 create-snapshot --volume-id ${jobConfig['volumeId']} --description 'Automatic ${jobConfig['jenkinsInstance']} jenkins snapshot' > \${WORKSPACE}/snapshot-out\n"
        script += "cat \${WORKSPACE}/snapshot-out"
        steps {
            virtualenv {
                clear()
                name('venv')
                nature('shell')
                command(script)
            }
        }

{'fraction_non_alphanumeric': 0.09917930171094728, 'fraction_numerical': 0.0054249547920434, 'mean_word_length': 2.1604395604395603, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4918454', 'n_tokens_mistral': 2001, 'n_tokens_neox': 1918, 'n_words': 514}