<commit_before>// This project will generate new job generators based on changes to the repolist.txt, which will
// in turn cause those generators to run over the job generation in the local repos.

import jobs.generation.Utilities

// About this generator:
//
//  This generator is designed to run every time there is a change to
//    * The dotnet-ci repo's generation dir
//
//  A rerun of this could mean any number of things:
//    * Updated repolist.txt
//    * Updated utilities
//
//  Rerunning this generator will not automatically rerun all of the generators
//  below it (by design).  Users making changes to the overall generation structure
//  should recognize when there is a functional change being made and rerun all of the
//  generated jobs.

class Repo {
    String project
    String[] folders
    String branch
    String server
    String definitionScript
    // Indicates this is a branch for 'default' PRs. ORs to branches other than the tracked branches would use jobs
    // defined here.  If true, then the generator will receive .* + branch +  for GithubPRTargetBranches with the other tracked
    // branches  in the GithubPRSkipBranches parameter.
    boolean isDefaultPRBranch
    // Indicates this branch provides PR coverage for the listed branches.  If set, the GithubPRTargetBranches will be
    // set to this branch + prsForBranches.  GithubPRSkipBranches remains empty
    String[] additionalPRBranches = []
    
    // Lazily set up data
    // Branches that should be targeted for PRs against this job definition
    String[] prTargetBranches 
    // Branches that should be skipped for PRs against this job definition
    String[] prSkipBranches 
    
    def Repo(String project, 
             String[] folders,
             String branch,
             String server,
             String definitionScript,
             boolean isDefaultPRBranch,
             String[] additionalPRBranches) {
        this.project = project
        this.folders = folders
        this.branch = branch
        this.server = server
        this.definitionScript = definitionScript
        this.isDefaultPRBranch = isDefaultPRBranch
        this.additionalPRBranches = additionalPRBranches
    }    

    // Parse the input string and return a Repo object
    def static parseInputString(String input, def out) {
        // First element is the repo name.  Should be in <org>/<repo> format
        def projectInfo = input.tokenize()

        assert projectInfo.size() >= 1

        // First element is the repo name
        String project = projectInfo[0]
        String[] folders = null
        String branch = null
        // Server defaults to the primary server, dotnet-ci
        String server = 'dotnet-ci'
        // File name/path is usually netci.groovy, but can be set arbitrarily
        String definitionScript = 'netci.groovy'
        // Additional PR branches
        String[] additionalPRBranches = []
        // Is the default PR branch?
        boolean isDefaultPRBranch = false

        // Check whether it contains a single forward slash
        assert project.indexOf('/') != -1 && project.indexOf('/') == project.lastIndexOf('/')

        // Now walk the rest of the elements and set the rest of the properties
        def i = 1
        while (i < (projectInfo.size())) {
            def element = projectInfo[i]

            if (element.startsWith('folder=')) {
                // Parse out the folder names
                folders = element.substring('folder='.length()).tokenize('/')

                // If the folder name was root, just zero it out.  If they chose root, there should
                // only be one element
                assert folders.size() >= 1

                if (folders[0] == '<root>') {
                    assert folders.size() == 1
                    // Set an initial empty folder
                    folders = []
                }
            }
            else if(element.startsWith('branch=')) {
                branch = element.substring('branch='.length())
            }
            else if(element.startsWith('server=')) {
                server = element.substring('server='.length())
            }
            else if(element.startsWith('definitionScript=')) {
                definitionScript = element.substring('definitionScript='.length())
            }
            else if(element.startsWith('additionalPRBranches=')) {
                // Parse out the folder names
                additionalPRBranches = element.substring('additionalPRBranches='.length()).tokenize(',')
            }
            else if(element.startsWith('isDefaultPRBranch=')) {
                // Parse out the folder names
                isDefaultPRBranch = element.substring('isDefaultPRBranch='.length()).toBoolean()
            }
            else {
                println("Unknown element " + element);
                assert false
            }
            i++
        }

        // If the folder was unset but branch was set to something, then we set the folder to the
        // repo name plus branch subfolder

        if (folders == null) {
            folders = [Utilities.getFolderName(project)]
            if (branch == null) {
                branch = 'master'
            }
            else {
                // Append the branch name to the folder list
                folders += Utilities.getFolderName(branch)
            }
        }
        else {
            // Folder already set, but set a default value for branch if it's not specified
            // Branch name is not appended to the folder list.
            if (branch == null) {
                branch = 'master'
            }
        }
        
        // Construct a new object and return
        
        return new Repo(project, folders, branch, server, definitionScript, isDefaultPRBranch, additionalPRBranches)
    }
}

Repo[] repos = []

streamFileFromWorkspace('dotnet-ci/jobs/data/repolist.txt').eachLine { line ->
    // Skip comment lines
    boolean skip = (line ==~ / *#.*/);
    line.trim()
    skip |= (line == '')
    if (skip) {
        // Return from closure
        return;
    }

    repos += Repo.parseInputString(line, out)
}

// Post Processing
// Process each repo and determine the other tracked branches, the PR target branches,
// and the PR skip branches

repos.each { repoInfo ->
    def otherRepos = repos.findAll { searchRepoInfo -> 
        // Same project
        searchRepoInfo.project == repoInfo.project &&
        // Different branch
        searchRepoInfo.branch != repoInfo.branch
    }
    
    repoInfo.prTargetBranches = []
    repoInfo.prSkipBranches = []
    
    // Determine the prTargetBranches and prSkipBranches
    if (repoInfo.isDefaultPRBranch) {
        // If we're the default PR branch, the pr target branch is set to .*
        repoInfo.prTargetBranches = ['.*']
        repoInfo.prSkipBranches = otherRepos.branch + otherRepos.additionalPRBranches.flatten()
    }
    else {
        // Otherwise, the target branch is the branch + additional Branches
        repoInfo.prTargetBranches = repoInfo.additionalPRBranches
        repoInfo.prSkipBranches = otherRepos.branch + otherRepos.additionalPRBranches.flatten()
    }
    repoInfo.prTargetBranches += ((String[])[repoInfo.branch])
    
    repoInfo.prTargetBranches = repoInfo.prTargetBranches.toUnique()
    repoInfo.prSkipBranches = repoInfo.prSkipBranches.toUnique()
    
    println("${repoInfo.project} - ${repoInfo.branch}\n    PR Target Branches = ${repoInfo.prTargetBranches}\n     PR Skip Branches = ${repoInfo.prSkipBranches}")
}

// Now that we have all the repos, generate the jobs
repos.each { repoInfo ->

    // Determine whether we should skip this repo becuase it resides on a different server
    if (repoInfo.server != ServerName) {
        return;
    }

    // Make the folders
    def generatorFolder = ''
    for (folderElement in repoInfo.folders) {
        if (generatorFolder == '') {
            generatorFolder = folderElement
        }
        else {
            // Append a new folder
            generatorFolder += "/${folderElement}"
        }
        folder(generatorFolder) {}
    }

    // Make the PR test folder
    def generatorPRTestFolder = "${generatorFolder}/GenPRTest"

    // Create a Folder for generator PR tests under that.
    folder(generatorPRTestFolder) {}
<commit_msg>Remove toUnique usage, since it's not available in the version of Jenkins running the primary CI instance<commit_after>// This project will generate new job generators based on changes to the repolist.txt, which will
// in turn cause those generators to run over the job generation in the local repos.

import jobs.generation.Utilities

// About this generator:
//
//  This generator is designed to run every time there is a change to
//    * The dotnet-ci repo's generation dir
//
//  A rerun of this could mean any number of things:
//    * Updated repolist.txt
//    * Updated utilities
//
//  Rerunning this generator will not automatically rerun all of the generators
//  below it (by design).  Users making changes to the overall generation structure
//  should recognize when there is a functional change being made and rerun all of the
//  generated jobs.

class Repo {
    String project
    String[] folders
    String branch
    String server
    String definitionScript
    // Indicates this is a branch for 'default' PRs. ORs to branches other than the tracked branches would use jobs
    // defined here.  If true, then the generator will receive .* + branch +  for GithubPRTargetBranches with the other tracked
    // branches  in the GithubPRSkipBranches parameter.
    boolean isDefaultPRBranch
    // Indicates this branch provides PR coverage for the listed branches.  If set, the GithubPRTargetBranches will be
    // set to this branch + prsForBranches.  GithubPRSkipBranches remains empty
    String[] additionalPRBranches = []
    
    // Lazily set up data
    // Branches that should be targeted for PRs against this job definition
    String[] prTargetBranches 
    // Branches that should be skipped for PRs against this job definition
    String[] prSkipBranches 
    
    def Repo(String project, 
             String[] folders,
             String branch,
             String server,
             String definitionScript,
             boolean isDefaultPRBranch,
             String[] additionalPRBranches) {
        this.project = project
        this.folders = folders
        this.branch = branch
        this.server = server
        this.definitionScript = definitionScript
        this.isDefaultPRBranch = isDefaultPRBranch
        this.additionalPRBranches = additionalPRBranches
    }    

    // Parse the input string and return a Repo object
    def static parseInputString(String input, def out) {
        // First element is the repo name.  Should be in <org>/<repo> format
        def projectInfo = input.tokenize()

        assert projectInfo.size() >= 1

        // First element is the repo name
        String project = projectInfo[0]
        String[] folders = null
        String branch = null
        // Server defaults to the primary server, dotnet-ci
        String server = 'dotnet-ci'
        // File name/path is usually netci.groovy, but can be set arbitrarily
        String definitionScript = 'netci.groovy'
        // Additional PR branches
        String[] additionalPRBranches = []
        // Is the default PR branch?
        boolean isDefaultPRBranch = false

        // Check whether it contains a single forward slash
        assert project.indexOf('/') != -1 && project.indexOf('/') == project.lastIndexOf('/')

        // Now walk the rest of the elements and set the rest of the properties
        def i = 1
        while (i < (projectInfo.size())) {
            def element = projectInfo[i]

            if (element.startsWith('folder=')) {
                // Parse out the folder names
                folders = element.substring('folder='.length()).tokenize('/')

                // If the folder name was root, just zero it out.  If they chose root, there should
                // only be one element
                assert folders.size() >= 1

                if (folders[0] == '<root>') {
                    assert folders.size() == 1
                    // Set an initial empty folder
                    folders = []
                }
            }
            else if(element.startsWith('branch=')) {
                branch = element.substring('branch='.length())
            }
            else if(element.startsWith('server=')) {
                server = element.substring('server='.length())
            }
            else if(element.startsWith('definitionScript=')) {
                definitionScript = element.substring('definitionScript='.length())
            }
            else if(element.startsWith('additionalPRBranches=')) {
                // Parse out the folder names
                additionalPRBranches = element.substring('additionalPRBranches='.length()).tokenize(',')
            }
            else if(element.startsWith('isDefaultPRBranch=')) {
                // Parse out the folder names
                isDefaultPRBranch = element.substring('isDefaultPRBranch='.length()).toBoolean()
            }
            else {
                println("Unknown element " + element);
                assert false
            }
            i++
        }

        // If the folder was unset but branch was set to something, then we set the folder to the
        // repo name plus branch subfolder

        if (folders == null) {
            folders = [Utilities.getFolderName(project)]
            if (branch == null) {
                branch = 'master'
            }
            else {
                // Append the branch name to the folder list
                folders += Utilities.getFolderName(branch)
            }
        }
        else {
            // Folder already set, but set a default value for branch if it's not specified
            // Branch name is not appended to the folder list.
            if (branch == null) {
                branch = 'master'
            }
        }
        
        // Construct a new object and return
        
        return new Repo(project, folders, branch, server, definitionScript, isDefaultPRBranch, additionalPRBranches)
    }
}

Repo[] repos = []

streamFileFromWorkspace('dotnet-ci/jobs/data/repolist.txt').eachLine { line ->
    // Skip comment lines
    boolean skip = (line ==~ / *#.*/);
    line.trim()
    skip |= (line == '')
    if (skip) {
        // Return from closure
        return;
    }

    repos += Repo.parseInputString(line, out)
}

// Post Processing
// Process each repo and determine the other tracked branches, the PR target branches,
// and the PR skip branches

repos.each { repoInfo ->
    def otherRepos = repos.findAll { searchRepoInfo -> 
        // Same project
        searchRepoInfo.project == repoInfo.project &&
        // Different branch
        searchRepoInfo.branch != repoInfo.branch
    }
    
    repoInfo.prTargetBranches = []
    repoInfo.prSkipBranches = []
    
    // Determine the prTargetBranches and prSkipBranches
    if (repoInfo.isDefaultPRBranch) {
        // If we're the default PR branch, the pr target branch is set to .*
        repoInfo.prTargetBranches = ['.*']
        repoInfo.prSkipBranches = otherRepos.branch + otherRepos.additionalPRBranches.flatten()
    }
    else {
        // Otherwise, the target branch is the branch + additional Branches
        repoInfo.prTargetBranches = repoInfo.additionalPRBranches
        repoInfo.prSkipBranches = otherRepos.branch + otherRepos.additionalPRBranches.flatten()
    }
    repoInfo.prTargetBranches += ((String[])[repoInfo.branch])
    
    println("${repoInfo.project} - ${repoInfo.branch}\n    PR Target Branches = ${repoInfo.prTargetBranches}\n     PR Skip Branches = ${repoInfo.prSkipBranches}")
}

// Now that we have all the repos, generate the jobs
repos.each { repoInfo ->

    // Determine whether we should skip this repo becuase it resides on a different server
    if (repoInfo.server != ServerName) {
        return;
    }

    // Make the folders
    def generatorFolder = ''
    for (folderElement in repoInfo.folders) {
        if (generatorFolder == '') {
            generatorFolder = folderElement
        }
        else {
            // Append a new folder
            generatorFolder += "/${folderElement}"
        }
        folder(generatorFolder) {}
    }

    // Make the PR test folder
    def generatorPRTestFolder = "${generatorFolder}/GenPRTest"

    // Create a Folder for generator PR tests under that.
    folder(generatorPRTestFolder) {}

{'fraction_non_alphanumeric': 0.08749325822496554, 'fraction_numerical': 0.0008389764487325463, 'mean_word_length': 2.0710342289289656, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7374113', 'n_tokens_mistral': 4289, 'n_tokens_neox': 4151, 'n_words': 1446}