<commit_before>package com.beust.kobalt.app.remote

import com.beust.kobalt.Args
import com.beust.kobalt.api.IClasspathDependency
import com.beust.kobalt.api.Project
import com.beust.kobalt.app.BuildFileCompiler
import com.beust.kobalt.internal.DynamicGraph
import com.beust.kobalt.internal.GraphUtil
import com.beust.kobalt.internal.PluginInfo
import com.beust.kobalt.internal.TaskManager
import com.beust.kobalt.internal.build.BuildFile
import com.beust.kobalt.maven.DependencyManager
import com.beust.kobalt.maven.dependency.FileDependency
import com.beust.kobalt.misc.KFiles
import com.beust.kobalt.misc.KobaltExecutors
import com.beust.kobalt.misc.Versions
import com.beust.kobalt.misc.log
import com.google.inject.Inject
import java.io.File
import java.nio.file.Paths

interface IProgressListener {
    /**
     * progress is an integer between 0 and 100 that represents the percentage.
     */
    fun onProgress(progress: Int? = null, message: String? = null)
}

class RemoteDependencyData @Inject constructor(val executors: KobaltExecutors, val dependencyManager: DependencyManager,
        val buildFileCompilerFactory: BuildFileCompiler.IFactory, val pluginInfo: PluginInfo,
        val taskManager: TaskManager) {

    fun dependenciesDataFor(buildFilePath: String, args: Args, progressListener: IProgressListener? = null,
            useGraph : Boolean = false): GetDependenciesData {
        val projectDatas = arrayListOf<ProjectData>()

        fun toDependencyData(d: IClasspathDependency, scope: String): DependencyData {
            val dep = dependencyManager.create(d.id)
            return DependencyData(d.id, scope, dep.jarFile.get().absolutePath)
        }

        fun allDeps(l: List<IClasspathDependency>, name: String) = dependencyManager.transitiveClosure(l,
                requiredBy = name)

        val buildFile = BuildFile(Paths.get(buildFilePath), "GetDependenciesCommand")
        val buildFileCompiler = buildFileCompilerFactory.create(listOf(buildFile), pluginInfo)
        val projectResult = buildFileCompiler.compileBuildFiles(args)

        val pluginDependencies = projectResult.pluginUrls.map { File(it.toURI()) }.map {
            FileDependency(it.absolutePath)
        }

        fun compileDependencies(project: Project, name: String): List<DependencyData> {
            val result =
                    (pluginDependencies +
                    allDeps(project.compileDependencies, name) +
                    allDeps(project.compileProvidedDependencies, name))
                .map { toDependencyData(it, "compile") }
            return result
        }

        fun toDependencyData2(scope: String, node: DynamicGraph.Companion.Node<IClasspathDependency>): DependencyData {
            val d = node.value
            val dep = dependencyManager.create(d.id)
            return DependencyData(d.id, scope, dep.jarFile.get().absolutePath,
                    children = node.children.map { toDependencyData2(scope, it) })
        }

        val OPTIONAL_FILTER = { dep: IClasspathDependency -> ! dep.optional }

        fun compileDependenciesGraph(project: Project, name: String): List<DependencyData> {
            val depLambda = IClasspathDependency::directDependencies
            val result =
                    (DynamicGraph.Companion.transitiveClosureGraph(pluginDependencies, depLambda, OPTIONAL_FILTER) +
                    DynamicGraph.Companion.transitiveClosureGraph(project.compileDependencies, depLambda,
                            OPTIONAL_FILTER) +
                    DynamicGraph.Companion.transitiveClosureGraph(project.compileProvidedDependencies, depLambda,
                            OPTIONAL_FILTER))
                .map { toDependencyData2("compile", it)}

            fun mapOfLatestVersions(l: List<DependencyData>) : Map<String, String> {
                fun p(l: List<DependencyData>, latestVersions: java.util.HashMap<String, String>) {
                    l.forEach {
                        val mid = dependencyManager.create(it.id)
                        if (mid.isMaven) {
                            val shortId = mid.shortId
                            val currentLatest = latestVersions[shortId]
                            if (currentLatest == null) latestVersions[shortId] = mid.version!!
                            else mid.version?.let { v ->
                                if (Versions.toLongVersion(currentLatest) < Versions.toLongVersion(v)) {
                                    latestVersions[shortId] = v
                                }
                            }
                        }
                        p(it.children, latestVersions)
                    }
                }
                val result = hashMapOf<String, String>()
                p(l, result)
                return result
            }
            val map = mapOfLatestVersions(result)
            GraphUtil.map(result, { d: DependencyData -> d.children },
                    {d: DependencyData ->
                        val mid = dependencyManager.create(d.id)
                        if (mid.isMaven) {
                            val version = map[mid.shortId]
                            d.isLatest = version == mid.version
                        }
                    })
            return result
        }

        fun testDependencies(project: Project, name: String): List<DependencyData> {
            return allDeps(project.testDependencies, name).map { toDependencyData(it, "testCompile") }
        }

        fun testDependenciesGraph(project: Project, name: String): List<DependencyData> {
            val depLambda = { dep : IClasspathDependency -> dep.directDependencies() }
            return DynamicGraph.Companion.transitiveClosureGraph(project.testDependencies, depLambda, OPTIONAL_FILTER)
                    .map { toDependencyData2("testCompile", it)}
        }

        val allTasks = hashSetOf<TaskData>()
        projectResult.projects.withIndex().forEach { wi ->
            val project = wi.value
            val name = project.name
            progressListener?.onProgress(message = "Synchronizing project ${project.name} "
                    + (wi.index + 1) + "/" + projectResult.projects.size)

            val dependentProjects = project.dependsOn.map { it.name }

            // Separate resource from source directories
            fun partition(project: Project, dirs: Collection<String>)
                    = dirs.filter { File(project.directory, it).exists() }
                    .partition { KFiles.isResource(it) }
            val sources = partition(project, project.sourceDirectories)
            val tests = partition(project, project.sourceDirectoriesTest)

            val projectTasks = taskManager.tasksByNames(project).values().map {
                TaskData(it.name, it.doc, it.group)
            }
<commit_msg>Simplify.<commit_after>package com.beust.kobalt.app.remote

import com.beust.kobalt.Args
import com.beust.kobalt.api.IClasspathDependency
import com.beust.kobalt.api.Project
import com.beust.kobalt.app.BuildFileCompiler
import com.beust.kobalt.internal.DynamicGraph
import com.beust.kobalt.internal.GraphUtil
import com.beust.kobalt.internal.PluginInfo
import com.beust.kobalt.internal.TaskManager
import com.beust.kobalt.internal.build.BuildFile
import com.beust.kobalt.maven.DependencyManager
import com.beust.kobalt.maven.dependency.FileDependency
import com.beust.kobalt.misc.KFiles
import com.beust.kobalt.misc.KobaltExecutors
import com.beust.kobalt.misc.Versions
import com.beust.kobalt.misc.log
import com.google.inject.Inject
import java.io.File
import java.nio.file.Paths

interface IProgressListener {
    /**
     * progress is an integer between 0 and 100 that represents the percentage.
     */
    fun onProgress(progress: Int? = null, message: String? = null)
}

class RemoteDependencyData @Inject constructor(val executors: KobaltExecutors, val dependencyManager: DependencyManager,
        val buildFileCompilerFactory: BuildFileCompiler.IFactory, val pluginInfo: PluginInfo,
        val taskManager: TaskManager) {

    fun dependenciesDataFor(buildFilePath: String, args: Args, progressListener: IProgressListener? = null,
            useGraph : Boolean = false): GetDependenciesData {
        val projectDatas = arrayListOf<ProjectData>()

        fun toDependencyData(d: IClasspathDependency, scope: String): DependencyData {
            val dep = dependencyManager.create(d.id)
            return DependencyData(d.id, scope, dep.jarFile.get().absolutePath)
        }

        fun allDeps(l: List<IClasspathDependency>, name: String) = dependencyManager.transitiveClosure(l,
                requiredBy = name)

        val buildFile = BuildFile(Paths.get(buildFilePath), "GetDependenciesCommand")
        val buildFileCompiler = buildFileCompilerFactory.create(listOf(buildFile), pluginInfo)
        val projectResult = buildFileCompiler.compileBuildFiles(args)

        val pluginDependencies = projectResult.pluginUrls.map { File(it.toURI()) }.map {
            FileDependency(it.absolutePath)
        }

        fun compileDependencies(project: Project, name: String): List<DependencyData> {
            val result =
                    (pluginDependencies +
                    allDeps(project.compileDependencies, name) +
                    allDeps(project.compileProvidedDependencies, name))
                .map { toDependencyData(it, "compile") }
            return result
        }

        fun toDependencyData2(scope: String, node: DynamicGraph.Companion.Node<IClasspathDependency>): DependencyData {
            val d = node.value
            val dep = dependencyManager.create(d.id)
            return DependencyData(d.id, scope, dep.jarFile.get().absolutePath,
                    children = node.children.map { toDependencyData2(scope, it) })
        }

        val OPTIONAL_FILTER = { dep: IClasspathDependency -> ! dep.optional }

        fun compileDependenciesGraph(project: Project, name: String): List<DependencyData> {
            val depLambda = IClasspathDependency::directDependencies
            val result =
                    (DynamicGraph.Companion.transitiveClosureGraph(pluginDependencies, depLambda, OPTIONAL_FILTER) +
                    DynamicGraph.Companion.transitiveClosureGraph(project.compileDependencies, depLambda,
                            OPTIONAL_FILTER) +
                    DynamicGraph.Companion.transitiveClosureGraph(project.compileProvidedDependencies, depLambda,
                            OPTIONAL_FILTER))
                .map { toDependencyData2("compile", it)}

            fun mapOfLatestVersions(l: List<DependencyData>) : Map<String, String> {
                fun p(l: List<DependencyData>, latestVersions: java.util.HashMap<String, String>) {
                    l.forEach {
                        val mid = dependencyManager.create(it.id)
                        if (mid.isMaven) {
                            val shortId = mid.shortId
                            val currentLatest = latestVersions[shortId]
                            if (currentLatest == null) latestVersions[shortId] = mid.version!!
                            else mid.version?.let { v ->
                                if (Versions.toLongVersion(currentLatest) < Versions.toLongVersion(v)) {
                                    latestVersions[shortId] = v
                                }
                            }
                        }
                        p(it.children, latestVersions)
                    }
                }
                val result = hashMapOf<String, String>()
                p(l, result)
                return result
            }
            val map = mapOfLatestVersions(result)
            GraphUtil.map(result, { d: DependencyData -> d.children },
                    {d: DependencyData ->
                        val mid = dependencyManager.create(d.id)
                        if (mid.isMaven) {
                            val version = map[mid.shortId]
                            d.isLatest = version == mid.version
                        }
                    })
            return result
        }

        fun testDependencies(project: Project, name: String): List<DependencyData> {
            return allDeps(project.testDependencies, name).map { toDependencyData(it, "testCompile") }
        }

        fun testDependenciesGraph(project: Project, name: String): List<DependencyData> {
            val depLambda = IClasspathDependency::directDependencies
            return DynamicGraph.Companion.transitiveClosureGraph(project.testDependencies, depLambda, OPTIONAL_FILTER)
                    .map { toDependencyData2("testCompile", it)}
        }

        val allTasks = hashSetOf<TaskData>()
        projectResult.projects.withIndex().forEach { wi ->
            val project = wi.value
            val name = project.name
            progressListener?.onProgress(message = "Synchronizing project ${project.name} "
                    + (wi.index + 1) + "/" + projectResult.projects.size)

            val dependentProjects = project.dependsOn.map { it.name }

            // Separate resource from source directories
            fun partition(project: Project, dirs: Collection<String>)
                    = dirs.filter { File(project.directory, it).exists() }
                    .partition { KFiles.isResource(it) }
            val sources = partition(project, project.sourceDirectories)
            val tests = partition(project, project.sourceDirectoriesTest)

            val projectTasks = taskManager.tasksByNames(project).values().map {
                TaskData(it.name, it.doc, it.group)
            }

{'fraction_non_alphanumeric': 0.08760777436796727, 'fraction_numerical': 0.00131521262604121, 'mean_word_length': 2.127742230347349, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '938435', 'n_tokens_mistral': 3628, 'n_tokens_neox': 3417, 'n_words': 686}