<commit_before>package repo.build.maven

import groovy.transform.CompileStatic
import repo.build.ActionContext
import repo.build.ComponentDependencyGraph
import repo.build.Maven
import repo.build.RepoBuildException

import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentMap
import java.util.concurrent.Executors

/**
 * @author <NAME> <EMAIL>
 */
//@CompileStatic
class Build {
    Collection<MavenComponent> components
    ActionContext context

    Build(ActionContext context, Collection<MavenComponent> components) {
        this.context = context
        this.components = components
    }

    boolean execute(ActionContext context) {
        // check circular dependencies
        def graph = ComponentDependencyGraph.build(components)
        if (graph.hasCycles()) {
            for (def component : graph.findCycles()) {
                def componentCycle = graph.findCycles(component).collect { it.path }
                def error = new RepoBuildException("Component ${component.@path} has circular refs in cycle ${componentCycle}")
                context.addError(error)
            }
            throw new RepoBuildException("project has circular dependencies")
        }

        Map<MavenArtifactRef, MavenComponent> componentMap = components.collectEntries {
            [new MavenArtifactRef(it), it]
        }

        Map<MavenArtifactRef, BuildState> buildStates = componentMap.collectEntries {
            [it.key, BuildState.NEW]
        } as Map<MavenArtifactRef, BuildState>

        Map<MavenArtifactRef, MavenComponent> moduleToComponentMap =
                ComponentDependencyGraph.getModuleToComponentMap(components)

        ConcurrentMap<MavenArtifactRef, List<MavenArtifactRef>> buildDeps = new ConcurrentHashMap<>()
        buildStates.keySet().forEach { MavenArtifactRef key ->
            def depsTasks = componentMap.get(key).modules
                    .collectMany { it.dependencies }
                    .findAll { moduleToComponentMap.containsKey(it) }
                    .unique()
                    .collect { new MavenArtifactRef(moduleToComponentMap.get(it)) }
                    .findAll { !key.equals(it) }
            buildDeps.put(key, depsTasks)
            return
        }

        Set<MavenArtifactRef> tasks = new HashSet<>(buildStates.keySet())

        def pool = Executors.newFixedThreadPool(context.getParallel())
        while (!tasks.isEmpty()) {
            def iter = tasks.iterator()
            while (iter.hasNext()) {
                def key = iter.next()
                def component = componentMap.get(key)
                def deps = buildDeps.get(key)
                if (!deps.any { buildStates.get(it) != BuildState.SUCCESS }) {
                    pool.execute {
                        // build component
                        try {
                            def pomFile = new File(component.basedir, 'pom.xml')
                            Maven.execute(context, pomFile, ['clean', 'install'])
                            buildStates.put(key, BuildState.SUCCESS)
                        } catch (Exception e) {
                            buildStates.put(key, BuildState.ERROR)
                            context.setErrorFlag()
                            context.addError(new RepoBuildException(" component ${component.path} build ERROR", e))
                        }
                    }
                    iter.remove()
                } else if (deps.any {
                    buildStates.get(it) == BuildState.DEPS_ERROR ||
                            buildStates.get(it) == BuildState.ERROR
                }) {
                    buildStates.put(key, BuildState.DEPS_ERROR)
                    context.setErrorFlag()
                    context.addError(new RepoBuildException(" component ${component.path} build DEPS_ERROR"))
                    iter.remove()
                }
            }
            // throttle cpu
            Thread.sleep(500L)
        }
        return !context.getErrorFlag()
    }
}<commit_msg>#22 fix shutdown logic<commit_after>package repo.build.maven

import groovy.transform.CompileStatic
import repo.build.ActionContext
import repo.build.ComponentDependencyGraph
import repo.build.Maven
import repo.build.RepoBuildException

import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentMap
import java.util.concurrent.Executors
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit

/**
 * @author <NAME> <EMAIL>
 */
//@CompileStatic
class Build {
    Collection<MavenComponent> components
    ActionContext context

    Build(ActionContext context, Collection<MavenComponent> components) {
        this.context = context
        this.components = components
    }

    boolean execute(ActionContext context) {
        // check circular dependencies
        def graph = ComponentDependencyGraph.build(components)
        if (graph.hasCycles()) {
            for (def component : graph.findCycles()) {
                def componentCycle = graph.findCycles(component).collect { it.path }
                def error = new RepoBuildException("Component ${component.@path} has circular refs in cycle ${componentCycle}")
                context.addError(error)
            }
            throw new RepoBuildException("project has circular dependencies")
        }

        Map<MavenArtifactRef, MavenComponent> componentMap = components.collectEntries {
            [new MavenArtifactRef(it), it]
        }

        Map<MavenArtifactRef, BuildState> buildStates = componentMap.collectEntries {
            [it.key, BuildState.NEW]
        } as Map<MavenArtifactRef, BuildState>

        Map<MavenArtifactRef, MavenComponent> moduleToComponentMap =
                ComponentDependencyGraph.getModuleToComponentMap(components)

        ConcurrentMap<MavenArtifactRef, List<MavenArtifactRef>> buildDeps = new ConcurrentHashMap<>()
        buildStates.keySet().forEach { MavenArtifactRef key ->
            def depsTasks = componentMap.get(key).modules
                    .collectMany { it.dependencies }
                    .findAll { moduleToComponentMap.containsKey(it) }
                    .unique()
                    .collect { new MavenArtifactRef(moduleToComponentMap.get(it)) }
                    .findAll { !key.equals(it) }
            buildDeps.put(key, depsTasks)
            return
        }

        Set<MavenArtifactRef> tasks = new HashSet<>(buildStates.keySet())

        def executionQueue = new LinkedBlockingQueue<Runnable>()
        def pool = new ThreadPoolExecutor(context.getParallel(), context.getParallel(),
                0L, TimeUnit.MILLISECONDS,
                executionQueue)
        try {
            while (!tasks.isEmpty()) {
                def iter = tasks.iterator()
                while (iter.hasNext()) {
                    def key = iter.next()
                    def component = componentMap.get(key)
                    def deps = buildDeps.get(key)
                    if (!deps.any { buildStates.get(it) != BuildState.SUCCESS }) {
                        pool.execute {
                            // build component
                            try {
                                def pomFile = new File(component.basedir, 'pom.xml')
                                try {
                                    Maven.execute(context, pomFile, ['clean'])
                                } catch (RepoBuildException ignore) {
                                }
                                Maven.execute(context, pomFile, ['install'])
                                buildStates.put(key, BuildState.SUCCESS)
                            } catch (Exception e) {
                                buildStates.put(key, BuildState.ERROR)
                                context.setErrorFlag()
                                context.addError(new RepoBuildException(" component ${component.path} build ERROR", e))
                            }
                        }
                        iter.remove()
                    } else if (deps.any {
                        buildStates.get(it) == BuildState.DEPS_ERROR ||
                                buildStates.get(it) == BuildState.ERROR
                    }) {
                        buildStates.put(key, BuildState.DEPS_ERROR)
                        context.setErrorFlag()
                        context.addError(new RepoBuildException(" component ${component.path} build DEPS_ERROR"))
                        iter.remove()
                    }
                }
                // throttle cpu
                Thread.sleep(500L)
            }
        } finally {
            pool.shutdown()
        }
        return !context.getErrorFlag()
    }
}
{'fraction_non_alphanumeric': 0.08829545454545455, 'fraction_numerical': 0.0010227272727272728, 'mean_word_length': 1.6955589586523736, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2549905', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2096, 'n_words': 417}