<commit_before>package mb.statix.concurrent.p_raffrayi.impl;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import javax.annotation.Nullable;

import org.metaborg.util.unit.Unit;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;

import mb.nabl2.util.Tuple2;
import mb.statix.concurrent.actors.IActor;
import mb.statix.concurrent.actors.IActorRef;
import mb.statix.concurrent.actors.futures.CompletableFuture;
import mb.statix.concurrent.actors.futures.IFuture;
import mb.statix.concurrent.p_raffrayi.IUnitResult;
import mb.statix.concurrent.p_raffrayi.impl.tokens.Query;
import mb.statix.concurrent.p_raffrayi.nameresolution.DataLeq;
import mb.statix.concurrent.p_raffrayi.nameresolution.DataWf;
import mb.statix.concurrent.p_raffrayi.nameresolution.LabelOrder;
import mb.statix.concurrent.p_raffrayi.nameresolution.LabelWf;
import mb.statix.scopegraph.IScopeGraph;
import mb.statix.scopegraph.IScopeGraph.Immutable;
import mb.statix.scopegraph.reference.EdgeOrData;
import mb.statix.scopegraph.reference.Env;
import mb.statix.scopegraph.terms.newPath.ScopePath;

class StaticScopeGraphUnit<S, L, D> extends AbstractUnit<S, L, D, Unit> {

    private static final int PARALLEL_WORKERS = 8;

    protected List<S> givenRootScopes;
    protected Set<S> givenOwnScopes;
    protected IScopeGraph.Immutable<S, L, D> givenScopeGraph;

    private final List<IActorRef<? extends IUnit<S, L, D, Unit>>> workers;

    StaticScopeGraphUnit(IActor<? extends IUnit<S, L, D, Unit>> self,
            @Nullable IActorRef<? extends IUnit<S, L, D, ?>> parent, IUnitContext<S, L, D> context,
            Iterable<L> edgeLabels, List<S> givenRootScopes, Set<S> givenOwnScopes,
            IScopeGraph.Immutable<S, L, D> givenScopeGraph) {
        super(self, parent, context, edgeLabels);

        // these are replaced once started
        this.givenRootScopes = ImmutableList.copyOf(givenRootScopes);
        this.givenOwnScopes = ImmutableSet.copyOf(givenOwnScopes);
        this.givenScopeGraph = givenScopeGraph;

        this.workers = new ArrayList<>();
    }

    protected void clearGiven() {
        this.givenRootScopes = null;
        this.givenOwnScopes = null;
        this.givenScopeGraph = null;
    }

    @SuppressWarnings("unused") @Override protected IFuture<D> getExternalDatum(D datum) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    ///////////////////////////////////////////////////////////////////////////
    // IBroker2UnitProtocol interface, called by IBroker implementations
    ///////////////////////////////////////////////////////////////////////////

    @Override public IFuture<IUnitResult<S, L, D, Unit>> _start(List<S> rootScopes) {
        doStart(rootScopes);

        final List<EdgeOrData<L>> edges = edgeLabels.stream().map(EdgeOrData::edge).collect(Collectors.toList());

        final Map<S, S> scopeMap = new HashMap<>();
        final Set<S> ownScopes = new HashSet<>();

        if(givenRootScopes.size() != rootScopes.size()) {
            throw new IllegalArgumentException("Number of root scopes does not match.");
        }
        for(int i = 0; i < rootScopes.size(); i++) {
            final S libRootScope = givenRootScopes.get(i);
            if(scopeMap.containsKey(libRootScope)) {
                continue;
            }
            final S rootScope = rootScopes.get(i);
            scopeMap.put(libRootScope, rootScope);
            doInitShare(self, rootScope, edges, false);
        }

        for(S libScope : givenOwnScopes) {
            if(scopeMap.containsKey(libScope)) {
                throw new IllegalStateException("Scope already initialized.");
            }
            final S scope = doFreshScope("s"/*libScope.getName()*/, edgeLabels,
                    givenScopeGraph.getData(libScope).isPresent(), false);
            ownScopes.add(scope);
            scopeMap.put(libScope, scope);
        }

        for(S libScope : Iterables.concat(givenRootScopes, givenOwnScopes)) {
            final S scope = scopeMap.get(libScope);

            final D libDatum;
            if((libDatum = givenScopeGraph.getData(libScope).orElse(null)) != null) {
                final D datum = context.substituteScopes(libDatum, scopeMap);
                doSetDatum(scope, datum);
            }

            for(L label : edgeLabels) {
                final EdgeOrData<L> l = EdgeOrData.edge(label);
                for(S libTarget : givenScopeGraph.getEdges(libScope, label)) {
                    final S target = scopeMap.get(libTarget);
                    doAddEdge(self, scope, label, target);
                }
                doCloseLabel(self, scope, l);
            }
        }

        clearGiven();

        for(int i = 0; i < PARALLEL_WORKERS; i++) {
            final Tuple2<IFuture<IUnitResult<S, L, D, Unit>>, IActorRef<? extends IUnit<S, L, D, Unit>>> worker =
                    context.<Unit>add("worker-" + i, (subself, subcontext) -> {
                        return new StaticScopeGraphWorker<>(subself, self, subcontext, edgeLabels, rootScopes,
                                ownScopes, scopeGraph.get());
                    }, Collections.emptyList());
            workers.add(worker._2());
        }

        return doFinish(CompletableFuture.completedFuture(Unit.unit));
    }

    ///////////////////////////////////////////////////////////////////////////
    // IUnit2UnitProtocol interface, called by IUnit implementations
    ///////////////////////////////////////////////////////////////////////////

    @SuppressWarnings("unused") @Override public void _initShare(S scope, Iterable<EdgeOrData<L>> edges,
            boolean sharing) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    @SuppressWarnings("unused") @Override public void _addShare(S scope) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    @SuppressWarnings("unused") @Override public void _doneSharing(S scope) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    @SuppressWarnings("unused") @Override public void _addEdge(S source, L label, S target) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    @SuppressWarnings("unused") @Override public void _closeEdge(S scope, EdgeOrData<L> edge) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");<commit_msg>Speed up library loading by skipping local wait-for admin.<commit_after>package mb.statix.concurrent.p_raffrayi.impl;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import javax.annotation.Nullable;

import org.metaborg.util.log.ILogger;
import org.metaborg.util.log.LoggerUtils;
import org.metaborg.util.unit.Unit;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

import mb.nabl2.util.Tuple2;
import mb.statix.concurrent.actors.IActor;
import mb.statix.concurrent.actors.IActorRef;
import mb.statix.concurrent.actors.futures.CompletableFuture;
import mb.statix.concurrent.actors.futures.IFuture;
import mb.statix.concurrent.p_raffrayi.IUnitResult;
import mb.statix.concurrent.p_raffrayi.impl.tokens.Query;
import mb.statix.concurrent.p_raffrayi.nameresolution.DataLeq;
import mb.statix.concurrent.p_raffrayi.nameresolution.DataWf;
import mb.statix.concurrent.p_raffrayi.nameresolution.LabelOrder;
import mb.statix.concurrent.p_raffrayi.nameresolution.LabelWf;
import mb.statix.scopegraph.IScopeGraph;
import mb.statix.scopegraph.IScopeGraph.Immutable;
import mb.statix.scopegraph.reference.EdgeOrData;
import mb.statix.scopegraph.reference.Env;
import mb.statix.scopegraph.terms.newPath.ScopePath;

class StaticScopeGraphUnit<S, L, D> extends AbstractUnit<S, L, D, Unit> {

    private static final ILogger logger = LoggerUtils.logger(StaticScopeGraphUnit.class);

    private static final int PARALLEL_WORKERS = 8;

    protected List<S> givenRootScopes;
    protected Set<S> givenOwnScopes;
    protected IScopeGraph.Immutable<S, L, D> givenScopeGraph;

    private final List<IActorRef<? extends IUnit<S, L, D, Unit>>> workers;

    StaticScopeGraphUnit(IActor<? extends IUnit<S, L, D, Unit>> self,
            @Nullable IActorRef<? extends IUnit<S, L, D, ?>> parent, IUnitContext<S, L, D> context,
            Iterable<L> edgeLabels, List<S> givenRootScopes, Set<S> givenOwnScopes,
            IScopeGraph.Immutable<S, L, D> givenScopeGraph) {
        super(self, parent, context, edgeLabels);

        // these are replaced once started
        this.givenRootScopes = ImmutableList.copyOf(givenRootScopes);
        this.givenOwnScopes = ImmutableSet.copyOf(givenOwnScopes);
        this.givenScopeGraph = givenScopeGraph;

        this.workers = new ArrayList<>();
    }

    protected void clearGiven() {
        this.givenRootScopes = null;
        this.givenOwnScopes = null;
        this.givenScopeGraph = null;
    }

    @SuppressWarnings("unused") @Override protected IFuture<D> getExternalDatum(D datum) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    ///////////////////////////////////////////////////////////////////////////
    // IBroker2UnitProtocol interface, called by IBroker implementations
    ///////////////////////////////////////////////////////////////////////////

    @Override public IFuture<IUnitResult<S, L, D, Unit>> _start(List<S> rootScopes) {
        doStart(rootScopes);

        final Set<S> ownScopes = buildScopeGraph(rootScopes);

        clearGiven();

        startWorkers(rootScopes, ownScopes);

        return doFinish(CompletableFuture.completedFuture(Unit.unit));
    }

    private Set<S> buildScopeGraph(List<S> rootScopes) {
        final long t0 = System.currentTimeMillis();

        final List<EdgeOrData<L>> edges = edgeLabels.stream().map(EdgeOrData::edge).collect(Collectors.toList());

        final Map<S, S> scopeMap = new HashMap<>();
        final Set<S> ownScopes = new HashSet<>();

        // map given scopes to actual scopes

        if(givenRootScopes.size() != rootScopes.size()) {
            throw new IllegalArgumentException("Number of root scopes does not match.");
        }
        for(int i = 0; i < rootScopes.size(); i++) {
            final S libRootScope = givenRootScopes.get(i);
            if(scopeMap.containsKey(libRootScope)) {
                continue;
            }
            final S rootScope = rootScopes.get(i);
            scopeMap.put(libRootScope, rootScope);
            doInitShare(self, rootScope, edges, false);
        }
        for(S libScope : givenOwnScopes) {
            if(scopeMap.containsKey(libScope)) {
                throw new IllegalStateException("Scope already initialized.");
            }
            final S scope = makeScope("s"/*libScope.getName()*/);
            ownScopes.add(scope);
            scopeMap.put(libScope, scope);
        }

        // add data and edges to actual scopes

        for(S libScope : givenRootScopes) {
            final S scope = scopeMap.get(libScope);

            for(L label : edgeLabels) {
                final EdgeOrData<L> l = EdgeOrData.edge(label);
                for(S libTarget : givenScopeGraph.getEdges(libScope, label)) {
                    final S target = scopeMap.get(libTarget);
                    doAddEdge(self, scope, label, target);
                }
                doCloseLabel(self, scope, l);
            }
        }
        for(S libScope : givenOwnScopes) {
            final S scope = scopeMap.get(libScope);

            final D libDatum;
            if((libDatum = givenScopeGraph.getData(libScope).orElse(null)) != null) {
                final D datum = context.substituteScopes(libDatum, scopeMap);
                scopeGraph.set(scopeGraph.get().setDatum(scope, datum));
            }

            for(L label : edgeLabels) {
                for(S libTarget : givenScopeGraph.getEdges(libScope, label)) {
                    final S target = scopeMap.get(libTarget);
                    scopeGraph.set(scopeGraph.get().addEdge(scope, label, target));
                }
            }
        }

        final long dt = System.currentTimeMillis() - t0;
        logger.info("Initialized {} in {} s", self.id(), TimeUnit.SECONDS.convert(dt, TimeUnit.MILLISECONDS));

        return ownScopes;
    }

    private void startWorkers(List<S> rootScopes, final Set<S> ownScopes) {
        for(int i = 0; i < PARALLEL_WORKERS; i++) {
            final Tuple2<IFuture<IUnitResult<S, L, D, Unit>>, IActorRef<? extends IUnit<S, L, D, Unit>>> worker =
                    context.<Unit>add("worker-" + i, (subself, subcontext) -> {
                        return new StaticScopeGraphWorker<>(subself, self, subcontext, edgeLabels, rootScopes,
                                ownScopes, scopeGraph.get());
                    }, Collections.emptyList());
            workers.add(worker._2());
        }
    }

    ///////////////////////////////////////////////////////////////////////////
    // IUnit2UnitProtocol interface, called by IUnit implementations
    ///////////////////////////////////////////////////////////////////////////

    @SuppressWarnings("unused") @Override public void _initShare(S scope, Iterable<EdgeOrData<L>> edges,
            boolean sharing) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    @SuppressWarnings("unused") @Override public void _addShare(S scope) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    @SuppressWarnings("unused") @Override public void _doneSharing(S scope) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    @SuppressWarnings("unused") @Override public void _addEdge(S source, L label, S target) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
    }

    @SuppressWarnings("unused") @Override public void _closeEdge(S scope, EdgeOrData<L> edge) {
        throw new UnsupportedOperationException("Not supported by static scope graph units.");
{'fraction_non_alphanumeric': 0.14383794439535041, 'fraction_numerical': 0.0013595268846441438, 'mean_word_length': 2.507868383404864, 'pattern_counts': {'":': 0, '<': 90, '<?xml version=': 0, '>': 88, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '144817', 'n_tokens_mistral': 4189, 'n_tokens_neox': 4081, 'n_words': 915}