<commit_before>package org.javacs;

import com.google.common.collect.Maps;
import com.sun.source.tree.*;
import com.sun.source.util.*;
import com.sun.tools.javac.api.JavacTool;
import com.sun.tools.javac.file.JavacFileManager;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.logging.Logger;
import java.util.stream.Stream;
import javax.lang.model.element.*;
import javax.tools.Diagnostic;
import javax.tools.JavaFileObject;
import org.eclipse.lsp4j.*;

/**
 * Global index of exported symbol declarations and references. such as classes, methods, and
 * fields.
 */
public class SymbolIndex {

    private final Path workspaceRoot;
    private final Supplier<Collection<URI>> openFiles;
    private final Function<URI, Optional<String>> activeContent;
    private final JavacTool parser = JavacTool.create();
    private final JavacFileManager emptyFileManager =
            parser.getStandardFileManager(__ -> {}, null, Charset.defaultCharset());

    /** Source path files, for which we support methods and classes */
    private final Map<URI, SourceFileIndex> sourcePathFiles = new ConcurrentHashMap<>();

    private final CompletableFuture<?> finishedInitialIndex = new CompletableFuture<>();

    SymbolIndex(
            Path workspaceRoot,
            Supplier<Collection<URI>> openFiles,
            Function<URI, Optional<String>> activeContent) {
        this.workspaceRoot = workspaceRoot;
        this.openFiles = openFiles;
        this.activeContent = activeContent;

        Runnable doIndex =
                () -> {
                    updateIndex(InferConfig.allJavaFiles(workspaceRoot).map(Path::toUri));

                    finishedInitialIndex.complete(null);
                };

        new Thread(doIndex, "Initial-Index").start();
    }

    private void updateIndex(Stream<URI> files) {
        // TODO send a progress bar to the user
        files.forEach(
                each -> {
                    if (needsUpdate(each)) {
                        CompilationUnitTree tree = parse(each);

                        update(tree);
                    }
                });
    }

    private boolean needsUpdate(URI file) {
        if (!sourcePathFiles.containsKey(file)) return true;
        else
            try {
                Instant updated = sourcePathFiles.get(file).updated;
                Instant modified = Files.getLastModifiedTime(Paths.get(file)).toInstant();

                return updated.isBefore(modified);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
    }

    /**
     * Guess the source path by looking at package declarations in .java files.
     *
     * <p>For example, if the file src/com/example/Test.java has the package declaration `package
     * com.example;` then the source root is `src`.
     */
    Set<Path> sourcePath() {
        updateOpenFiles();

        Set<Path> result = new HashSet<>();

        sourcePathFiles.forEach(
                (uri, index) -> {
                    Path dir = Paths.get(uri).getParent();
                    String packagePath = index.packageName.replace('.', File.separatorChar);

                    if (!dir.endsWith(packagePath))
                        LOG.warning("Java source file " + uri + " is not in " + packagePath);
                    else {
                        int up = Paths.get(packagePath).getNameCount();
                        Path truncate = dir;

                        for (int i = 0; i < up; i++) truncate = truncate.getParent();

                        result.add(truncate);
                    }
                });

        return result;
    }

    /** Search all indexed symbols */
    public Stream<SymbolInformation> search(String query) {
        updateOpenFiles();

        Predicate<URI> matchesQuery =
                uri ->
                        sourcePathFiles
                                .get(uri)
                                .declarations
                                .stream()
                                .anyMatch(
                                        name ->
                                                Completions.containsCharactersInOrder(
                                                        name, query, true));
        Collection<URI> open = openFiles.get();
        Stream<URI> openFirst =
                Stream.concat(
                        open.stream(),
                        sourcePathFiles.keySet().stream().filter(uri -> !open.contains(uri)));

        return openFirst
                .filter(matchesQuery)
                .flatMap(this::allInFile)
                .filter(info -> Completions.containsCharactersInOrder(info.getName(), query, true));
    }

    void updateOpenFiles() {
        finishedInitialIndex.join();

        updateIndex(openFiles.get().stream());
    }

    /** Get all declarations in an open file */
    Stream<SymbolInformation> allInFile(URI source) {
        LOG.info("Search " + source);

        JavacTask task = parseTask(source);
        Trees trees = Trees.instance(task);

        class FindDeclarations extends TreePathScanner<Void, Void> {
            List<SymbolInformation> result = new ArrayList<>();
            Optional<ClassTree> currentClass = Optional.empty();

            @Override
            public Void visitClass(ClassTree node, Void aVoid) {
                String name = node.getSimpleName().toString();
                SymbolInformation info = new SymbolInformation();

                info.setContainerName(packageName());<commit_msg>Simplify<commit_after>package org.javacs;

import com.google.common.collect.Maps;
import com.sun.source.tree.*;
import com.sun.source.util.*;
import com.sun.tools.javac.api.JavacTool;
import com.sun.tools.javac.file.JavacFileManager;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.logging.Logger;
import java.util.stream.Stream;
import javax.lang.model.element.*;
import javax.tools.Diagnostic;
import javax.tools.JavaFileObject;
import org.eclipse.lsp4j.*;

/**
 * Global index of exported symbol declarations and references. such as classes, methods, and
 * fields.
 */
public class SymbolIndex {

    private final Path workspaceRoot;
    private final Supplier<Collection<URI>> openFiles;
    private final Function<URI, Optional<String>> activeContent;
    private final JavacTool parser = JavacTool.create();
    private final JavacFileManager emptyFileManager =
            parser.getStandardFileManager(__ -> {}, null, Charset.defaultCharset());

    /** Source path files, for which we support methods and classes */
    private final Map<URI, SourceFileIndex> sourcePathFiles = new ConcurrentHashMap<>();

    private final CompletableFuture<?> finishedInitialIndex = new CompletableFuture<>();

    SymbolIndex(
            Path workspaceRoot,
            Supplier<Collection<URI>> openFiles,
            Function<URI, Optional<String>> activeContent) {
        this.workspaceRoot = workspaceRoot;
        this.openFiles = openFiles;
        this.activeContent = activeContent;

        Runnable doIndex =
                () -> {
                    updateIndex(InferConfig.allJavaFiles(workspaceRoot).map(Path::toUri));

                    finishedInitialIndex.complete(null);
                };

        new Thread(doIndex, "Initial-Index").start();
    }

    private void updateIndex(Stream<URI> files) {
        // TODO send a progress bar to the user
        files.forEach(
                each -> {
                    if (needsUpdate(each)) {
                        CompilationUnitTree tree = parse(each);

                        update(tree);
                    }
                });
    }

    private boolean needsUpdate(URI file) {
        if (!sourcePathFiles.containsKey(file)) return true;
        else
            try {
                Instant updated = sourcePathFiles.get(file).updated;
                Instant modified = Files.getLastModifiedTime(Paths.get(file)).toInstant();

                return updated.isBefore(modified);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
    }

    /**
     * Guess the source path by looking at package declarations in .java files.
     *
     * <p>For example, if the file src/com/example/Test.java has the package declaration `package
     * com.example;` then the source root is `src`.
     */
    Set<Path> sourcePath() {
        updateOpenFiles();

        Set<Path> result = new HashSet<>();

        sourcePathFiles.forEach(
                (uri, index) -> {
                    Path dir = Paths.get(uri).getParent();
                    String packagePath = index.packageName.replace('.', File.separatorChar);

                    if (!dir.endsWith(packagePath))
                        LOG.warning("Java source file " + uri + " is not in " + packagePath);
                    else {
                        int up = Paths.get(packagePath).getNameCount();
                        Path truncate = dir;

                        for (int i = 0; i < up; i++) truncate = truncate.getParent();

                        result.add(truncate);
                    }
                });

        return result;
    }

    /** Search all indexed symbols */
    public Stream<SymbolInformation> search(String query) {
        updateOpenFiles();

        Predicate<CharSequence> nameMatchesQuery =
                name -> Completions.containsCharactersInOrder(name, query, true);
        Predicate<URI> fileMatchesQuery =
                uri -> sourcePathFiles.get(uri).declarations.stream().anyMatch(nameMatchesQuery);
        Collection<URI> open = openFiles.get();
        Stream<URI> openFirst =
                Stream.concat(
                        open.stream(),
                        sourcePathFiles.keySet().stream().filter(uri -> !open.contains(uri)));

        return openFirst
                .filter(fileMatchesQuery)
                .flatMap(this::allInFile)
                .filter(info -> nameMatchesQuery.test(info.getName()));
    }

    void updateOpenFiles() {
        finishedInitialIndex.join();

        updateIndex(openFiles.get().stream());
    }

    /** Get all declarations in an open file */
    Stream<SymbolInformation> allInFile(URI source) {
        LOG.info("Search " + source);

        JavacTask task = parseTask(source);
        Trees trees = Trees.instance(task);

        class FindDeclarations extends TreePathScanner<Void, Void> {
            List<SymbolInformation> result = new ArrayList<>();
            Optional<ClassTree> currentClass = Optional.empty();

            @Override
            public Void visitClass(ClassTree node, Void aVoid) {
                String name = node.getSimpleName().toString();
                SymbolInformation info = new SymbolInformation();

                info.setContainerName(packageName());
{'fraction_non_alphanumeric': 0.10342754058929646, 'fraction_numerical': 0.0003436130916587922, 'mean_word_length': 2.0262542240707044, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4542876', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2864, 'n_words': 686}