<commit_before>/*******************************************************************************
 * Copyright (c) 2017 Pivotal, Inc.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Pivotal, Inc. - initial API and implementation
 *******************************************************************************/
package org.springframework.ide.vscode.boot.java.utils;

import java.io.File;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;

import org.eclipse.jdt.core.JavaCore;
import org.eclipse.jdt.core.dom.AST;
import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.core.dom.ASTVisitor;
import org.eclipse.jdt.core.dom.Annotation;
import org.eclipse.jdt.core.dom.CompilationUnit;
import org.eclipse.jdt.core.dom.FileASTRequestor;
import org.eclipse.jdt.core.dom.ITypeBinding;
import org.eclipse.jdt.core.dom.MarkerAnnotation;
import org.eclipse.jdt.core.dom.NormalAnnotation;
import org.eclipse.jdt.core.dom.SingleMemberAnnotation;
import org.eclipse.lsp4j.Location;
import org.eclipse.lsp4j.SymbolInformation;
import org.eclipse.lsp4j.SymbolKind;
import org.springframework.ide.vscode.boot.java.handlers.SymbolProvider;
import org.springframework.ide.vscode.commons.java.IClasspath;
import org.springframework.ide.vscode.commons.java.IJavaProject;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
import org.springframework.ide.vscode.commons.util.text.LanguageId;
import org.springframework.ide.vscode.commons.util.text.TextDocument;

/**
 * @author <NAME>
 */
public class AnnotationIndexer {

	private JavaProjectFinder projectFinder;
	private Map<String, SymbolProvider> symbolProviders;

	private List<SymbolInformation> symbols;

	public AnnotationIndexer(JavaProjectFinder projectFinder, Map<String, SymbolProvider> specificProviders) {
		this.projectFinder = projectFinder;
		this.symbolProviders = specificProviders;

		this.symbols = new ArrayList<>();
	}

	public void reset() {
		this.symbols.clear();
	}

	public List<? extends SymbolInformation> getAllSymbols() {
		return symbols;
	}

	public void scanFiles(File directory) {
		try {
			Path[] javaFiles = Files.walk(directory.toPath())
					.filter(Files::isRegularFile)
					.filter(path -> path.getFileName().toString().endsWith(".java"))
					.toArray(Path[]::new);

			Map<IJavaProject, List<String>> projects = new HashMap<>();
			for (Path javaFile : javaFiles) {
				IJavaProject project = projectFinder.find(javaFile.toFile());
				if (project != null) {
					if (!projects.containsKey(project)) {
						projects.put(project, new ArrayList<>());
					}

					projects.get(project).add(javaFile.toAbsolutePath().toString());
				}
			}

			projects.forEach((project, files) -> scanProject(project, files.toArray(new String[0])));

		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}

	private void scanProject(IJavaProject project, String[] files) {
		try {
			ASTParser parser = ASTParser.newParser(AST.JLS8);
			String[] classpathEntries = getClasspathEntries(project);
			scanFiles(parser, files, classpathEntries);
		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}

	private void scanFiles(ASTParser parser, String[] javaFiles, String[] classpathEntries) throws Exception {
		Map<String, String> options = JavaCore.getOptions();
		JavaCore.setComplianceOptions(JavaCore.VERSION_1_8, options);
		parser.setCompilerOptions(options);
		parser.setKind(ASTParser.K_COMPILATION_UNIT);<commit_msg>refactored code to use stream collector grouping instead of manual code<commit_after>/*******************************************************************************
 * Copyright (c) 2017 Pivotal, Inc.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Pivotal, Inc. - initial API and implementation
 *******************************************************************************/
package org.springframework.ide.vscode.boot.java.utils;

import java.io.File;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.eclipse.jdt.core.JavaCore;
import org.eclipse.jdt.core.dom.AST;
import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.core.dom.ASTVisitor;
import org.eclipse.jdt.core.dom.Annotation;
import org.eclipse.jdt.core.dom.CompilationUnit;
import org.eclipse.jdt.core.dom.FileASTRequestor;
import org.eclipse.jdt.core.dom.ITypeBinding;
import org.eclipse.jdt.core.dom.MarkerAnnotation;
import org.eclipse.jdt.core.dom.NormalAnnotation;
import org.eclipse.jdt.core.dom.SingleMemberAnnotation;
import org.eclipse.lsp4j.Location;
import org.eclipse.lsp4j.SymbolInformation;
import org.eclipse.lsp4j.SymbolKind;
import org.springframework.ide.vscode.boot.java.handlers.SymbolProvider;
import org.springframework.ide.vscode.commons.java.IClasspath;
import org.springframework.ide.vscode.commons.java.IJavaProject;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
import org.springframework.ide.vscode.commons.util.text.LanguageId;
import org.springframework.ide.vscode.commons.util.text.TextDocument;

/**
 * @author <NAME>
 */
public class AnnotationIndexer {

	private JavaProjectFinder projectFinder;
	private Map<String, SymbolProvider> symbolProviders;

	private List<SymbolInformation> symbols;

	public AnnotationIndexer(JavaProjectFinder projectFinder, Map<String, SymbolProvider> specificProviders) {
		this.projectFinder = projectFinder;
		this.symbolProviders = specificProviders;

		this.symbols = new ArrayList<>();
	}

	public void reset() {
		this.symbols.clear();
	}

	public List<? extends SymbolInformation> getAllSymbols() {
		return symbols;
	}

	public void scanFiles(File directory) {
		try {
			Map<IJavaProject, List<String>> projects = Files.walk(directory.toPath())
					.filter(Files::isRegularFile)
					.filter(path -> path.getFileName().toString().endsWith(".java"))
					.map(path -> path.toAbsolutePath().toString())
					.collect(Collectors.groupingBy((javaFile) -> projectFinder.find(new File(javaFile))));

			projects.forEach((project, files) -> scanProject(project, files.toArray(new String[0])));

		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}

	private void scanProject(IJavaProject project, String[] files) {
		try {
			ASTParser parser = ASTParser.newParser(AST.JLS8);
			String[] classpathEntries = getClasspathEntries(project);
			scanFiles(parser, files, classpathEntries);
		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}

	private void scanFiles(ASTParser parser, String[] javaFiles, String[] classpathEntries) throws Exception {
		Map<String, String> options = JavaCore.getOptions();
		JavaCore.setComplianceOptions(JavaCore.VERSION_1_8, options);
		parser.setCompilerOptions(options);
		parser.setKind(ASTParser.K_COMPILATION_UNIT);
{'fraction_non_alphanumeric': 0.15651595744680852, 'fraction_numerical': 0.003989361702127659, 'mean_word_length': 5.352195945945946, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3027382', 'n_tokens_mistral': 2387, 'n_tokens_neox': 2219, 'n_words': 441}