<commit_before>package nl.han.ica.core.parser;

import nl.han.ica.core.SourceFile;
import org.apache.log4j.Logger;
import org.eclipse.jdt.core.JavaCore;
import org.eclipse.jdt.core.dom.AST;
import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.core.dom.CompilationUnit;

import java.util.Set;

public class Parser {

    private ASTParser astParser;
    private Set<SourceFile> sourceFiles;
    private Logger logger;

    public Parser() {
        logger = Logger.getLogger(getClass().getName());

        initializeASTParser();
    }

    private void initializeASTParser() {
        astParser = ASTParser.newParser(AST.JLS3);
        astParser.setKind(ASTParser.K_COMPILATION_UNIT);
        astParser.setResolveBindings(true);
        astParser.setCompilerOptions(JavaCore.getOptions());
    }

    public Set<CompilationUnit> parse(Set<SourceFile> sourceFiles) {
        this.sourceFiles = sourceFiles;

        astParser.setEnvironment(directoryPaths(sourceFiles), null, null, false);
        astParser.setUnitName("test"); // TODO: Check what this does (and whether it can be useful);

        String[] bindings = new String[0];
        ASTRequestor astRequestor = new ASTRequestor(sourceFiles);
        astParser.createASTs(filePaths(sourceFiles), null, bindings, astRequestor, null);
        return astRequestor.getCompilationUnits();
    }

    private String[] filePaths(Set<SourceFile> files) {
        String[] filePaths = new String[files.size()];
        int i = 0;
        for (SourceFile file : files) {
            filePaths[i] = file.getFile().getPath();
            i++;
        }
        return filePaths;
    }

    private String[] directoryPaths(Set<SourceFile> files) {
        String[] filePaths = new String[files.size()];
        int i = 0;
        for (SourceFile file : files) {
            filePaths[i] = file.getFile().getParentFile().getPath();
        }
        return filePaths;
    }

}<commit_msg>Selecting multiple files resulted in a NullPointerException. Fixes #48<commit_after>package nl.han.ica.core.parser;

import nl.han.ica.core.SourceFile;
import org.apache.log4j.Logger;
import org.eclipse.jdt.core.JavaCore;
import org.eclipse.jdt.core.dom.AST;
import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.core.dom.CompilationUnit;

import java.util.Set;

public class Parser {

    private ASTParser astParser;
    private Set<SourceFile> sourceFiles;
    private Logger logger;

    public Parser() {
        logger = Logger.getLogger(getClass().getName());

        initializeASTParser();
    }

    private void initializeASTParser() {
        astParser = ASTParser.newParser(AST.JLS3);
        astParser.setKind(ASTParser.K_COMPILATION_UNIT);
        astParser.setResolveBindings(true);
        astParser.setCompilerOptions(JavaCore.getOptions());
    }

    public Set<CompilationUnit> parse(Set<SourceFile> sourceFiles) {
        this.sourceFiles = sourceFiles;

        astParser.setEnvironment(directoryPaths(sourceFiles), null, null, false);
        astParser.setUnitName("test"); // TODO: Check what this does (and whether it can be useful);

        String[] bindings = new String[0];
        ASTRequestor astRequestor = new ASTRequestor(sourceFiles);
        astParser.createASTs(filePaths(sourceFiles), null, bindings, astRequestor, null);
        return astRequestor.getCompilationUnits();
    }

    private String[] filePaths(Set<SourceFile> files) {
        String[] filePaths = new String[files.size()];
        int i = 0;
        for (SourceFile file : files) {
            filePaths[i] = file.getFile().getPath();
            i++;
        }
        return filePaths;
    }

    private String[] directoryPaths(Set<SourceFile> files) {
        String[] filePaths = new String[files.size()];
        int i = 0;
        for (SourceFile file : files) {
            filePaths[i] = file.getFile().getParentFile().getPath();
            i++;
        }
        return filePaths;
    }

}
{'fraction_non_alphanumeric': 0.11478304742684158, 'fraction_numerical': 0.0030272452068617556, 'mean_word_length': 2.9849246231155777, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3131422', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1149, 'n_words': 243}