<commit_before>package edu.rosehulman.minijavac;

import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.io.Files;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import static org.junit.Assert.assertEquals;

@RunWith(value = Parameterized.class)
public class LexerTest {

    private static final String INPUT_FOLDER = "src/test/resources/lexer/java";
    private static final String OUTPUT_FOLDER = "src/test/resources/lexer/out";

    static class DirectoryFilter implements FileFilter {

        boolean wantDirectories;

        public DirectoryFilter(boolean wantDirectories) {
            this.wantDirectories = wantDirectories;
        }

        @Override
        public boolean accept(File pathname) {
            return this.wantDirectories == pathname.isDirectory();
        }
    }

    public static List<File> findFiles(File file) {
        if(file.isDirectory()) {
            File[] files = file.listFiles(new DirectoryFilter(false));
            File[] directories = file.listFiles(new DirectoryFilter(true));
            List<File> fileList = Arrays.asList(files);
            for(File directory : directories) {
                fileList.addAll(LexerTest.findFiles(directory));
            }
            return fileList;
        } else {
            List<File> files = new ArrayList<File>(1);
            files.add(file);
            return files;
        }
    }

    @Parameterized.Parameters(name = "{0}")
    public static Collection<File[]> testCases() {
        File inputFolder = new File(INPUT_FOLDER);
        File outputFolder = new File(OUTPUT_FOLDER);

        List<File> inputFiles = findFiles(inputFolder);
        List<File> outputFiles = findFiles(outputFolder);

        Collections.sort(inputFiles);
        Collections.sort(outputFiles);

        if(inputFiles.size() != outputFiles.size()) {
            return null;
        }

        List<File[]> fileCollection = new ArrayList<>();
        for(int k = 0; k < inputFiles.size(); k++) {
            File[] filePair = new File[2];
            filePair[0] = inputFiles.get(k);
            filePair[1] = outputFiles.get(k);
            fileCollection.add(filePair);
        }

        return fileCollection;
    }

    private File testFile;
    private File outputFile;

    public LexerTest(File testFile, File outputFile) {
        this.testFile = testFile;
        this.outputFile = outputFile;
    }

    @Test
    public void test() throws IOException {
        ImmutableList<Token> tokens = new Lexer(new FileReader(testFile)).getTokens();
        List<String> expectedLines = Files.readLines(outputFile, Charsets.UTF_8);
        assertEquals(expectedLines.size(), tokens.size());
        for (int i = 0; i < expectedLines.size(); i++) {
            assertEquals(expectedLines.get(i), tokens.get(i).toString());
        }
    }<commit_msg>Cleaned up testing code.<commit_after>package edu.rosehulman.minijavac;

import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.io.Files;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import static org.junit.Assert.assertEquals;

@RunWith(value = Parameterized.class)
public class LexerTest {

    private static final String INPUT_FOLDER = "src/test/resources/lexer/java";
    private static final String OUTPUT_FOLDER = "src/test/resources/lexer/out";
    private static final String OUTPUT_EXTENSION = ".out";

    @Parameterized.Parameters(name = "{0}")
    public static Collection<File[]> testCases() {
        List<File[]> fileCollection = new ArrayList<>();
        for (File testFile : Files.fileTreeTraverser().children(new File(INPUT_FOLDER))) {
            File outputFile = new File(OUTPUT_FOLDER,
                    Files.getNameWithoutExtension(testFile.getName()) + OUTPUT_EXTENSION);
            File[] filePair = {testFile, outputFile};
            fileCollection.add(filePair);
        }
        return fileCollection;
    }

    private final File testFile;
    private final File outputFile;

    public LexerTest(File testFile, File outputFile) {
        this.testFile = testFile;
        this.outputFile = outputFile;
    }

    @Test
    public void test() throws IOException {
        ImmutableList<Token> tokens = new Lexer(new FileReader(testFile)).getTokens();
        List<String> expectedLines = Files.readLines(outputFile, Charsets.UTF_8);
        assertEquals(expectedLines.size(), tokens.size());
        for (int i = 0; i < expectedLines.size(); i++) {
            assertEquals(expectedLines.get(i), tokens.get(i).toString());
        }
    }
{'fraction_non_alphanumeric': 0.11187036313939867, 'fraction_numerical': 0.002147598594299102, 'mean_word_length': 2.974398758727696, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3745280', 'n_tokens_mistral': 1468, 'n_tokens_neox': 1441, 'n_words': 317}