<commit_before>package parsers;

import java.util.Scanner;

import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import eAdapter.Document;
import parsers.ParserArguments;
import parsers.TextDelimitedParser;

public class TU_TextDelimitedParser {

    private TextDelimitedParser parser;

    @Before
    public void testSetup() {
        parser = new TextDelimitedParser();
    }

    @Test
    public void commaDelimitedTest() {
        Scanner scanner = new Scanner("one,two,buckle,my,shoe");
        Delimiters delimiters = Delimiters.COMMA_DELIMITED;
        String[] header = new String[] { "1", "2", "3", "4", "5" };
        ParserArguments args = ParserArguments.of(scanner, delimiters, header);
        Document doc = parser.parseDocument(args);
        Assert.assertEquals("one", doc.getMetadata().get("1"));
        Assert.assertEquals("two", doc.getMetadata().get("2"));
        Assert.assertEquals("buckle", doc.getMetadata().get("3"));
        Assert.assertEquals("my", doc.getMetadata().get("4"));
        Assert.assertEquals("shoe", doc.getMetadata().get("5"));
    }

}<commit_msg>Resolved compile error.<commit_after>package parsers;

import java.util.Arrays;
import java.util.List;
import java.util.Scanner;

import org.apache.commons.lang3.tuple.Triple;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import eAdapter.Document;
import eAdapter.Representative;
import parsers.ParserArguments;
import parsers.TextDelimitedParser;

public class TU_TextDelimitedParser {

    private TextDelimitedParser parser;

    @Before
    public void testSetup() {
        parser = new TextDelimitedParser();
    }

    @Test
    public void commaDelimitedTest() {
        Scanner scanner = new Scanner("one,two,buckle,my,shoe");
        Delimiters delimiters = Delimiters.COMMA_DELIMITED;
        String[] header = new String[] { "1", "2", "3", "4", "5" };
        List<String> fields = Arrays.asList(header);
        List<Triple<String, String, Representative.Type>> reps = null;
        ParserArguments args = ParserArguments.of(scanner, delimiters, header);
        Document doc = parser.parse(scanner, delimiters, fields, "1", reps);
        Assert.assertEquals("one", doc.getMetadata().get("1"));
        Assert.assertEquals("two", doc.getMetadata().get("2"));
        Assert.assertEquals("buckle", doc.getMetadata().get("3"));
        Assert.assertEquals("my", doc.getMetadata().get("4"));
        Assert.assertEquals("shoe", doc.getMetadata().get("5"));
    }

}
{'fraction_non_alphanumeric': 0.1423473433782712, 'fraction_numerical': 0.008723235527359239, 'mean_word_length': 3.851923076923077, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7243039', 'n_tokens_mistral': 780, 'n_tokens_neox': 753, 'n_words': 156}