<commit_before>package org.simpleflatmapper.csv.test;

import org.junit.Test;
import org.simpleflatmapper.csv.CellValueReader;
import org.simpleflatmapper.csv.CloseableCsvReader;
import org.simpleflatmapper.csv.CsvColumnDefinition;
import org.simpleflatmapper.csv.CsvColumnKey;
import org.simpleflatmapper.csv.CsvMapper;
import org.simpleflatmapper.csv.CsvMapperBuilder;
import org.simpleflatmapper.csv.CsvMapperFactory;
import org.simpleflatmapper.csv.CsvParser;
import org.simpleflatmapper.csv.CsvReader;
import org.simpleflatmapper.csv.ParsingContext;
import org.simpleflatmapper.test.beans.DbObject;
import org.simpleflatmapper.util.CheckedConsumer;
import org.simpleflatmapper.csv.parser.BufferOverflowException;
import org.simpleflatmapper.csv.parser.CellConsumer;
import org.simpleflatmapper.tuple.Tuple2;
import org.simpleflatmapper.tuple.Tuple3;
import org.simpleflatmapper.tuple.Tuple4;
import org.simpleflatmapper.tuple.Tuple5;
import org.simpleflatmapper.tuple.Tuple6;
import org.simpleflatmapper.tuple.Tuple7;
import org.simpleflatmapper.tuple.Tuple8;
import org.simpleflatmapper.tuple.Tuples;
import org.simpleflatmapper.util.TypeReference;
import org.simpleflatmapper.util.CloseableIterator;
import org.simpleflatmapper.util.ListCollector;
import org.simpleflatmapper.util.Predicate;

import java.io.CharArrayReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
//IFJAVA8_START
import java.util.stream.Collectors;
import java.util.stream.Stream;
//IFJAVA8_END
import static org.junit.Assert.*;

public class CsvParserTest {


	private static String[][] SAMPLE_CSV_MIX_EXPECTATION =
			{
					{"cell1", "cell2", ""},
					{"cell\r\"value\"", "val2"},
					{"val3"},
					{"val4", ""}
			};
	@Test
	public void testReadCsvReaderLF() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '\\', "\n");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '"', "\n");
	}


	@Test
	public void testReadCsvReaderLFSINGLEQUOTE() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '\'', '\'', "\n");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '\'', '\\', "\n");
	}

	@Test
	public void testReadCsvReaderCR() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '"',"\r");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '\\',"\r");
	}

	@Test
	public void testReadCsvReaderCRLF() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '"',"\r\n");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '\\',"\r\n");
	}

	@Test
	public void testReadCsvReaderTabs() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, '\t', '"','"', "\n");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, '\t', '"','\\', "\n");
	}

    @Test
    public void testReadCsvReaderOneChar() throws IOException {
        Iterator<String[]> iterator = CsvParser.iterator(new StringReader("0"));
        String[] strs = iterator.next();
        assertEquals("0", strs[0]);
    }

    @Test
    public void testSimpleCsv() throws  IOException {
		List<String[]> list = CsvParser.forEach("a,b\n" +
				"c", new ListCollector<String[]>()).getList();

		assertArrayEquals(new String[] {"a", "b"}, list.get(0));<commit_msg> #459 add unescaping special char test<commit_after>package org.simpleflatmapper.csv.test;

import org.junit.Test;
import org.simpleflatmapper.csv.CellValueReader;
import org.simpleflatmapper.csv.CloseableCsvReader;
import org.simpleflatmapper.csv.CsvColumnDefinition;
import org.simpleflatmapper.csv.CsvColumnKey;
import org.simpleflatmapper.csv.CsvMapper;
import org.simpleflatmapper.csv.CsvMapperBuilder;
import org.simpleflatmapper.csv.CsvMapperFactory;
import org.simpleflatmapper.csv.CsvParser;
import org.simpleflatmapper.csv.CsvReader;
import org.simpleflatmapper.csv.ParsingContext;
import org.simpleflatmapper.test.beans.DbObject;
import org.simpleflatmapper.util.CheckedConsumer;
import org.simpleflatmapper.csv.parser.BufferOverflowException;
import org.simpleflatmapper.csv.parser.CellConsumer;
import org.simpleflatmapper.tuple.Tuple2;
import org.simpleflatmapper.tuple.Tuple3;
import org.simpleflatmapper.tuple.Tuple4;
import org.simpleflatmapper.tuple.Tuple5;
import org.simpleflatmapper.tuple.Tuple6;
import org.simpleflatmapper.tuple.Tuple7;
import org.simpleflatmapper.tuple.Tuple8;
import org.simpleflatmapper.tuple.Tuples;
import org.simpleflatmapper.util.TypeReference;
import org.simpleflatmapper.util.CloseableIterator;
import org.simpleflatmapper.util.ListCollector;
import org.simpleflatmapper.util.Predicate;

import java.io.CharArrayReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
//IFJAVA8_START
import java.util.stream.Collectors;
import java.util.stream.Stream;
//IFJAVA8_END
import static org.junit.Assert.*;

public class CsvParserTest {


	private static String[][] SAMPLE_CSV_MIX_EXPECTATION =
			{
					{"cell1", "cell2", ""},
					{"cell\r\"value\"", "val2"},
					{"val3"},
					{"val4", ""}
			};
	@Test
	public void testReadCsvReaderLF() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '\\', "\n");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '"', "\n");
	}


	@Test
	public void testReadCsvReaderLFSINGLEQUOTE() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '\'', '\'', "\n");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '\'', '\\', "\n");
	}

	@Test
	public void testReadCsvReaderCR() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '"',"\r");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '\\',"\r");
	}

	@Test
	public void testReadCsvReaderCRLF() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '"',"\r\n");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, ',', '"', '\\',"\r\n");
	}

	@Test
	public void testReadCsvReaderTabs() throws IOException {
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, '\t', '"','"', "\n");
		testCsvReader(SAMPLE_CSV_MIX_EXPECTATION, '\t', '"','\\', "\n");
	}

    @Test
    public void testReadCsvReaderOneChar() throws IOException {
        Iterator<String[]> iterator = CsvParser.iterator(new StringReader("0"));
        String[] strs = iterator.next();
        assertEquals("0", strs[0]);
    }
    
    @Test
	public void test459EscapeChar() throws IOException {
		Iterator<String[]> iterator = CsvParser.dsl().escape('\\').iterator(new StringReader("\"blah\\r\\n\\\"helo\\\"\",ggg"));
		String[] strs = iterator.next();
		assertEquals("blah\r\n\"helo\"", strs[0]);
		assertEquals("ggg", strs[1]);
	}

    @Test
    public void testSimpleCsv() throws  IOException {
		List<String[]> list = CsvParser.forEach("a,b\n" +
				"c", new ListCollector<String[]>()).getList();

		assertArrayEquals(new String[] {"a", "b"}, list.get(0));
{'fraction_non_alphanumeric': 0.15498310810810811, 'fraction_numerical': 0.006193693693693694, 'mean_word_length': 5.536338546458142, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6542360', 'n_tokens_mistral': 2572, 'n_tokens_neox': 2466, 'n_words': 386}