<commit_before>package org.minimalj.util;

import java.io.ByteArrayInputStream;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.util.List;

import org.junit.Assert;
import org.junit.Test;

public class CsvReaderTest {

	@Test
	public void testReadFile() throws Exception {
		CsvReader reader = reader("a,b\na,b");
		List<String> values = reader.readRecord();
		Assert.assertTrue(isAB(values));
		values = reader.readRecord();
		Assert.assertTrue(isAB(values));
	}
	
	@Test
	public void testReadRecord() throws Exception {
		CsvReader reader = reader("a,b");
		List<String> values = reader.readRecord();
		Assert.assertTrue(isAB(values));
		
		reader = reader("a,\"b\"");
		values = reader.readRecord();
		Assert.assertTrue(isAB(values));
	}
	
	private boolean isAB(List<String> strings) {
		if (strings.size() != 2) return false;
		return strings.get(0).equals("a") && strings.get(1).equals("b");
	}
	
	@Test
	public void testReadField() throws Exception {
		CsvReader reader = reader("\"ab\"");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("\"ab\"\n");
		Assert.assertEquals("ab", reader.readField());

		reader = reader("\"ab\",");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("ab");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("ab\n");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("ab,");
		Assert.assertEquals("ab", reader.readField());

		reader = reader(" ab,");
		Assert.assertEquals("ab", reader.readField());

		reader = reader("  ab,");
		Assert.assertEquals("ab", reader.readField());

		reader = reader("ab ,");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader(" \"ab\",");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("\t\"ab\"\t,");
		Assert.assertEquals("ab", reader.readField());
	}
	
	
	@Test
	public void testReadEscaped() throws Exception {
		CsvReader reader = reader("\"ab\"");
		Assert.assertEquals("ab", reader.readEscaped());
		
		reader = reader("\"ab\"\n");
		Assert.assertEquals("ab", reader.readEscaped());

		reader = reader("\"ab\",");
		Assert.assertEquals("ab", reader.readEscaped());
	}
	
	@Test<commit_msg>Test escaped separator and escaping character<commit_after>package org.minimalj.util;

import java.io.ByteArrayInputStream;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.util.List;

import org.junit.Assert;
import org.junit.Test;

public class CsvReaderTest {

	@Test
	public void testReadFile() throws Exception {
		CsvReader reader = reader("a,b\na,b");
		List<String> values = reader.readRecord();
		Assert.assertTrue(isAB(values));
		values = reader.readRecord();
		Assert.assertTrue(isAB(values));
	}
	
	@Test
	public void testReadRecord() throws Exception {
		CsvReader reader = reader("a,b");
		List<String> values = reader.readRecord();
		Assert.assertTrue(isAB(values));
		
		reader = reader("a,\"b\"");
		values = reader.readRecord();
		Assert.assertTrue(isAB(values));
	}
	
	private boolean isAB(List<String> strings) {
		if (strings.size() != 2) return false;
		return strings.get(0).equals("a") && strings.get(1).equals("b");
	}
	
	@Test
	public void testReadField() throws Exception {
		CsvReader reader = reader("\"ab\"");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("\"ab\"\n");
		Assert.assertEquals("ab", reader.readField());

		reader = reader("\"ab\",");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("ab");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("ab\n");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("ab,");
		Assert.assertEquals("ab", reader.readField());

		reader = reader(" ab,");
		Assert.assertEquals("ab", reader.readField());

		reader = reader("  ab,");
		Assert.assertEquals("ab", reader.readField());

		reader = reader("ab ,");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader(" \"ab\",");
		Assert.assertEquals("ab", reader.readField());
		
		reader = reader("\t\"ab\"\t,");
		Assert.assertEquals("ab", reader.readField());
	}
	
	
	@Test
	public void testReadEscaped() throws Exception {
		CsvReader reader = reader("\"ab\"");
		Assert.assertEquals("ab", reader.readEscaped());
		
		reader = reader("\"ab\"\n");
		Assert.assertEquals("ab", reader.readEscaped());

		reader = reader("\"ab\",");
		Assert.assertEquals("ab", reader.readEscaped());

		reader = reader("\"a,b\",");
		Assert.assertEquals("a,b", reader.readEscaped());

		reader = reader("\"a\"\"b\",");
		Assert.assertEquals("a\"b", reader.readEscaped());
	}
	
	@Test
{'fraction_non_alphanumeric': 0.19393939393939394, 'fraction_numerical': 0.0012987012987012987, 'mean_word_length': 4.614823815309842, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3428945', 'n_tokens_mistral': 1603, 'n_tokens_neox': 1430, 'n_words': 286}