<commit_before>package net.sf.flatpack.pzparser;

import java.io.StringReader;
import java.util.Iterator;
import java.util.NoSuchElementException;

import junit.framework.TestCase;
import net.sf.flatpack.DataError;
import net.sf.flatpack.DataSet;
import net.sf.flatpack.DefaultParserFactory;
import net.sf.flatpack.Parser;
import net.sf.flatpack.ordering.OrderBy;
import net.sf.flatpack.ordering.OrderColumn;
import net.sf.flatpack.util.FPConstants;
import net.sf.flatpack.util.FPInvalidUsageException;

/**
 * Test the different options that can be 
 * set on the parser
 * 
 * @author <NAME>
 */
public class PZParserOptsTest extends TestCase {

    public void testEmptyToNull() {
        DataSet ds;
        final String cols = "COLUMN1,column2,Column3\r\n value1,,value3";
        Parser p = DefaultParserFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', FPConstants.NO_QUALIFIER);
        p.setNullEmptyStrings(true);
        ds = p.parse();
        ds.next();

        assertEquals("String should be null...", null, ds.getString("column2"));
<commit_msg>added a couple junit tests to try to replicate a bug reported on SF.  So far everything is passing and it cannot be recreated.<commit_after>package net.sf.flatpack.pzparser;

import java.io.StringReader;
import java.util.Iterator;
import java.util.NoSuchElementException;

import junit.framework.TestCase;
import net.sf.flatpack.DataError;
import net.sf.flatpack.DataSet;
import net.sf.flatpack.DefaultParserFactory;
import net.sf.flatpack.Parser;
import net.sf.flatpack.brparse.BuffReaderParseFactory;
import net.sf.flatpack.ordering.OrderBy;
import net.sf.flatpack.ordering.OrderColumn;
import net.sf.flatpack.util.FPConstants;
import net.sf.flatpack.util.FPInvalidUsageException;

/**
 * Test the different options that can be 
 * set on the parser
 * 
 * @author <NAME>
 */
public class PZParserOptsTest extends TestCase {
	
	public void testHandleShortLines() {
        DataSet ds;
        final String cols = "COLUMN1,column2,Column3\r\n value1";
        Parser p = DefaultParserFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', FPConstants.NO_QUALIFIER);
        p.setHandlingShortLines(true);
      //  p.setIgnoreParseWarnings(true);
        ds = p.parse();
        assertEquals("Should have a row of data", true, ds.next());
        
        //re-test the buffered reader
        p = BuffReaderParseFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', FPConstants.NO_QUALIFIER);
        p.setHandlingShortLines(true);
      //  p.setIgnoreParseWarnings(true);
        ds = p.parse();
        assertEquals("Should have a row of data", true, ds.next());
	}
	
	public void testIgnoreExtraColumns() {
        DataSet ds;
        final String cols = "COLUMN1,column2,Column3\r\n \"value1\",value2,value3,value4";
        Parser p = DefaultParserFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', '"');
        p.setIgnoreExtraColumns(true);
      //  p.setIgnoreParseWarnings(true);
        ds = p.parse();
        assertEquals("Should have a row of data", true, ds.next());
        
        //re-test the buffered reader
        p = BuffReaderParseFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', '"');
        p.setIgnoreExtraColumns(true);
      //  p.setIgnoreParseWarnings(true);
        ds = p.parse();
        assertEquals("Should have a row of data", true, ds.next());
	}

    public void testEmptyToNull() {
        DataSet ds;
        final String cols = "COLUMN1,column2,Column3\r\n value1,,value3";
        Parser p = DefaultParserFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', FPConstants.NO_QUALIFIER);
        p.setNullEmptyStrings(true);
        ds = p.parse();
        ds.next();

        assertEquals("String should be null...", null, ds.getString("column2"));

{'fraction_non_alphanumeric': 0.12056921086675292, 'fraction_numerical': 0.0059508408796895215, 'mean_word_length': 3.6188769414575868, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '17193', 'n_tokens_mistral': 1168, 'n_tokens_neox': 1148, 'n_words': 256}