<commit_before>/*
Copyright 2005 <NAME>

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

http://www.apache.org/licenses/LICENSE-2.0 

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.  
*/
package com.pz.reader;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.Serializable;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.NoSuchElementException;
import java.util.Vector;

import com.pz.reader.ordering.OrderBy;
import com.pz.reader.structure.ColumnMetaData;
import com.pz.reader.structure.Row;
import com.pz.reader.util.ParserUtils;
import com.pz.reader.xml.PZMapParser;

/**
 * This class parses a datafile and holds methods to scroll back 
 * and forth through the datafile along with methods to retreive values 
 * from columns.
 * 
 * @author	<NAME>
 * @version 2.0
*/
public class DataSet implements Serializable{
	/**Array to hold the rows and their values in the text file*/
	public Vector rows = null;
	/**Array of errors that have occured during processing*/
	private Vector errors = null;
	/**Array of column metadata*/
	private Vector columnMD = null;
	/**Pointer for the current row in the array we are on*/
	private int pointer = -1;
	/**flag to indicate if data should be pulled as lower case*/
	private boolean lowerCase = false;
	/**flag to inidicate if data should be pulled as upper case*/
	private boolean upperCase = false;
	/**flag to indicate if a strict parse should be used when getting doubles and ints*/
	private boolean strictNumericParse = false;
	/**Flag to indicate that we can cope with lines shorter than the required lengh*/ 
	private boolean handleShortLines = false;
	
	
	/**
	 * Constructs a new DataSet using the database table file layout method.  <commit_msg>*** empty log message ***<commit_after>/*
Copyright 2005 <NAME>

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

http://www.apache.org/licenses/LICENSE-2.0 

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.  
*/
package com.pz.reader;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.Serializable;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.NoSuchElementException;
import java.util.Vector;

import com.pz.reader.ordering.OrderBy;
import com.pz.reader.structure.ColumnMetaData;
import com.pz.reader.structure.Row;
import com.pz.reader.util.ParserUtils;
import com.pz.reader.xml.PZMapParser;

/**
 * This class parses a datafile and holds methods to scroll back 
 * and forth through the datafile along with methods to retreive values 
 * from columns.
 * 
 * @author	<NAME>
 * @version 2.0
*/
public class DataSet implements Serializable{
	/**Array to hold the rows and their values in the text file*/
	public Vector rows = null;
	/**Array of errors that have occured during processing*/
	public Vector errors = null;
	/**Array of column metadata*/
	public Vector columnMD = null;
	/**Pointer for the current row in the array we are on*/
	private int pointer = -1;
	/**flag to indicate if data should be pulled as lower case*/
	private boolean lowerCase = false;
	/**flag to inidicate if data should be pulled as upper case*/
	private boolean upperCase = false;
	/**flag to indicate if a strict parse should be used when getting doubles and ints*/
	private boolean strictNumericParse = false;
	/**Flag to indicate that we can cope with lines shorter than the required lengh*/ 
	private boolean handleShortLines = false;
	
    
    
    /**
     * empty constructor.  THIS SHOULD ONLY BE USED FOR CUSTOM DataSet implementations.  It provides NO
     * parsing abilities
     */
    public DataSet(){
        
    }
    
	
	/**
	 * Constructs a new DataSet using the database table file layout method.  
{'fraction_non_alphanumeric': 0.07932643538243052, 'fraction_numerical': 0.0044633799959423815, 'mean_word_length': 4.140771637122002, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1871257', 'n_tokens_mistral': 1362, 'n_tokens_neox': 1317, 'n_words': 594}