<commit_before>/*
 Copyright 2006 <NAME>

 Licensed under the Apache License, Version 2.0 (the "License"); 
 you may not use this file except in compliance with the License. 
 You may obtain a copy of the License at 

 http://www.apache.org/licenses/LICENSE-2.0 

 Unless required by applicable law or agreed to in writing, software distributed 
 under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
 CONDITIONS OF ANY KIND, either express or implied. See the License for 
 the specific language governing permissions and limitations under the License.  
 */
package net.sf.pzfilereader.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;

import net.sf.pzfilereader.structure.ColumnMetaData;
import net.sf.pzfilereader.xml.XMLRecordElement;

/**
 * @author zepernick Static utilities that are used to perform parsing in the
 *         DataSet class These can also be used for low level parsing, if not
 *         wishing to use the DataSet class.
 * @version 2.0
 */
public final class ParserUtils {
    private ParserUtils() {
    }

    /**
     * @deprecated should only use the splitLine with a CHAR.
     * @param line
     * @param delimiter
     * @param qualifier
     * @return
     */
    public static List splitLine(String line, final String delimiter, final String qualifier) {
        return splitLine(line, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier.charAt(0) : 0);
    }

    /**
     * Returns an ArrayList of items in a delimited string. If there is no
     * qualifier around the text, the qualifier parameter can be left null, or
     * empty. There should not be any line breaks in the string. Each line of
     * the file should be passed in individually.
     * 
     * @param line -
     *            String of data to be parsed
     * @param delimiter -
     *            Delimiter seperating each element
     * @param qualifier -
     *            qualifier which is surrounding the text
     * @return ArrayList
     */
    public static List splitLine(String line, final char delimiter, final char qualifier) {
        final ArrayList list = new ArrayList();
        // String temp = "";
        boolean beginQualifier = false;
        // this will be used for delimted files that have some items qualified
        // and some items dont
        boolean beginNoQualifier = false;
        StringBuffer sb = new StringBuffer();

        // trim hard leading spaces at the begining of the line
        line = lTrim(line);
        for (int i = 0; i < line.length(); i++) {
            final String remainderOfLine = line.substring(i); // data of the
            // line which has not yet been read
            // check to see if there is a text qualifier
            final char currentChar = line.charAt(i);
//            final String currentString = String.valueOf(currentChar);
            if (qualifier > 0) {
                if (currentChar == qualifier && !beginQualifier && !beginNoQualifier) {
                    // begining of a set of data
                    beginQualifier = true;
                } else if (!beginQualifier && !beginNoQualifier && currentChar != qualifier
                        && lTrim(remainderOfLine).charAt(0) != qualifier) {
                    // try to account for empty space before qualifier starts
                    // we have not yet begun a qualifier and the char we are on
                    // is NOT
                    // a qualifier. Start reading data
                    beginNoQualifier = true;
                    // make sure that this is not just an empty column with no
                    // qualifiers. ie
                    // "data",,"data"
                    if (currentChar == delimiter) {
                        list.add(sb.toString());
                        sb.delete(0, sb.length());
                        beginNoQualifier = false;
                        continue;// grab the next char
                    }
                    sb.append(currentChar);
                } else if (!beginNoQualifier && currentChar == qualifier && beginQualifier
                        && (i == line.length() - 1 || lTrim(remainderOfLine.substring(1)).length() == 0
                        // this will be true on empty undelmited columns at the
                        // end of theline
                        || lTrimKeepTabs(remainderOfLine).charAt(1) == delimiter)) {
                    // end of a set of data that was qualified
                    list.add(sb.toString());
                    sb.delete(0, sb.length());
                    beginQualifier = false;
                    // add to "i" so we can get past the qualifier, otherwise it
                    // is read into a set
                    // of data which
                    // may not be qualified. Find out how many spaces to the
                    // delimiter
                    final int offset = getDelimiterOffset(line, i, delimiter) - 1;
                    // subtract 1 since i is going to get incremented again at
                    // the top of the loop
                    if (offset < 1) {
                        i++;
                    } else {
                        i += offset;
                    }
                } else if (beginNoQualifier && currentChar == delimiter) {
                    // check to see if we are done with an element that was not
                    // being qulified
                    list.add(sb.toString());
                    sb.delete(0, sb.length());
                    beginNoQualifier = false;
                } else if (beginNoQualifier || beginQualifier) {
                    // getting data in a NO qualifier element or qualified
                    // element
                    sb.append(currentChar);
                }

            } else {
                // not using a qualifier. Using a delimiter only
                if (currentChar == delimiter) {
                    list.add(sb.toString());
                    sb.delete(0, sb.length());
                } else {
                    sb.append(currentChar);
                }
            }
        }

        // + this needs to be revisited...
        final String trimmed = sb.toString().trim();
        // remove the ending text qualifier if needed
        if (qualifier > 0 && trimmed.length() > 0) {
            if (trimmed.charAt(trimmed.length() - 1) == qualifier) {
                final String s = trimmed.substring(0, trimmed.length() - 1);
                sb.delete(0, sb.length());
                sb.append(s);
            }<commit_msg>fixed bug, should not trim off qualifier unless the element  began with a qulifier<commit_after>/*
 Copyright 2006 <NAME>

 Licensed under the Apache License, Version 2.0 (the "License"); 
 you may not use this file except in compliance with the License. 
 You may obtain a copy of the License at 

 http://www.apache.org/licenses/LICENSE-2.0 

 Unless required by applicable law or agreed to in writing, software distributed 
 under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
 CONDITIONS OF ANY KIND, either express or implied. See the License for 
 the specific language governing permissions and limitations under the License.  
 */
package net.sf.pzfilereader.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;

import net.sf.pzfilereader.structure.ColumnMetaData;
import net.sf.pzfilereader.xml.XMLRecordElement;

/**
 * @author zepernick Static utilities that are used to perform parsing in the
 *         DataSet class These can also be used for low level parsing, if not
 *         wishing to use the DataSet class.
 * @version 2.0
 */
public final class ParserUtils {
    private ParserUtils() {
    }

    /**
     * @deprecated should only use the splitLine with a CHAR.
     * @param line
     * @param delimiter
     * @param qualifier
     * @return
     */
    public static List splitLine(String line, final String delimiter, final String qualifier) {
        return splitLine(line, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier.charAt(0) : 0);
    }

    /**
     * Returns an ArrayList of items in a delimited string. If there is no
     * qualifier around the text, the qualifier parameter can be left null, or
     * empty. There should not be any line breaks in the string. Each line of
     * the file should be passed in individually.
     * 
     * @param line -
     *            String of data to be parsed
     * @param delimiter -
     *            Delimiter seperating each element
     * @param qualifier -
     *            qualifier which is surrounding the text
     * @return ArrayList
     */
    public static List splitLine(String line, final char delimiter, final char qualifier) {
        final ArrayList list = new ArrayList();
        // String temp = "";
        boolean beginQualifier = false;
        // this will be used for delimted files that have some items qualified
        // and some items dont
        boolean beginNoQualifier = false;
        StringBuffer sb = new StringBuffer();

        // trim hard leading spaces at the begining of the line
        line = lTrim(line);
        for (int i = 0; i < line.length(); i++) {
            final String remainderOfLine = line.substring(i); // data of the
            // line which has not yet been read
            // check to see if there is a text qualifier
            final char currentChar = line.charAt(i);
//            final String currentString = String.valueOf(currentChar);
            if (qualifier > 0) {
                if (currentChar == qualifier && !beginQualifier && !beginNoQualifier) {
                    // begining of a set of data
                    beginQualifier = true;
                } else if (!beginQualifier && !beginNoQualifier && currentChar != qualifier
                        && lTrim(remainderOfLine).charAt(0) != qualifier) {
                    // try to account for empty space before qualifier starts
                    // we have not yet begun a qualifier and the char we are on
                    // is NOT
                    // a qualifier. Start reading data
                    beginNoQualifier = true;
                    // make sure that this is not just an empty column with no
                    // qualifiers. ie
                    // "data",,"data"
                    if (currentChar == delimiter) {
                        list.add(sb.toString());
                        sb.delete(0, sb.length());
                        beginNoQualifier = false;
                        continue;// grab the next char
                    }
                    sb.append(currentChar);
                } else if (!beginNoQualifier && currentChar == qualifier && beginQualifier
                        && (i == line.length() - 1 || lTrim(remainderOfLine.substring(1)).length() == 0
                        // this will be true on empty undelmited columns at the
                        // end of theline
                        || lTrimKeepTabs(remainderOfLine).charAt(1) == delimiter)) {
                    // end of a set of data that was qualified
                    list.add(sb.toString());
                    sb.delete(0, sb.length());
                    beginQualifier = false;
                    // add to "i" so we can get past the qualifier, otherwise it
                    // is read into a set
                    // of data which
                    // may not be qualified. Find out how many spaces to the
                    // delimiter
                    final int offset = getDelimiterOffset(line, i, delimiter) - 1;
                    // subtract 1 since i is going to get incremented again at
                    // the top of the loop
                    if (offset < 1) {
                        i++;
                    } else {
                        i += offset;
                    }
                } else if (beginNoQualifier && currentChar == delimiter) {
                    // check to see if we are done with an element that was not
                    // being qulified
                    list.add(sb.toString());
                    sb.delete(0, sb.length());
                    beginNoQualifier = false;
                } else if (beginNoQualifier || beginQualifier) {
                    // getting data in a NO qualifier element or qualified
                    // element
                    sb.append(currentChar);
                }

            } else {
                // not using a qualifier. Using a delimiter only
                if (currentChar == delimiter) {
                    list.add(sb.toString());
                    sb.delete(0, sb.length());
                } else {
                    sb.append(currentChar);
                }
            }
        }

        // + this needs to be revisited...
        final String trimmed = sb.toString().trim();
        // remove the ending text qualifier if needed
        //only if the last element was truly qualified
        if (beginQualifier && qualifier > 0 && trimmed.length() > 0) {
            if (trimmed.charAt(trimmed.length() - 1) == qualifier) {
                System.out.println(">>>>>>>Triming Off Qualifier");
                final String s = trimmed.substring(0, trimmed.length() - 1);
                sb.delete(0, sb.length());
                sb.append(s);
            }
{'fraction_non_alphanumeric': 0.08407454468445574, 'fraction_numerical': 0.0048002258929831995, 'mean_word_length': 1.6061442236938925, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3523369', 'n_tokens_mistral': 3589, 'n_tokens_neox': 3362, 'n_words': 1295}