<commit_before>##############################################################################
# Name: csv.py
# Purpose: Implementation of CSV parser in arpeggio.
# Author: <NAME> <<EMAIL>>
# Copyright: (c) 2014 <NAME> <<EMAIL>>
# License: MIT License
##############################################################################
from arpeggio import *
from arpeggio.export import PMDOTExport, PTDOTExport
from arpeggio import RegExMatch as _

def newline():                  return _(r'\n+')
def record():                   return field, ZeroOrMore(",", field)
def field():                    return [quoted_field, field_content]
def quoted_field():             return '"', field_content_quoted, '"'
def field_content():            return _(r'([^,\n])+')
def field_content_quoted():     return _(r'(("")|([^"]))+')
def csvfile():                  return OneOrMore(ZeroOrMore(newline), record, OneOrMore(newline)), EndOfFile


if __name__ == "__main__":

    test_data = '''
Unquoted test, "Quoted test", 23234, One Two Three, "343456.45"

Unquoted test 2, "Quoted test 2", 23234, One Two Three, "343456.45"
Unquoted test 3, "Quoted test 3", 23234, One Two Three, "343456.45"
    '''

    try:
        # First we will make a parser - an instance of the CVS parser model.
        # Parser model is given in the form of python constructs therefore we
        # are using ParserPython class.
        # Skipping of whitespace will be done only for tabs and spaces. Newlines
        # have semantics in csv files. They are used to separate records.
        parser = ParserPython(csvfile, ws='\t ', reduce_tree=True, debug=True)

        # Then we export it to a dot file in order to visualise it. This is<commit_msg>Fix in the CSV top rule.<commit_after>##############################################################################
# Name: csv.py
# Purpose: Implementation of CSV parser in arpeggio.
# Author: <NAME> <igor DOT dejanovic AT gmail DOT com>
# Copyright: (c) 2014 <NAME> <igor DOT dejanovic AT gmail DOT com>
# License: MIT License
##############################################################################
from arpeggio import *
from arpeggio.export import PMDOTExport, PTDOTExport
from arpeggio import RegExMatch as _

def newline():                  return _(r'\n+')
def record():                   return field, ZeroOrMore(",", field)
def field():                    return [quoted_field, field_content]
def quoted_field():             return '"', field_content_quoted, '"'
def field_content():            return _(r'([^,\n])+')
def field_content_quoted():     return _(r'(("")|([^"]))+')
def csvfile():                  return OneOrMore([record, newline]), EndOfFile


if __name__ == "__main__":

    test_data = '''
Unquoted test, "Quoted test", 23234, One Two Three, "343456.45"

Unquoted test 2, "Quoted test with ""inner"" quotes", 23234, One Two Three, "343456.45"
Unquoted test 3, "Quoted test 3", 23234, One Two Three, "343456.45"
    '''

    try:
        # First we will make a parser - an instance of the CVS parser model.
        # Parser model is given in the form of python constructs therefore we
        # are using ParserPython class.
        # Skipping of whitespace will be done only for tabs and spaces. Newlines
        # have semantics in csv files. They are used to separate records.
        parser = ParserPython(csvfile, ws='\t ', reduce_tree=True, debug=True)

        # Then we export it to a dot file in order to visualise it. This is
{'fraction_non_alphanumeric': 0.193158953722334, 'fraction_numerical': 0.026731819488358725, 'mean_word_length': 3.3828715365239295, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7437583', 'n_tokens_mistral': 1089, 'n_tokens_neox': 966, 'n_words': 369}