<commit_before>#!/usr/bin/env python

# Natural Language Toolkit: Toolbox Reader
#
# Copyright (C) 2001-2009 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.org>
# For license information, see LICENSE.TXT

"""
Module for reading, writing and manipulating 
Toolbox databases and settings files.
"""

import os, re, codecs
from StringIO import StringIO
from nltk.etree.ElementTree import TreeBuilder, Element, SubElement
from nltk.data import PathPointer, ZipFilePathPointer
from nltk import data

class StandardFormat(object):
    """
    Class for reading and processing standard format marker files and strings.
    """
    def __init__(self, filename=None, encoding=None):
        self._encoding = encoding
        if filename is not None:
            self.open(filename)

    def open(self, sfm_file):
        """Open a standard format marker file for sequential reading. 
        
        @param sfm_file: name of the standard format marker input file
        @type sfm_file: C{string}
        """
        if isinstance(sfm_file, PathPointer):
            # [xx] We don't use 'rU' mode here -- do we need to?
            #      (PathPointer.open doesn't take a mode option)
            self._file = sfm_file.open(self._encoding)
        else:
            self._file = codecs.open(sfm_file, 'rU', self._encoding)

    def open_string(self, s):
        """Open a standard format marker string for sequential reading. 
        
        @param s: string to parse as a standard format marker input file
        @type s: C{string}
        """
        self._file = StringIO(s)

    def raw_fields(self):
        """Return an iterator for the fields in the standard format marker
        file.
        
        @return: an iterator that returns the next field in a (marker, value) 
            tuple. Linebreaks and trailing white space are preserved except 
            for the final newline in each field.
        @rtype: iterator over C{(marker, value)} tuples
        """
        join_string = '\n'
        line_regexp = r'^%s(?:\\(\S+)\s*)?(.*)$'
        first_line_pat = re.compile(line_regexp % u'\ufeff?')
        line_pat = re.compile(line_regexp % '')
        # need to get first line outside the loop for correct handling
        # of the first marker if it spans multiple lines
        file_iter = iter(self._file)
        line = file_iter.next()
        mobj = re.match(first_line_pat, line)<commit_msg>fixed bug with BOM handling<commit_after>#!/usr/bin/env python

# Natural Language Toolkit: Toolbox Reader
#
# Copyright (C) 2001-2009 NLTK Project
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.org>
# For license information, see LICENSE.TXT

"""
Module for reading, writing and manipulating 
Toolbox databases and settings files.
"""

import os, re, codecs
from StringIO import StringIO
from nltk.etree.ElementTree import TreeBuilder, Element, SubElement
from nltk.data import PathPointer, ZipFilePathPointer
from nltk import data

class StandardFormat(object):
    """
    Class for reading and processing standard format marker files and strings.
    """
    def __init__(self, filename=None, encoding=None):
        self._encoding = encoding
        if filename is not None:
            self.open(filename)

    def open(self, sfm_file):
        """Open a standard format marker file for sequential reading. 
        
        @param sfm_file: name of the standard format marker input file
        @type sfm_file: C{string}
        """
        if isinstance(sfm_file, PathPointer):
            # [xx] We don't use 'rU' mode here -- do we need to?
            #      (PathPointer.open doesn't take a mode option)
            self._file = sfm_file.open(self._encoding)
        else:
            self._file = codecs.open(sfm_file, 'rU', self._encoding)

    def open_string(self, s):
        """Open a standard format marker string for sequential reading. 
        
        @param s: string to parse as a standard format marker input file
        @type s: C{string}
        """
        self._file = StringIO(s)

    def raw_fields(self):
        """Return an iterator for the fields in the standard format marker
        file.
        
        @return: an iterator that returns the next field in a (marker, value) 
            tuple. Linebreaks and trailing white space are preserved except 
            for the final newline in each field.
        @rtype: iterator over C{(marker, value)} tuples
        """
        join_string = '\n'
        line_regexp = r'^%s(?:\\(\S+)\s*)?(.*)$'
        # discard a BOM in the first line
        first_line_pat = re.compile(line_regexp % u'(?:\ufeff)?'.encode('utf8'))
        line_pat = re.compile(line_regexp % '')
        # need to get first line outside the loop for correct handling
        # of the first marker if it spans multiple lines
        file_iter = iter(self._file)
        line = file_iter.next()
        mobj = re.match(first_line_pat, line)
{'fraction_non_alphanumeric': 0.09465772902701605, 'fraction_numerical': 0.0034531789559211863, 'mean_word_length': 2.507122507122507, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4787964', 'n_tokens_mistral': 1420, 'n_tokens_neox': 1371, 'n_words': 502}