<commit_before># Natural Language Toolkit: Shoebox Reader
#
# Copyright (C) 2001-2005 University of Pennsylvania
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT

# For a more sophisticated, validating Shoebox parser,
# please see the nltk_contrib.misc.shoebox package,
# contributed by <NAME> <<EMAIL>>

from nltk_lite.corpora import get_basedir
from nltk_lite import tokenize
from string import split
import os

def _parse_entry(str):
    for field in tokenize.line(str):
        field = field[1:].strip()   # remove leading backslash and trailing whitespace
        field = tuple(split(field, sep=" ", maxsplit=1))
        if len(field) == 1:
            field = (field[0], None)
        yield field

def raw(files = 'rotokas'):
    """
    Read entries from the Shoebox corpus sample, returning a list
    of pairs, where each pair consists of a field name and value

    Contents: lexicons for various languages, with entries of the form:

    \\lx kaakauko
    \\ps N.M
    \\ge gray weevil
    \\sf FAUNA.INSECT
    \\nt pictured on PNG postage stamp
    \\dt 28/Jan/2005
    \\ex Kaakauko ira toupareveira aue-ia niugini stemp.
    \\xp Kaakauko em insect em i istap long niugini.
    \\xe The gray weevil is found on the PNG stamp.
    \\ex Kaakauko iria toupaeveira niugini stamia.
    \\xp Weevil i stap long niguini stamp.
    \\xe The gray weevil is on the New Guinea stamp.

    @param files: One or more treebank files to be processed
    @type files: L{string} or L{tuple(string)}
    @param basedir: Base directory of the files
    @type basedir: L{string}
    @rtype: iterator over L{list(string)}
    """       

    # Just one file to process?  If so convert to a tuple so we can iterate
    if type(files) is str: files = (files,)

    for file in files:
        path = os.path.join(get_basedir(), "shoebox", file + ".dic")
        f = open(path).read()
        for entry in tokenize.blankline(f):
            yield list(_parse_entry(entry))

def dict(files = 'rotokas'):
    if type(files) is str: files = (files,)
    for entry in raw(files):
        yield dict(entry)

def _dict_list_entry(entry):
    d = {}
    for field in entry:
        if len(field) == 2:
            name, value = field
            if name not in d:
                d[name] = []
            d[name].append(value)
    return d

def dict_list(files = 'rotokas'):
    # Just one file to process?  If so convert to a tuple so we can iterate
    if type(files) is str: files = (files,)

    for entry in raw(files):
        yield _dict_list_entry(entry)

def demo():
    from nltk_lite.corpora import shoebox
    from itertools import islice
    from pprint import pprint

    print 'Raw:'
    pprint(list(islice(shoebox.raw(), 3)))

    print 'Dictionary:'
    pprint(list(islice(shoebox.dict(), 3)))

    print 'Dictionary-List:'
    pprint(list(islice(shoebox.dict_list(), 3)))

if __name__ == '__main__':
    demo()


<commit_msg>fixed naming problem<commit_after># Natural Language Toolkit: Shoebox Reader
#
# Copyright (C) 2001-2005 University of Pennsylvania
# Author: <NAME> <<EMAIL>>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT

# For a more sophisticated, validating Shoebox parser,
# please see the nltk_contrib.misc.shoebox package,
# contributed by <NAME> <<EMAIL>.<EMAIL>>

from nltk_lite.corpora import get_basedir
from nltk_lite import tokenize
from string import split
from itertools import imap
import os

def _parse_entry(str):
    for field in tokenize.line(str):
        field = field[1:].strip()   # remove leading backslash and trailing whitespace
        field = tuple(split(field, sep=" ", maxsplit=1))
        if len(field) == 1:
            field = (field[0], None)
        yield field

def raw(files = 'rotokas'):
    """
    Read entries from the Shoebox corpus sample, returning a list
    of pairs, where each pair consists of a field name and value

    Contents: lexicons for various languages, with entries of the form:

    \\lx kaakauko
    \\ps N.M
    \\ge gray weevil
    \\sf FAUNA.INSECT
    \\nt pictured on PNG postage stamp
    \\dt 28/Jan/2005
    \\ex Kaakauko ira toupareveira aue-ia niugini stemp.
    \\xp Kaakauko em insect em i istap long niugini.
    \\xe The gray weevil is found on the PNG stamp.
    \\ex Kaakauko iria toupaeveira niugini stamia.
    \\xp Weevil i stap long niguini stamp.
    \\xe The gray weevil is on the New Guinea stamp.

    @param files: One or more treebank files to be processed
    @type files: L{string} or L{tuple(string)}
    @param basedir: Base directory of the files
    @type basedir: L{string}
    @rtype: iterator over L{list(string)}
    """       

    # Just one file to process?  If so convert to a tuple so we can iterate
    if type(files) is str: files = (files,)

    for file in files:
        print "FILE:", file
        path = os.path.join(get_basedir(), "shoebox", file + ".dic")
        f = open(path).read()
        for entry in tokenize.blankline(f):
            yield list(_parse_entry(entry))

def dictionary(files = 'rotokas'):
    return imap(dict, raw(files))

def _dict_list_entry(entry):
    d = {}
    for field in entry:
        if len(field) == 2:
            name, value = field
            if name not in d:
                d[name] = []
            d[name].append(value)
    return d

def dict_list(files = 'rotokas'):
    # Just one file to process?  If so convert to a tuple so we can iterate
    if type(files) is str: files = (files,)

    for entry in raw(files):
        yield _dict_list_entry(entry)

def demo():
    from nltk_lite.corpora import shoebox
    from itertools import islice
    from pprint import pprint

    print 'Raw:'
    pprint(list(islice(shoebox.raw(), 3)))

    print 'Dictionary:'
    pprint(list(islice(shoebox.dictionary(), 3)))

    print 'Dictionary-List:'
    pprint(list(islice(shoebox.dict_list(), 3)))

if __name__ == '__main__':
    demo()



{'fraction_non_alphanumeric': 0.1035696329813977, 'fraction_numerical': 0.007373889726830903, 'mean_word_length': 2.751099937146449, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7466098', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1911, 'n_words': 686}