<commit_before># Natural Language Toolkit: Penn Treebank Reader
#
# Copyright (C) 2001-2005 University of Pennsylvania
# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT

from nltk_lite.corpora import get_basedir
from nltk_lite import tokenize
from nltk_lite.tag import tag2tuple
from nltk_lite.parse import tree
import os

def parsed(files = 'parsed'):

    """
    Read trees from the Penn Treebank corpus sample.

    This is a ~5% fragment of Penn Treebank, (C) LDC 1995.  It is made
    available under fair use for the purposes of illustrating NLTK
    tools for tokenizing, tagging, chunking and parsing.  This data is
    for non-commercial use only.

    Contents: parsed data from Wall Street Journal for 1650 sentences, e.g.:

    ( (S 
      (NP-SBJ 
        (NP (NNP Pierre) (NNP Vinken) )
        (, ,) 
        (ADJP 
          (NP (CD 61) (NNS years) )
          (JJ old) )
        (, ,) )
      (VP (MD will) 
        (VP (VB join) 
          (NP (DT the) (NN board) )
          (PP-CLR (IN as) 
            (NP (DT a) (JJ nonexecutive) (NN director) ))
          (NP-TMP (NNP Nov.) (CD 29) )))
      (. .) ))

    @param files: One or more treebank files to be processed
    @type files: L{string} or L{tuple(string)}
    @param basedir: Base directory of the files
    @type basedir: L{string}
    @rtype: iterator over L{tree}
    """       

    # Just one file to process?  If so convert to a tuple so we can iterate
    if type(files) is str: files = (files,)

    for file in files:
        path = os.path.join(get_basedir(), "treebank", file)
        s = open(path).read()
        for t in tokenize.blankline(s):
            print t
            yield tree.parse(t)


def chunked(files = 'chunked', basedir = "corpora/treebank"):

    """
    Read chunks from the Penn Treebank corpus sample.

    This is a ~5% fragment of Penn Treebank, (C) LDC 1995.  It is made
    available under fair use for the purposes of illustrating NLTK
    tools for tokenizing, tagging, chunking and parsing.  This data is
    for non-commercial use only.

    Contents: chunked data from Wall Street Journal for 1650 sentences, e.g.:

    [ Pierre/NNP Vinken/NNP ]
    ,/, 
    [ 61/CD years/NNS ]
    old/JJ ,/, will/MD join/VB 
    [ the/DT board/NN ]
    as/IN 
    [ a/DT nonexecutive/JJ director/NN Nov./NNP 29/CD ]
    ./.

    @param files: One or more treebank files to be processed<commit_msg>*** empty log message ***<commit_after># Natural Language Toolkit: Penn Treebank Reader
#
# Copyright (C) 2001-2005 University of Pennsylvania
# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT

from nltk_lite.corpora import get_basedir
from nltk_lite import tokenize
from nltk_lite.tag import tag2tuple
from nltk_lite.parse import tree
import os

def parsed(files = 'parsed'):

    """
    Read trees from the Penn Treebank corpus sample.

    This is a ~5% fragment of Penn Treebank, (C) LDC 1995.  It is made
    available under fair use for the purposes of illustrating NLTK
    tools for tokenizing, tagging, chunking and parsing.  This data is
    for non-commercial use only.

    Contents: parsed data from Wall Street Journal for 1650 sentences, e.g.:

    ( (S 
      (NP-SBJ 
        (NP (NNP Pierre) (NNP Vinken) )
        (, ,) 
        (ADJP 
          (NP (CD 61) (NNS years) )
          (JJ old) )
        (, ,) )
      (VP (MD will) 
        (VP (VB join) 
          (NP (DT the) (NN board) )
          (PP-CLR (IN as) 
            (NP (DT a) (JJ nonexecutive) (NN director) ))
          (NP-TMP (NNP Nov.) (CD 29) )))
      (. .) ))

    @param files: One or more treebank files to be processed
    @type files: L{string} or L{tuple(string)}
    @param basedir: Base directory of the files
    @type basedir: L{string}
    @rtype: iterator over L{tree}
    """       

    # Just one file to process?  If so convert to a tuple so we can iterate
    if type(files) is str: files = (files,)

    for file in files:
        path = os.path.join(get_basedir(), "treebank", file)
        s = open(path).read()
        for t in tokenize.blankline(s):
            print t
            yield tree.bracket_parse(t)


def chunked(files = 'chunked', basedir = "corpora/treebank"):

    """
    Read chunks from the Penn Treebank corpus sample.

    This is a ~5% fragment of Penn Treebank, (C) LDC 1995.  It is made
    available under fair use for the purposes of illustrating NLTK
    tools for tokenizing, tagging, chunking and parsing.  This data is
    for non-commercial use only.

    Contents: chunked data from Wall Street Journal for 1650 sentences, e.g.:

    [ Pierre/NNP Vinken/NNP ]
    ,/, 
    [ 61/CD years/NNS ]
    old/JJ ,/, will/MD join/VB 
    [ the/DT board/NN ]
    as/IN 
    [ a/DT nonexecutive/JJ director/NN Nov./NNP 29/CD ]
    ./.

    @param files: One or more treebank files to be processed
{'fraction_non_alphanumeric': 0.10517864311521477, 'fraction_numerical': 0.014050582095543959, 'mean_word_length': 2.3465413028878443, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1346115', 'n_tokens_mistral': 1736, 'n_tokens_neox': 1578, 'n_words': 601}