natural language toolkit penn treebank reader c 20012023 nltk project steven bird stevenbird1gmail com edward loper edlopergmail com url https www nltk org for license information see license txt corpus reader for corpora that consist of parenthesisdelineated parse trees we use s instead of s to avoid matching reader for corpora that consist of parenthesisdelineated parse trees like those found in the combined section of the penn treebank e g s np dt the jj little nn dog vp vbd barked param root the root directory for this corpus param fileids a list or regexp specifying the fileids in this corpus param commentchar the character which can appear at the start of a line to indicate that the rest of the line is a comment param detectblocks the method that is used to find blocks in the corpus can be unindentedparen every unindented parenthesis starts a new parse or sexpr brackets are matched param tagset the name of the tagset used by this corpus to be used for normalizing or converting the pos tags returned by the tagged methods tokens start with unindented left parens strip any comments out of the tokens replace leaves of the form with replace leaves of the form tag word root with tag word if there s an empty node at the top strip it off try to recover if we can try something else sys stderr write joint split n a reader for parsed corpora whose documents are divided into categories based on their file identifiers nathan schneider nschneidcs cmu edu initialize the corpus reader categorization arguments ccatpattern ccatmap and ccatfile are passed to the lcategorizedcorpusreader constructor categorizedcorpusreader init the remaining arguments are passed to the lbracketparsecorpusreader constructor bracketparsecorpusreader init reader for the alpino dutch treebank this corpus has a lexical breakdown structure embedded as read by parse unfortunately this puts punctuation and some other words out of the sentence order in the xml element tree this is no good for tag and word tag and word will be overridden to use a nondefault new parameter ordered to the overridden normalize function the parse function can then remain untouched normalize the xml sentence element in t the sentence elements alpinods although embedded in a few overall xml elements are separated by blank lines that s how the reader can deliver them one at a time each sentence has a few category subnodes that are of no use to us the remaining word nodes may or may not appear in the proper order each word node has attributes among which begin the position of the word in the sentence pos part of speech the tag word the actual word the return value is a string with all xml elementes replaced by clauses either a cat clause with nested clauses or a word clause the order of the bracket clauses closely follows the xml if ordered true the word clauses include an order sequence number if ordered false the word clauses only have pos and word parts convert xml to sexpr notation return a correctly ordered list if words taggedsent self tagt return w for w p in taggedsent natural language toolkit penn treebank reader c 2001 2023 nltk project steven bird stevenbird1 gmail com edward loper edloper gmail com url https www nltk org for license information see license txt corpus reader for corpora that consist of parenthesis delineated parse trees we use s instead of s to avoid matching reader for corpora that consist of parenthesis delineated parse trees like those found in the combined section of the penn treebank e g s np dt the jj little nn dog vp vbd barked param root the root directory for this corpus param fileids a list or regexp specifying the fileids in this corpus param comment_char the character which can appear at the start of a line to indicate that the rest of the line is a comment param detect_blocks the method that is used to find blocks in the corpus can be unindented_paren every unindented parenthesis starts a new parse or sexpr brackets are matched param tagset the name of the tagset used by this corpus to be used for normalizing or converting the pos tags returned by the tagged_ methods tokens start with unindented left parens strip any comments out of the tokens replace leaves of the form with replace leaves of the form tag word root with tag word if there s an empty node at the top strip it off try to recover if we can try something else sys stderr write join t split n a reader for parsed corpora whose documents are divided into categories based on their file identifiers nathan schneider nschneid cs cmu edu initialize the corpus reader categorization arguments c cat_pattern c cat_map and c cat_file are passed to the l categorizedcorpusreader constructor categorizedcorpusreader __init__ the remaining arguments are passed to the l bracketparsecorpusreader constructor bracketparsecorpusreader __init__ reader for the alpino dutch treebank this corpus has a lexical breakdown structure embedded as read by _parse unfortunately this puts punctuation and some other words out of the sentence order in the xml element tree this is no good for tag_ and word_ _tag and _word will be overridden to use a non default new parameter ordered to the overridden _normalize function the _parse function can then remain untouched normalize the xml sentence element in t the sentence elements alpino_ds although embedded in a few overall xml elements are separated by blank lines that s how the reader can deliver them one at a time each sentence has a few category subnodes that are of no use to us the remaining word nodes may or may not appear in the proper order each word node has attributes among which begin the position of the word in the sentence pos part of speech the tag word the actual word the return value is a string with all xml elementes replaced by clauses either a cat clause with nested clauses or a word clause the order of the bracket clauses closely follows the xml if ordered true the word clauses include an order sequence number if ordered false the word clauses only have pos and word parts convert xml to sexpr notation return a correctly ordered list if words
import sys

from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.tag import map_tag
from nltk.tree import Tree


SORTTAGWRD = re.compile(r"\((\d+) ([^\s()]+) ([^\s()]+)\)")
TAGWORD = re.compile(r"\(([^\s()]+) ([^\s()]+)\)")
WORD = re.compile(r"\([^\s()]+ ([^\s()]+)\)")
EMPTY_BRACKETS = re.compile(r"\s*\(\s*\(")


class BracketParseCorpusReader(SyntaxCorpusReader):
    

    def __init__(
        self,
        root,
        fileids,
        comment_char=None,
        detect_blocks="unindented_paren",
        encoding="utf8",
        tagset=None,
    ):
        
        SyntaxCorpusReader.__init__(self, root, fileids, encoding)
        self._comment_char = comment_char
        self._detect_blocks = detect_blocks
        self._tagset = tagset

    def _read_block(self, stream):
        if self._detect_blocks == "sexpr":
            return read_sexpr_block(stream, comment_char=self._comment_char)
        elif self._detect_blocks == "blankline":
            return read_blankline_block(stream)
        elif self._detect_blocks == "unindented_paren":
            
            toks = read_regexp_block(stream, start_re=r"^\(")
            
            if self._comment_char:
                toks = [
                    re.sub("(?m)^%s.*" % re.escape(self._comment_char), "", tok)
                    for tok in toks
                ]
            return toks
        else:
            assert 0, "bad block type"

    def _normalize(self, t):
        
        t = re.sub(r"\((.)\)", r"(\1 \1)", t)
        
        t = re.sub(r"\(([^\s()]+) ([^\s()]+) [^\s()]+\)", r"(\1 \2)", t)
        return t

    def _parse(self, t):
        try:
            tree = Tree.fromstring(self._normalize(t))
            
            if tree.label() == "" and len(tree) == 1:
                return tree[0]
            else:
                return tree

        except ValueError as e:
            sys.stderr.write("Bad tree detected; trying to recover...\n")
            
            if e.args == ("mismatched parens",):
                for n in range(1, 5):
                    try:
                        v = Tree(self._normalize(t + ")" * n))
                        sys.stderr.write(
                            "  Recovered by adding %d close " "paren(s)\n" % n
                        )
                        return v
                    except ValueError:
                        pass
            
            sys.stderr.write("  Recovered by returning a flat parse.\n")
            
            return Tree("S", self._tag(t))

    def _tag(self, t, tagset=None):
        tagged_sent = [(w, p) for (p, w) in TAGWORD.findall(self._normalize(t))]
        if tagset and tagset != self._tagset:
            tagged_sent = [
                (w, map_tag(self._tagset, tagset, p)) for (w, p) in tagged_sent
            ]
        return tagged_sent

    def _word(self, t):
        return WORD.findall(self._normalize(t))


class CategorizedBracketParseCorpusReader(
    CategorizedCorpusReader, BracketParseCorpusReader
):
    

    def __init__(self, *args, **kwargs):
        
        CategorizedCorpusReader.__init__(self, kwargs)
        BracketParseCorpusReader.__init__(self, *args, **kwargs)

    def tagged_words(self, fileids=None, categories=None, tagset=None):
        return super().tagged_words(self._resolve(fileids, categories), tagset)

    def tagged_sents(self, fileids=None, categories=None, tagset=None):
        return super().tagged_sents(self._resolve(fileids, categories), tagset)

    def tagged_paras(self, fileids=None, categories=None, tagset=None):
        return super().tagged_paras(self._resolve(fileids, categories), tagset)

    def parsed_words(self, fileids=None, categories=None):
        return super().parsed_words(self._resolve(fileids, categories))

    def parsed_sents(self, fileids=None, categories=None):
        return super().parsed_sents(self._resolve(fileids, categories))

    def parsed_paras(self, fileids=None, categories=None):
        return super().parsed_paras(self._resolve(fileids, categories))


class AlpinoCorpusReader(BracketParseCorpusReader):
    

    def __init__(self, root, encoding="ISO-8859-1", tagset=None):
        BracketParseCorpusReader.__init__(
            self,
            root,
            r"alpino\.xml",
            detect_blocks="blankline",
            encoding=encoding,
            tagset=tagset,
        )

    def _normalize(self, t, ordered=False):
        
        if t[:10] != "<alpino_ds":
            return ""
        
        t = re.sub(r'  <node .*? cat="(\w+)".*>', r"(\1", t)
        if ordered:
            t = re.sub(
                r'  <node. *?begin="(\d+)".*? pos="(\w+)".*? word="([^"]+)".*?/>',
                r"(\1 \2 \3)",
                t,
            )
        else:
            t = re.sub(r'  <node .*?pos="(\w+)".*? word="([^"]+)".*?/>', r"(\1 \2)", t)
        t = re.sub(r"  </node>", r")", t)
        t = re.sub(r"<sentence>.*</sentence>", r"", t)
        t = re.sub(r"</?alpino_ds.*>", r"", t)
        return t

    def _tag(self, t, tagset=None):
        tagged_sent = [
            (int(o), w, p)
            for (o, p, w) in SORTTAGWRD.findall(self._normalize(t, ordered=True))
        ]
        tagged_sent.sort()
        if tagset and tagset != self._tagset:
            tagged_sent = [
                (w, map_tag(self._tagset, tagset, p)) for (o, w, p) in tagged_sent
            ]
        else:
            tagged_sent = [(w, p) for (o, w, p) in tagged_sent]
        return tagged_sent

    def _word(self, t):
        
        tagged_sent = self._tag(t)
        return [w for (w, p) in tagged_sent]