natural language toolkit chunked corpus reader c 20012023 nltk project steven bird stevenbird1gmail com edward loper edlopergmail com url https www nltk org for license information see license txt a reader for corpora that contain chunked and optionally tagged documents reader for chunked and optionally tagged corpora paragraphs are split using a block reader they are then tokenized into sentences using a sentence tokenizer finally these sentences are parsed into chunk trees using a stringtochunktree conversion function each of these steps can be performed using a default function or a custom function by default paragraphs are split on blank lines sentences are listed one per line and sentences are parsed into chunk trees using nltk chunk tagstr2tree param root the root directory for this corpus param fileids a list or regexp specifying the fileids in this corpus arguments for corpus views generated by this corpus a tuple str2chunktree senttokenizer parablocktokenizer def wordsself fileidsnone return concat chunkedcorpusviewf enc 0 0 0 0 self cvargs for f enc in self abspathsfileids true def sentsself fileidsnone return concat chunkedcorpusviewf enc 0 1 0 0 self cvargs for f enc in self abspathsfileids true def parasself fileidsnone return concat chunkedcorpusviewf enc 0 1 1 0 self cvargs for f enc in self abspathsfileids true def taggedwordsself fileidsnone tagsetnone return concat chunkedcorpusview f enc 1 0 0 0 self cvargs targettagsettagset for f enc in self abspathsfileids true def taggedsentsself fileidsnone tagsetnone return concat chunkedcorpusview f enc 1 1 0 0 self cvargs targettagsettagset for f enc in self abspathsfileids true def taggedparasself fileidsnone tagsetnone return concat chunkedcorpusview f enc 1 1 1 0 self cvargs targettagsettagset for f enc in self abspathsfileids true def chunkedwordsself fileidsnone tagsetnone return concat chunkedcorpusview f enc 1 0 0 1 self cvargs targettagsettagset for f enc in self abspathsfileids true def chunkedsentsself fileidsnone tagsetnone return concat chunkedcorpusview f enc 1 1 0 1 self cvargs targettagsettagset for f enc in self abspathsfileids true def chunkedparasself fileidsnone tagsetnone return concat chunkedcorpusview f enc 1 1 1 1 self cvargs targettagsettagset for f enc in self abspathsfileids true def readblockself stream return tagstr2treet for t in readblanklineblockstream class chunkedcorpusviewstreambackedcorpusview def init self fileid encoding tagged groupbysent groupbypara chunked str2chunktree senttokenizer parablockreader sourcetagsetnone targettagsetnone streambackedcorpusview initself fileid encodingencoding self tagged tagged self groupbysent groupbysent self groupbypara groupbypara self chunked chunked self str2chunktree str2chunktree self senttokenizer senttokenizer self parablockreader parablockreader self sourcetagset sourcetagset self targettagset targettagset def readblockself stream block for parastr in self parablockreaderstream para for sentstr in self senttokenizer tokenizeparastr sent self str2chunktree sentstr sourcetagsetself sourcetagset targettagsetself targettagset if requested throw away the tags if not self tagged sent self untagsent if requested throw away the chunks if not self chunked sent sent leaves add the sentence to para if self groupbysent para appendsent else para extendsent add the paragraph to block if self groupbypara block appendpara else block extendpara return the block return block def untagself tree for i child in enumeratetree if isinstancechild tree self untagchild elif isinstancechild tuple treei child0 else raise valueerrorexpected child to be tree or tuple return tree natural language toolkit chunked corpus reader c 2001 2023 nltk project steven bird stevenbird1 gmail com edward loper edloper gmail com url https www nltk org for license information see license txt a reader for corpora that contain chunked and optionally tagged documents reader for chunked and optionally tagged corpora paragraphs are split using a block reader they are then tokenized into sentences using a sentence tokenizer finally these sentences are parsed into chunk trees using a string to chunktree conversion function each of these steps can be performed using a default function or a custom function by default paragraphs are split on blank lines sentences are listed one per line and sentences are parsed into chunk trees using nltk chunk tagstr2tree param root the root directory for this corpus param fileids a list or regexp specifying the fileids in this corpus arguments for corpus views generated by this corpus a tuple str2chunktree sent_tokenizer para_block_tokenizer return the given file s as a list of words and punctuation symbols rtype list str return the given file s as a list of sentences or utterances each encoded as a list of word strings rtype list list str return the given file s as a list of paragraphs each encoded as a list of sentences which are in turn encoded as lists of word strings rtype list list list str return the given file s as a list of tagged words and punctuation symbols encoded as tuples word tag rtype list tuple str str return the given file s as a list of sentences each encoded as a list of word tag tuples rtype list list tuple str str return the given file s as a list of paragraphs each encoded as a list of sentences which are in turn encoded as lists of word tag tuples rtype list list list tuple str str return the given file s as a list of tagged words and chunks words are encoded as word tag tuples if the corpus has tags or word strings if the corpus has no tags chunks are encoded as depth one trees over word tag tuples or word strings rtype list tuple str str and tree return the given file s as a list of sentences each encoded as a shallow tree the leaves of these trees are encoded as word tag tuples if the corpus has tags or word strings if the corpus has no tags rtype list tree return the given file s as a list of paragraphs each encoded as a list of sentences which are in turn encoded as a shallow tree the leaves of these trees are encoded as word tag tuples if the corpus has tags or word strings if the corpus has no tags rtype list list tree if requested throw away the tags if requested throw away the chunks add the sentence to para add the paragraph to block return the block
import codecs
import os.path

import nltk
from nltk.chunk import tagstr2tree
from nltk.corpus.reader.api import *
from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
from nltk.corpus.reader.util import *
from nltk.tokenize import *
from nltk.tree import Tree


class ChunkedCorpusReader(CorpusReader):
    

    def __init__(
        self,
        root,
        fileids,
        extension="",
        str2chunktree=tagstr2tree,
        sent_tokenizer=RegexpTokenizer("\n", gaps=True),
        para_block_reader=read_blankline_block,
        encoding="utf8",
        tagset=None,
    ):
        
        CorpusReader.__init__(self, root, fileids, encoding)
        self._cv_args = (str2chunktree, sent_tokenizer, para_block_reader, tagset)
        

    def words(self, fileids=None):
        
        return concat(
            [
                ChunkedCorpusView(f, enc, 0, 0, 0, 0, *self._cv_args)
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def sents(self, fileids=None):
        
        return concat(
            [
                ChunkedCorpusView(f, enc, 0, 1, 0, 0, *self._cv_args)
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def paras(self, fileids=None):
        
        return concat(
            [
                ChunkedCorpusView(f, enc, 0, 1, 1, 0, *self._cv_args)
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def tagged_words(self, fileids=None, tagset=None):
        
        return concat(
            [
                ChunkedCorpusView(
                    f, enc, 1, 0, 0, 0, *self._cv_args, target_tagset=tagset
                )
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def tagged_sents(self, fileids=None, tagset=None):
        
        return concat(
            [
                ChunkedCorpusView(
                    f, enc, 1, 1, 0, 0, *self._cv_args, target_tagset=tagset
                )
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def tagged_paras(self, fileids=None, tagset=None):
        
        return concat(
            [
                ChunkedCorpusView(
                    f, enc, 1, 1, 1, 0, *self._cv_args, target_tagset=tagset
                )
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def chunked_words(self, fileids=None, tagset=None):
        
        return concat(
            [
                ChunkedCorpusView(
                    f, enc, 1, 0, 0, 1, *self._cv_args, target_tagset=tagset
                )
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def chunked_sents(self, fileids=None, tagset=None):
        
        return concat(
            [
                ChunkedCorpusView(
                    f, enc, 1, 1, 0, 1, *self._cv_args, target_tagset=tagset
                )
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def chunked_paras(self, fileids=None, tagset=None):
        
        return concat(
            [
                ChunkedCorpusView(
                    f, enc, 1, 1, 1, 1, *self._cv_args, target_tagset=tagset
                )
                for (f, enc) in self.abspaths(fileids, True)
            ]
        )

    def _read_block(self, stream):
        return [tagstr2tree(t) for t in read_blankline_block(stream)]


class ChunkedCorpusView(StreamBackedCorpusView):
    def __init__(
        self,
        fileid,
        encoding,
        tagged,
        group_by_sent,
        group_by_para,
        chunked,
        str2chunktree,
        sent_tokenizer,
        para_block_reader,
        source_tagset=None,
        target_tagset=None,
    ):
        StreamBackedCorpusView.__init__(self, fileid, encoding=encoding)
        self._tagged = tagged
        self._group_by_sent = group_by_sent
        self._group_by_para = group_by_para
        self._chunked = chunked
        self._str2chunktree = str2chunktree
        self._sent_tokenizer = sent_tokenizer
        self._para_block_reader = para_block_reader
        self._source_tagset = source_tagset
        self._target_tagset = target_tagset

    def read_block(self, stream):
        block = []
        for para_str in self._para_block_reader(stream):
            para = []
            for sent_str in self._sent_tokenizer.tokenize(para_str):
                sent = self._str2chunktree(
                    sent_str,
                    source_tagset=self._source_tagset,
                    target_tagset=self._target_tagset,
                )

                
                if not self._tagged:
                    sent = self._untag(sent)

                
                if not self._chunked:
                    sent = sent.leaves()

                
                if self._group_by_sent:
                    para.append(sent)
                else:
                    para.extend(sent)

            
            if self._group_by_para:
                block.append(para)
            else:
                block.extend(para)

        
        return block

    def _untag(self, tree):
        for i, child in enumerate(tree):
            if isinstance(child, Tree):
                self._untag(child)
            elif isinstance(child, tuple):
                tree[i] = child[0]
            else:
                raise ValueError("expected child to be Tree or tuple")
        return tree