c 20012023 nltk project masato hagiwara hagisangmail com url https www nltk org for license information see license txt a specialized corpus view for chasenreader similar to taggedcorpusview but this ll use fixed sets of word and sentence tokenizer reads one paragraph at a time block for parastr in readregexpblockstream r reosn para sent for line in parastr splitlines eos line strip eos cells line splitt w cells0 t joincells1 if not eos sent appendw if eos or self sentsplitter and self sentsplitterw if not self tagged sent w for w t in sent if self groupbysent para appendsent else para extendsent sent if lensent 0 if not self tagged sent w for w t in sent if self groupbysent para appendsent else para extendsent if self groupbypara block appendpara else block extendpara return block def demo import nltk from nltk corpus util import lazycorpusloader jeita lazycorpusloaderjeita chasencorpusreader r chasen encodingutf8 print joinjeita words22100 22140 print neosn join n join formatw0 w1 splitt2 for w in sent for sent in jeita taggedsents2170 2173 def test from nltk corpus util import lazycorpusloader jeita lazycorpusloaderjeita chasencorpusreader r chasen encodingutf8 assert isinstancejeita taggedwords01 str if name main demo test c 2001 2023 nltk project masato hagiwara hagisan gmail com url https www nltk org for license information see license txt a specialized corpus view for chasenreader similar to taggedcorpusview but this ll use fixed sets of word and sentence tokenizer reads one paragraph at a time
import sys

from nltk.corpus.reader import util
from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *


class ChasenCorpusReader(CorpusReader):
    def __init__(self, root, fileids, encoding="utf8", sent_splitter=None):
        self._sent_splitter = sent_splitter
        CorpusReader.__init__(self, root, fileids, encoding)

    def words(self, fileids=None):
        return concat(
            [
                ChasenCorpusView(fileid, enc, False, False, False, self._sent_splitter)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def tagged_words(self, fileids=None):
        return concat(
            [
                ChasenCorpusView(fileid, enc, True, False, False, self._sent_splitter)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def sents(self, fileids=None):
        return concat(
            [
                ChasenCorpusView(fileid, enc, False, True, False, self._sent_splitter)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def tagged_sents(self, fileids=None):
        return concat(
            [
                ChasenCorpusView(fileid, enc, True, True, False, self._sent_splitter)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def paras(self, fileids=None):
        return concat(
            [
                ChasenCorpusView(fileid, enc, False, True, True, self._sent_splitter)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def tagged_paras(self, fileids=None):
        return concat(
            [
                ChasenCorpusView(fileid, enc, True, True, True, self._sent_splitter)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )


class ChasenCorpusView(StreamBackedCorpusView):
    

    def __init__(
        self,
        corpus_file,
        encoding,
        tagged,
        group_by_sent,
        group_by_para,
        sent_splitter=None,
    ):
        self._tagged = tagged
        self._group_by_sent = group_by_sent
        self._group_by_para = group_by_para
        self._sent_splitter = sent_splitter
        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)

    def read_block(self, stream):
        
        block = []
        for para_str in read_regexp_block(stream, r".", r"^EOS\n"):

            para = []

            sent = []
            for line in para_str.splitlines():

                _eos = line.strip() == "EOS"
                _cells = line.split("\t")
                w = (_cells[0], "\t".join(_cells[1:]))
                if not _eos:
                    sent.append(w)

                if _eos or (self._sent_splitter and self._sent_splitter(w)):
                    if not self._tagged:
                        sent = [w for (w, t) in sent]
                    if self._group_by_sent:
                        para.append(sent)
                    else:
                        para.extend(sent)
                    sent = []

            if len(sent) > 0:
                if not self._tagged:
                    sent = [w for (w, t) in sent]

                if self._group_by_sent:
                    para.append(sent)
                else:
                    para.extend(sent)

            if self._group_by_para:
                block.append(para)
            else:
                block.extend(para)

        return block


def demo():

    import nltk
    from nltk.corpus.util import LazyCorpusLoader

    jeita = LazyCorpusLoader("jeita", ChasenCorpusReader, r".*chasen", encoding="utf-8")
    print("/".join(jeita.words()[22100:22140]))

    print(
        "\nEOS\n".join(
            "\n".join("{}/{}".format(w[0], w[1].split("\t")[2]) for w in sent)
            for sent in jeita.tagged_sents()[2170:2173]
        )
    )


def test():

    from nltk.corpus.util import LazyCorpusLoader

    jeita = LazyCorpusLoader("jeita", ChasenCorpusReader, r".*chasen", encoding="utf-8")

    assert isinstance(jeita.tagged_words()[0][1], str)


if __name__ == "__main__":
    demo()
    test()