natural language toolkit indian language postagged corpus reader c 20012023 nltk project steven bird stevenbird1gmail com edward loper edlopergmail com url https www nltk org for license information see license txt indian language postagged corpus collected by a kumaran microsoft research india distributed with permission contents bangla iit kharagpur hindi microsoft research india marathi iit bombay telugu iiit hyderabad list of words one per line blank lines are ignored natural language toolkit indian language pos tagged corpus reader c 2001 2023 nltk project steven bird stevenbird1 gmail com edward loper edloper gmail com url https www nltk org for license information see license txt indian language pos tagged corpus collected by a kumaran microsoft research india distributed with permission contents bangla iit kharagpur hindi microsoft research india marathi iit bombay telugu iiit hyderabad list of words one per line blank lines are ignored
from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.tag import map_tag, str2tuple


class IndianCorpusReader(CorpusReader):
    

    def words(self, fileids=None):
        return concat(
            [
                IndianCorpusView(fileid, enc, False, False)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def tagged_words(self, fileids=None, tagset=None):
        if tagset and tagset != self._tagset:
            tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
        else:
            tag_mapping_function = None
        return concat(
            [
                IndianCorpusView(fileid, enc, True, False, tag_mapping_function)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def sents(self, fileids=None):
        return concat(
            [
                IndianCorpusView(fileid, enc, False, True)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def tagged_sents(self, fileids=None, tagset=None):
        if tagset and tagset != self._tagset:
            tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
        else:
            tag_mapping_function = None
        return concat(
            [
                IndianCorpusView(fileid, enc, True, True, tag_mapping_function)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )


class IndianCorpusView(StreamBackedCorpusView):
    def __init__(
        self, corpus_file, encoding, tagged, group_by_sent, tag_mapping_function=None
    ):
        self._tagged = tagged
        self._group_by_sent = group_by_sent
        self._tag_mapping_function = tag_mapping_function
        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)

    def read_block(self, stream):
        line = stream.readline()
        if line.startswith("<"):
            return []
        sent = [str2tuple(word, sep="_") for word in line.split()]
        if self._tag_mapping_function:
            sent = [(w, self._tag_mapping_function(t)) for (w, t) in sent]
        if not self._tagged:
            sent = [w for (w, t) in sent]
        if self._group_by_sent:
            return [sent]
        else:
            return sent