natural language toolkit string category corpus reader c 20012023 nltk project steven bird stevenbird1gmail com edward loper edlopergmail com url https www nltk org for license information see license txt read tuples from a corpus consisting of categorized strings for example from the question classification corpus num dist how far is it from denver to aspen loc city what county is modesto california in hum desc who was galileo desc def what is an atom num date when did hawaii become a state based on ppattachmentcorpusreader xx should the order of the tuple be reversed in most other places in nltk we use the form data tag e g tagged words and labeled texts for classifiers param root the root directory for this corpus param fileids a list or regexp specifying the fileids in this corpus param delimiter field delimiter natural language toolkit string category corpus reader c 2001 2023 nltk project steven bird stevenbird1 gmail com edward loper edloper gmail com url https www nltk org for license information see license txt read tuples from a corpus consisting of categorized strings for example from the question classification corpus num dist how far is it from denver to aspen loc city what county is modesto california in hum desc who was galileo desc def what is an atom num date when did hawaii become a state based on ppattachmentcorpusreader xx should the order of the tuple be reversed in most other places in nltk we use the form data tag e g tagged words and labeled texts for classifiers param root the root directory for this corpus param fileids a list or regexp specifying the fileids in this corpus param delimiter field delimiter
from nltk.corpus.reader.api import *


from nltk.corpus.reader.util import *





class StringCategoryCorpusReader(CorpusReader):
    def __init__(self, root, fileids, delimiter=" ", encoding="utf8"):
        
        CorpusReader.__init__(self, root, fileids, encoding)
        self._delimiter = delimiter

    def tuples(self, fileids=None):
        if fileids is None:
            fileids = self._fileids
        elif isinstance(fileids, str):
            fileids = [fileids]
        return concat(
            [
                StreamBackedCorpusView(fileid, self._read_tuple_block, encoding=enc)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def _read_tuple_block(self, stream):
        line = stream.readline().strip()
        if line:
            return [tuple(line.split(self._delimiter, 1))]
        else:
            return []