natural language toolkit ieer corpus reader c 20012023 nltk project steven bird stevenbird1gmail com edward loper edlopergmail com url https www nltk org for license information see license txt corpus reader for the information extraction and entity recognition corpus nist 1999 information extraction entity recognition evaluation https www itl nist goviad894 01testsieerer99er99 htm this corpus contains the newswire development test data for the nist 1999 ieer evaluation the files were taken from the subdirectory ieer99englishdevtestnewswire ref nwt and filenames were shortened the corpus contains the following files apw19980314 apw19980424 apw19980429 nyt19980315 nyt19980403 and nyt19980407 a dictionary whose keys are the names of documents in this corpus and whose values are descriptions of those documents contents a list of all documents in this corpus def docsself fileidsnone return concat streambackedcorpusviewfileid self readblock encodingenc for fileid enc in self abspathsfileids true def parseddocsself fileidsnone return concat streambackedcorpusviewfileid self readparsedblock encodingenc for fileid enc in self abspathsfileids true def readparsedblockself stream todo figure out while empty documents are being returned return self parsedoc for doc in self readblockstream if self parsedoc docno is not none def parseself doc val nltk chunk ieerstr2treedoc rootlabeldocument if isinstanceval dict return ieerdocumentval else return ieerdocumentval def readblockself stream out skip any preamble while true line stream readline if not line break if line strip doc break out appendline read the document while true line stream readline if not line break out appendline if line strip doc break return the document return n joinout natural language toolkit ieer corpus reader c 2001 2023 nltk project steven bird stevenbird1 gmail com edward loper edloper gmail com url https www nltk org for license information see license txt corpus reader for the information extraction and entity recognition corpus nist 1999 information extraction entity recognition evaluation https www itl nist gov iad 894 01 tests ie er er_99 er_99 htm this corpus contains the newswire development test data for the nist 1999 ie er evaluation the files were taken from the subdirectory ie_er_99 english devtest newswire ref nwt and filenames were shortened the corpus contains the following files apw_19980314 apw_19980424 apw_19980429 nyt_19980315 nyt_19980403 and nyt_19980407 a dictionary whose keys are the names of documents in this corpus and whose values are descriptions of those documents contents a list of all documents in this corpus todo figure out while empty documents are being returned skip any preamble read the document return the document
import nltk
from nltk.corpus.reader.api import *



titles = {
    "APW_19980314": "Associated Press Weekly, 14 March 1998",
    "APW_19980424": "Associated Press Weekly, 24 April 1998",
    "APW_19980429": "Associated Press Weekly, 29 April 1998",
    "NYT_19980315": "New York Times, 15 March 1998",
    "NYT_19980403": "New York Times, 3 April 1998",
    "NYT_19980407": "New York Times, 7 April 1998",
}


documents = sorted(titles)


class IEERDocument:
    def __init__(self, text, docno=None, doctype=None, date_time=None, headline=""):
        self.text = text
        self.docno = docno
        self.doctype = doctype
        self.date_time = date_time
        self.headline = headline

    def __repr__(self):
        if self.headline:
            headline = " ".join(self.headline.leaves())
        else:
            headline = (
                " ".join([w for w in self.text.leaves() if w[:1] != "<"][:12]) + "..."
            )
        if self.docno is not None:
            return f"<IEERDocument {self.docno}: {headline!r}>"
        else:
            return "<IEERDocument: %r>" % headline


class IEERCorpusReader(CorpusReader):
    

    def docs(self, fileids=None):
        return concat(
            [
                StreamBackedCorpusView(fileid, self._read_block, encoding=enc)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def parsed_docs(self, fileids=None):
        return concat(
            [
                StreamBackedCorpusView(fileid, self._read_parsed_block, encoding=enc)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def _read_parsed_block(self, stream):
        
        return [
            self._parse(doc)
            for doc in self._read_block(stream)
            if self._parse(doc).docno is not None
        ]

    def _parse(self, doc):
        val = nltk.chunk.ieerstr2tree(doc, root_label="DOCUMENT")
        if isinstance(val, dict):
            return IEERDocument(**val)
        else:
            return IEERDocument(val)

    def _read_block(self, stream):
        out = []
        
        while True:
            line = stream.readline()
            if not line:
                break
            if line.strip() == "<DOC>":
                break
        out.append(line)
        
        while True:
            line = stream.readline()
            if not line:
                break
            out.append(line)
            if line.strip() == "</DOC>":
                break
        
        return ["\n".join(out)]