natural language toolkit rte corpus reader c 20012023 nltk project ewan klein ewaninf ed ac uk url https www nltk org for license information see license txt corpus reader for the recognizing textual entailment rte challenge corpora the files were taken from the rte1 rte2 and rte3 datasets and the files were regularized filenames are of the form rtedev xml and rtetest xml the latter are the gold standard annotated files each entailment corpus is a list of text hypothesis pairs the following example is taken from rte3 pair id1 entailmentyes taskie lengthshort tthe sale was made to pay yukos us 27 5 billion tax bill yuganskneftegaz was originally sold for us 9 4 billion to a little known company baikalfinansgroup which was later bought by the russian stateowned oil company rosneft t hbaikalfinansgroup was sold to rosneft h pair in order to provide globally unique ids for each pair a new attribute challenge has been added to the root element entailmentcorpus of each file taking values 1 2 or 3 the gid is formatted mn where m is the challenge number and n is the pair id normalize the string value in an rte pair s value or entailment attribute as an integer 1 0 param valuestring the label used to classify a texthypothesis pair type valuestring str rtype int container for rte texthypothesis pairs the entailment relation is signalled by the value attribute in rte1 and by entailment in rte2 and rte3 these both get mapped on to the entailment attribute of this class param challenge version of the rte challenge i e rte1 rte2 or rte3 param id identifier for the pair param text the text component of the pair param hyp the hypothesis component of the pair param value classification label for the pair param task attribute for the particular nlp task that the data was drawn from param length attribute for the length of the text of the pair corpus reader for corpora in rte challenges this is just a wrapper around the xmlcorpusreader see module docstring above for the expected structure of input documents map the xml input into an rtepair this uses the getiterator method from the elementtree package to find all the pair elements param doc a parsed xml document rtype listrtepair build a list of rtepairs from a rte corpus param fileids a list of rte corpus fileids type list rtype listrtepair natural language toolkit rte corpus reader c 2001 2023 nltk project ewan klein ewan inf ed ac uk url https www nltk org for license information see license txt corpus reader for the recognizing textual entailment rte challenge corpora the files were taken from the rte1 rte2 and rte3 datasets and the files were regularized filenames are of the form rte _dev xml and rte _test xml the latter are the gold standard annotated files each entailment corpus is a list of text hypothesis pairs the following example is taken from rte3 pair id 1 entailment yes task ie length short t the sale was made to pay yukos us 27 5 billion tax bill yuganskneftegaz was originally sold for us 9 4 billion to a little known company baikalfinansgroup which was later bought by the russian state owned oil company rosneft t h baikalfinansgroup was sold to rosneft h pair in order to provide globally unique ids for each pair a new attribute challenge has been added to the root element entailment corpus of each file taking values 1 2 or 3 the gid is formatted m n where m is the challenge number and n is the pair id normalize the string value in an rte pair s value or entailment attribute as an integer 1 0 param value_string the label used to classify a text hypothesis pair type value_string str rtype int container for rte text hypothesis pairs the entailment relation is signalled by the value attribute in rte1 and by entailment in rte2 and rte3 these both get mapped on to the entailment attribute of this class param challenge version of the rte challenge i e rte1 rte2 or rte3 param id identifier for the pair param text the text component of the pair param hyp the hypothesis component of the pair param value classification label for the pair param task attribute for the particular nlp task that the data was drawn from param length attribute for the length of the text of the pair corpus reader for corpora in rte challenges this is just a wrapper around the xmlcorpusreader see module docstring above for the expected structure of input documents map the xml input into an rtepair this uses the getiterator method from the elementtree package to find all the pair elements param doc a parsed xml document rtype list rtepair build a list of rtepairs from a rte corpus param fileids a list of rte corpus fileids type list rtype list rtepair
from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.corpus.reader.xmldocs import *


def norm(value_string):
    

    valdict = {"TRUE": 1, "FALSE": 0, "YES": 1, "NO": 0}
    return valdict[value_string.upper()]


class RTEPair:
    

    def __init__(
        self,
        pair,
        challenge=None,
        id=None,
        text=None,
        hyp=None,
        value=None,
        task=None,
        length=None,
    ):
        
        self.challenge = challenge
        self.id = pair.attrib["id"]
        self.gid = f"{self.challenge}-{self.id}"
        self.text = pair[0].text
        self.hyp = pair[1].text

        if "value" in pair.attrib:
            self.value = norm(pair.attrib["value"])
        elif "entailment" in pair.attrib:
            self.value = norm(pair.attrib["entailment"])
        else:
            self.value = value
        if "task" in pair.attrib:
            self.task = pair.attrib["task"]
        else:
            self.task = task
        if "length" in pair.attrib:
            self.length = pair.attrib["length"]
        else:
            self.length = length

    def __repr__(self):
        if self.challenge:
            return f"<RTEPair: gid={self.challenge}-{self.id}>"
        else:
            return "<RTEPair: id=%s>" % self.id


class RTECorpusReader(XMLCorpusReader):
    

    def _read_etree(self, doc):
        
        try:
            challenge = doc.attrib["challenge"]
        except KeyError:
            challenge = None
        pairiter = doc.iter("pair")
        return [RTEPair(pair, challenge=challenge) for pair in pairiter]

    def pairs(self, fileids):
        
        if isinstance(fileids, str):
            fileids = [fileids]
        return concat([self._read_etree(self.xml(fileid)) for fileid in fileids])