natural language toolkit senseval 2 corpus reader c 20012023 nltk project trevor cohn tacohncs mu oz au steven bird stevenbird1gmail com modifications url https www nltk org for license information see license txt read from the senseval 2 corpus senseval http www senseval org evaluation exercises for word sense disambiguation organized by aclsiglex https www siglex org prepared by ted pedersen tpederseumn edu university of minnesota https www d umn edutpedersedata html distributed with permission the nltk version of the senseval 2 files uses wellformed xml each instance of the ambiguous words hard interest line and serve is tagged with a sense identifier and supplied with context decide which lexical element we re in start of a lexical element start of an instance body of an instance end of an instance some santiy checks record the position of the head add on the head word itself fix the various issues with senseval pseudoxml or or fix lone fix fix s snumdd s snumdd fix foreign word tag remove i fix word remove p p remove m and t and ms remove doctype lines remove hi and p etc take the thing out of the brackets hellip and remove the for those patterns that aren t regular xml fix abc pfoo style tags now wf posfooabcwf natural language toolkit senseval 2 corpus reader c 2001 2023 nltk project trevor cohn tacohn cs mu oz au steven bird stevenbird1 gmail com modifications url https www nltk org for license information see license txt read from the senseval 2 corpus senseval http www senseval org evaluation exercises for word sense disambiguation organized by acl siglex https www siglex org prepared by ted pedersen tpederse umn edu university of minnesota https www d umn edu tpederse data html distributed with permission the nltk version of the senseval 2 files uses well formed xml each instance of the ambiguous words hard interest line and serve is tagged with a sense identifier and supplied with context list of streampos list of lexelt names decide which lexical element we re in start of a lexical element lexelt has no item start of an instance body of an instance end of an instance is this ok to do some santiy checks record the position of the head add on the head word itself sentence boundary marker fix the various issues with senseval pseudo xml or or fix lone fix fix s snum dd s snum dd fix foreign word tag remove i fix word remove p p remove m and t and ms remove doctype lines remove hi and p etc take the thing out of the brackets hellip and remove the for those patterns that aren t regular xml fix abc p foo style tags now wf pos foo abc wf
import re
from xml.etree import ElementTree

from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.tokenize import *


class SensevalInstance:
    def __init__(self, word, position, context, senses):
        self.word = word
        self.senses = tuple(senses)
        self.position = position
        self.context = context

    def __repr__(self):
        return "SensevalInstance(word=%r, position=%r, " "context=%r, senses=%r)" % (
            self.word,
            self.position,
            self.context,
            self.senses,
        )


class SensevalCorpusReader(CorpusReader):
    def instances(self, fileids=None):
        return concat(
            [
                SensevalCorpusView(fileid, enc)
                for (fileid, enc) in self.abspaths(fileids, True)
            ]
        )

    def _entry(self, tree):
        elts = []
        for lexelt in tree.findall("lexelt"):
            for inst in lexelt.findall("instance"):
                sense = inst[0].attrib["senseid"]
                context = [(w.text, w.attrib["pos"]) for w in inst[1]]
                elts.append((sense, context))
        return elts


class SensevalCorpusView(StreamBackedCorpusView):
    def __init__(self, fileid, encoding):
        StreamBackedCorpusView.__init__(self, fileid, encoding=encoding)

        self._word_tokenizer = WhitespaceTokenizer()
        self._lexelt_starts = [0]  
        self._lexelts = [None]  

    def read_block(self, stream):
        
        lexelt_num = bisect.bisect_right(self._lexelt_starts, stream.tell()) - 1
        lexelt = self._lexelts[lexelt_num]

        instance_lines = []
        in_instance = False
        while True:
            line = stream.readline()
            if line == "":
                assert instance_lines == []
                return []

            
            if line.lstrip().startswith("<lexelt"):
                lexelt_num += 1
                m = re.search("item=(\"[^\"]+\"|'[^']+')", line)
                assert m is not None  
                lexelt = m.group(1)[1:-1]
                if lexelt_num < len(self._lexelts):
                    assert lexelt == self._lexelts[lexelt_num]
                else:
                    self._lexelts.append(lexelt)
                    self._lexelt_starts.append(stream.tell())

            
            if line.lstrip().startswith("<instance"):
                assert instance_lines == []
                in_instance = True

            
            if in_instance:
                instance_lines.append(line)

            
            if line.lstrip().startswith("</instance"):
                xml_block = "\n".join(instance_lines)
                xml_block = _fixXML(xml_block)
                inst = ElementTree.fromstring(xml_block)
                return [self._parse_instance(inst, lexelt)]

    def _parse_instance(self, instance, lexelt):
        senses = []
        context = []
        position = None
        for child in instance:
            if child.tag == "answer":
                senses.append(child.attrib["senseid"])
            elif child.tag == "context":
                context += self._word_tokenizer.tokenize(child.text)
                for cword in child:
                    if cword.tag == "compound":
                        cword = cword[0]  

                    if cword.tag == "head":
                        
                        assert position is None, "head specified twice"
                        assert cword.text.strip() or len(cword) == 1
                        assert not (cword.text.strip() and len(cword) == 1)
                        
                        position = len(context)
                        
                        if cword.text.strip():
                            context.append(cword.text.strip())
                        elif cword[0].tag == "wf":
                            context.append((cword[0].text, cword[0].attrib["pos"]))
                            if cword[0].tail:
                                context += self._word_tokenizer.tokenize(cword[0].tail)
                        else:
                            assert False, "expected CDATA or wf in <head>"
                    elif cword.tag == "wf":
                        context.append((cword.text, cword.attrib["pos"]))
                    elif cword.tag == "s":
                        pass  

                    else:
                        print("ACK", cword.tag)
                        assert False, "expected CDATA or <wf> or <head>"
                    if cword.tail:
                        context += self._word_tokenizer.tokenize(cword.tail)
            else:
                assert False, "unexpected tag %s" % child.tag
        return SensevalInstance(lexelt, position, context, senses)


def _fixXML(text):
    
    
    text = re.sub(r"<([~\^])>", r"\1", text)
    
    text = re.sub(r"(\s+)\&(\s+)", r"\1&amp;\2", text)
    
    text = re.sub(r'"""', "'\"'", text)
    
    text = re.sub(r'(<[^<]*snum=)([^">]+)>', r'\1"\2"/>', text)
    
    text = re.sub(r"<\&frasl>\s*<p[^>]*>", "FRASL", text)
    
    text = re.sub(r"<\&I[^>]*>", "", text)
    
    text = re.sub(r"<{([^}]+)}>", r"\1", text)
    
    text = re.sub(r"<(@|/?p)>", r"", text)
    
    text = re.sub(r"<&\w+ \.>", r"", text)
    
    text = re.sub(r"<!DOCTYPE[^>]*>", r"", text)
    
    text = re.sub(r"<\[\/?[^>]+\]*>", r"", text)
    
    text = re.sub(r"<(\&\w+;)>", r"\1", text)
    
    text = re.sub(r"&(?!amp|gt|lt|apos|quot)", r"", text)
    
    text = re.sub(
        r'[ \t]*([^<>\s]+?)[ \t]*<p="([^"]*"?)"/>', r' <wf pos="\2">\1</wf>', text
    )
    text = re.sub(r'\s*"\s*<p=\'"\'/>', " <wf pos='\"'>\"</wf>", text)
    return text