natural language toolkit semcor corpus reader c 20012023 nltk project nathan schneider nschneidcs cmu edu url https www nltk org for license information see license txt corpus reader for the semcor corpus corpus reader for the semcor corpus for access to the complete xml data structure use the xml method for access to simple word lists and tagged word lists use words sents taggedwords and taggedsents return the given files as a list of words and punctuation symbols rtype liststr return the given files as a list of chunks each of which is a list of words and punctuation symbols that form a unit rtype listliststr return the given files as a list of tagged chunks represented in tree form rtype listtree param tag pos part of speech sem semantic or both to indicate the kind of tags to include semantic tags consist of wordnet lemma ids plus an ne node if the chunk is a named entity without a specific entry in wordnet named entities of type other have no lemma other chunks not in wordnet have no semantic tag punctuation tokens have none for their part of speech tag return the given files as a list of sentences each encoded as a list of word strings rtype listliststr return the given files as a list of sentences each encoded as a list of chunks rtype listlistliststr return the given files as a list of sentences each sentence is represented as a list of tagged chunks in tree form rtype listlisttree param tag pos part of speech sem semantic or both to indicate the kind of tags to include semantic tags consist of wordnet lemma ids plus an ne node if the chunk is a named entity without a specific entry in wordnet named entities of type other have no lemma other chunks not in wordnet have no semantic tag punctuation tokens have none for their part of speech tag the result of the semcorwordview may be a multiword unit so the lazyconcatenation will make sure the sentence is flattened helper used to implement the view methods returns a list of tokens segmented words chunks or sentences the tokens and chunks may optionally be tagged with pos and sense information param fileid the name of the underlying file param unit one of token word or chunk param bracketsent if true include sentence bracketing param postag whether to include partofspeech tags param semtag whether to include semantic tags namely wordnet lemma and oov named entity status does not exactly match the enclosed string e g due to typographical adjustments or discontinuity of a multiword expression if a redefinition has occurred the rdf attribute holds its inflected form and lemma holds its lemma for nes rdf lemma and pn all hold the same value the ne class cannot retrieve the wordnet lemma object possible reasons a the wordnet corpus is not downloaded b a nonexistent sense is annotated e g such s 00 triggers nltk corpus reader wordnet wordneterror no synset found for key u such5 00 01 specified 00 solution just use the lemma name as a string a list of words augmented by an attribute num used to record the sentence identifier the n attribute from the xml a stream backed corpus view specialized for use with the bnc corpus param fileid the name of the underlying file param unit one of token word or chunk param bracketsent if true include sentence bracketing param postag whether to include partofspeech tags param semtag whether to include semantic tags namely wordnet lemma and oov named entity status natural language toolkit semcor corpus reader c 2001 2023 nltk project nathan schneider nschneid cs cmu edu url https www nltk org for license information see license txt corpus reader for the semcor corpus corpus reader for the semcor corpus for access to the complete xml data structure use the xml method for access to simple word lists and tagged word lists use words sents tagged_words and tagged_sents return the given file s as a list of words and punctuation symbols rtype list str return the given file s as a list of chunks each of which is a list of words and punctuation symbols that form a unit rtype list list str return the given file s as a list of tagged chunks represented in tree form rtype list tree param tag pos part of speech sem semantic or both to indicate the kind of tags to include semantic tags consist of wordnet lemma ids plus an ne node if the chunk is a named entity without a specific entry in wordnet named entities of type other have no lemma other chunks not in wordnet have no semantic tag punctuation tokens have none for their part of speech tag return the given file s as a list of sentences each encoded as a list of word strings rtype list list str return the given file s as a list of sentences each encoded as a list of chunks rtype list list list str return the given file s as a list of sentences each sentence is represented as a list of tagged chunks in tree form rtype list list tree param tag pos part of speech sem semantic or both to indicate the kind of tags to include semantic tags consist of wordnet lemma ids plus an ne node if the chunk is a named entity without a specific entry in wordnet named entities of type other have no lemma other chunks not in wordnet have no semantic tag punctuation tokens have none for their part of speech tag the result of the semcorwordview may be a multiword unit so the lazyconcatenation will make sure the sentence is flattened helper used to implement the view methods returns a list of tokens segmented words chunks or sentences the tokens and chunks may optionally be tagged with pos and sense information param fileid the name of the underlying file param unit one of token word or chunk param bracket_sent if true include sentence bracketing param pos_tag whether to include part of speech tags param sem_tag whether to include semantic tags namely wordnet lemma and oov named entity status fixes issue 337 lemma or ne class lex_sense locator for the lemma s sense see http wordnet princeton edu man senseidx 5wn html redefinition this indicates the lookup string does not exactly match the enclosed string e g due to typographical adjustments or discontinuity of a multiword expression if a redefinition has occurred the rdf attribute holds its inflected form and lemma holds its lemma for nes rdf lemma and pn all hold the same value the ne class wordnet sense number a personal name ne not in wordnet part of speech for the whole chunk none for punctuation todo case where punctuation intervenes in mwe lemma object cannot retrieve the wordnet lemma object possible reasons a the wordnet corpus is not downloaded b a nonexistent sense is annotated e g such s 00 triggers nltk corpus reader wordnet wordneterror no synset found for key u such 5 00 01 specified 00 solution just use the lemma name as a string e g reach v 02 e g the sense number may be 2 1 other ne chunk as a list a list of words augmented by an attribute num used to record the sentence identifier the n attribute from the xml a stream backed corpus view specialized for use with the bnc corpus param fileid the name of the underlying file param unit one of token word or chunk param bracket_sent if true include sentence bracketing param pos_tag whether to include part of speech tags param sem_tag whether to include semantic tags namely wordnet lemma and oov named entity status
__docformat__ = "epytext en"

from nltk.corpus.reader.api import *
from nltk.corpus.reader.xmldocs import XMLCorpusReader, XMLCorpusView
from nltk.tree import Tree


class SemcorCorpusReader(XMLCorpusReader):
    

    def __init__(self, root, fileids, wordnet, lazy=True):
        XMLCorpusReader.__init__(self, root, fileids)
        self._lazy = lazy
        self._wordnet = wordnet

    def words(self, fileids=None):
        
        return self._items(fileids, "word", False, False, False)

    def chunks(self, fileids=None):
        
        return self._items(fileids, "chunk", False, False, False)

    def tagged_chunks(self, fileids=None, tag=("pos" or "sem" or "both")):
        
        return self._items(fileids, "chunk", False, tag != "sem", tag != "pos")

    def sents(self, fileids=None):
        
        return self._items(fileids, "word", True, False, False)

    def chunk_sents(self, fileids=None):
        
        return self._items(fileids, "chunk", True, False, False)

    def tagged_sents(self, fileids=None, tag=("pos" or "sem" or "both")):
        
        return self._items(fileids, "chunk", True, tag != "sem", tag != "pos")

    def _items(self, fileids, unit, bracket_sent, pos_tag, sem_tag):
        if unit == "word" and not bracket_sent:
            
            
            _ = lambda *args: LazyConcatenation(
                (SemcorWordView if self._lazy else self._words)(*args)
            )
        else:
            _ = SemcorWordView if self._lazy else self._words
        return concat(
            [
                _(fileid, unit, bracket_sent, pos_tag, sem_tag, self._wordnet)
                for fileid in self.abspaths(fileids)
            ]
        )

    def _words(self, fileid, unit, bracket_sent, pos_tag, sem_tag):
        
        assert unit in ("token", "word", "chunk")
        result = []

        xmldoc = ElementTree.parse(fileid).getroot()
        for xmlsent in xmldoc.findall(".//s"):
            sent = []
            for xmlword in _all_xmlwords_in(xmlsent):
                itm = SemcorCorpusReader._word(
                    xmlword, unit, pos_tag, sem_tag, self._wordnet
                )
                if unit == "word":
                    sent.extend(itm)
                else:
                    sent.append(itm)

            if bracket_sent:
                result.append(SemcorSentence(xmlsent.attrib["snum"], sent))
            else:
                result.extend(sent)

        assert None not in result
        return result

    @staticmethod
    def _word(xmlword, unit, pos_tag, sem_tag, wordnet):
        tkn = xmlword.text
        if not tkn:
            tkn = ""  

        lemma = xmlword.get("lemma", tkn)  
        lexsn = xmlword.get("lexsn")  
        if lexsn is not None:
            sense_key = lemma + "%" + lexsn
            wnpos = ("n", "v", "a", "r", "s")[
                int(lexsn.split(":")[0]) - 1
            ]  
        else:
            sense_key = wnpos = None
        redef = xmlword.get(
            "rdf", tkn
        )  
        
        
        
        
        sensenum = xmlword.get("wnsn")  
        isOOVEntity = "pn" in xmlword.keys()  
        pos = xmlword.get(
            "pos"
        )  

        if unit == "token":
            if not pos_tag and not sem_tag:
                itm = tkn
            else:
                itm = (
                    (tkn,)
                    + ((pos,) if pos_tag else ())
                    + ((lemma, wnpos, sensenum, isOOVEntity) if sem_tag else ())
                )
            return itm
        else:
            ww = tkn.split("_")  
            if unit == "word":
                return ww
            else:
                if sensenum is not None:
                    try:
                        sense = wordnet.lemma_from_key(sense_key)  
                    except Exception:
                        
                        
                        
                        
                        
                        try:
                            sense = "%s.%s.%02d" % (
                                lemma,
                                wnpos,
                                int(sensenum),
                            )  
                        except ValueError:
                            sense = (
                                lemma + "." + wnpos + "." + sensenum
                            )  

                bottom = [Tree(pos, ww)] if pos_tag else ww

                if sem_tag and isOOVEntity:
                    if sensenum is not None:
                        return Tree(sense, [Tree("NE", bottom)])
                    else:  
                        return Tree("NE", bottom)
                elif sem_tag and sensenum is not None:
                    return Tree(sense, bottom)
                elif pos_tag:
                    return bottom[0]
                else:
                    return bottom  


def _all_xmlwords_in(elt, result=None):
    if result is None:
        result = []
    for child in elt:
        if child.tag in ("wf", "punc"):
            result.append(child)
        else:
            _all_xmlwords_in(child, result)
    return result


class SemcorSentence(list):
    

    def __init__(self, num, items):
        self.num = num
        list.__init__(self, items)


class SemcorWordView(XMLCorpusView):
    

    def __init__(self, fileid, unit, bracket_sent, pos_tag, sem_tag, wordnet):
        
        if bracket_sent:
            tagspec = ".*/s"
        else:
            tagspec = ".*/s/(punc|wf)"

        self._unit = unit
        self._sent = bracket_sent
        self._pos_tag = pos_tag
        self._sem_tag = sem_tag
        self._wordnet = wordnet

        XMLCorpusView.__init__(self, fileid, tagspec)

    def handle_elt(self, elt, context):
        if self._sent:
            return self.handle_sent(elt)
        else:
            return self.handle_word(elt)

    def handle_word(self, elt):
        return SemcorCorpusReader._word(
            elt, self._unit, self._pos_tag, self._sem_tag, self._wordnet
        )

    def handle_sent(self, elt):
        sent = []
        for child in elt:
            if child.tag in ("wf", "punc"):
                itm = self.handle_word(child)
                if self._unit == "word":
                    sent.extend(itm)
                else:
                    sent.append(itm)
            else:
                raise ValueError("Unexpected element %s" % child.tag)
        return SemcorSentence(elt.attrib["snum"], sent)