natural language toolkit plaintext corpus reader c 20012023 nltk project edward loper edlopergmail com url https www nltk org for license information see license txt corpus reader for the xml version of the british national corpus from nltk corpus reader util import concat from nltk corpus reader xmldocs import elementtree xmlcorpusreader xmlcorpusview class bnccorpusreaderxmlcorpusreader rcorpus reader for the xml version of the british national corpus for access to the complete xml data structure use the xml method for access to simple word lists and tagged word lists use words sents taggedwords and taggedsents you can obtain the full version of the bnc corpus at https www ota ox ac ukdesc2554 if you extracted the archive to a directory called bnc then you can instantiate the reader as bnccorpusreaderroot bnctexts fileidsr akww xml return the given files as a list of words and punctuation symbols rtype liststr param stripspace if true then strip trailing spaces from word tokens otherwise leave the spaces on the tokens param stem if true then use word stems instead of word strings return the given files as a list of tagged words and punctuation symbols encoded as tuples word tag rtype listtuplestr str param c5 if true then the tags used will be the more detailed c5 tags otherwise the simplified tags will be used param stripspace if true then strip trailing spaces from word tokens otherwise leave the spaces on the tokens param stem if true then use word stems instead of word strings return the given files as a list of sentences or utterances each encoded as a list of word strings rtype listliststr param stripspace if true then strip trailing spaces from word tokens otherwise leave the spaces on the tokens param stem if true then use word stems instead of word strings return the given files as a list of sentences each encoded as a list of word tag tuples rtype listlisttuplestr str param c5 if true then the tags used will be the more detailed c5 tags otherwise the simplified tags will be used param stripspace if true then strip trailing spaces from word tokens otherwise leave the spaces on the tokens param stem if true then use word stems instead of word strings a helper function that instantiates bncwordviews or the list of wordssentences f bncwordview if self lazy else self words return concat ffileid sent tag stripspace stem for fileid in self abspathsfileids def wordsself fileid bracketsent tag stripspace stem result xmldoc elementtree parsefileid getroot for xmlsent in xmldoc findall s sent for xmlword in allxmlwordsinxmlsent word xmlword text if not word word fixes issue 337 if stripspace or stem word word strip if stem word xmlword gethw word if tag c5 word word xmlword getc5 elif tag pos word word xmlword getpos xmlword getc5 sent appendword if bracketsent result appendbncsentencexmlsent attribn sent else result extendsent assert none not in result return result def allxmlwordsinelt resultnone if result is none result for child in elt if child tag in c w result appendchild else allxmlwordsinchild result return result class bncsentencelist def initself num items self num num list initself items class bncwordviewxmlcorpusview tagstoignore pb gap vocal event unclear shift pause align def initself fileid sent tag stripspace stem if sent tagspec s else tagspec s cw self sent sent self tag tag self stripspace stripspace self stem stem self title none title of the document self none of the document self editor none editor self resps none statement of responsibility xmlcorpusview initself fileid tagspec read in a tasty header self open self readblockself stream teiheader self handleheader self close reset tag context self tagcontext 0 def handleheaderself elt context set up some metadata titles elt findalltitlestmttitle if titles self title n jointitle text strip for title in titles s elt findalltitlestmt if s self n join text strip for in s editors elt findalltitlestmteditor if editors self editor n joineditor text strip for editor in editors resps elt findalltitlestmtrespstmt if resps self resps nn join n joinrespelt text strip for respelt in resp for resp in resps def handleeltself elt context if self sent return self handlesentelt else return self handlewordelt def handlewordself elt word elt text if not word word fixes issue 337 if self stripspace or self stem word word strip if self stem word elt gethw word if self tag c5 word word elt getc5 elif self tag pos word word elt getpos elt getc5 return word def handlesentself elt sent for child in elt if child tag in mw hi corr trunc sent self handlewordw for w in child elif child tag in w c sent appendself handlewordchild elif child tag not in self tagstoignore raise valueerrorunexpected element s child tag return bncsentenceelt attribn sent natural language toolkit plaintext corpus reader c 2001 2023 nltk project edward loper edloper gmail com url https www nltk org for license information see license txt corpus reader for the xml version of the british national corpus corpus reader for the xml version of the british national corpus for access to the complete xml data structure use the xml method for access to simple word lists and tagged word lists use words sents tagged_words and tagged_sents you can obtain the full version of the bnc corpus at https www ota ox ac uk desc 2554 if you extracted the archive to a directory called bnc then you can instantiate the reader as bnccorpusreader root bnc texts fileids r a k w w xml return the given file s as a list of words and punctuation symbols rtype list str param strip_space if true then strip trailing spaces from word tokens otherwise leave the spaces on the tokens param stem if true then use word stems instead of word strings return the given file s as a list of tagged words and punctuation symbols encoded as tuples word tag rtype list tuple str str param c5 if true then the tags used will be the more detailed c5 tags otherwise the simplified tags will be used param strip_space if true then strip trailing spaces from word tokens otherwise leave the spaces on the tokens param stem if true then use word stems instead of word strings return the given file s as a list of sentences or utterances each encoded as a list of word strings rtype list list str param strip_space if true then strip trailing spaces from word tokens otherwise leave the spaces on the tokens param stem if true then use word stems instead of word strings return the given file s as a list of sentences each encoded as a list of word tag tuples rtype list list tuple str str param c5 if true then the tags used will be the more detailed c5 tags otherwise the simplified tags will be used param strip_space if true then strip trailing spaces from word tokens otherwise leave the spaces on the tokens param stem if true then use word stems instead of word strings a helper function that instantiates bncwordviews or the list of words sentences helper used to implement the view methods returns a list of words or a list of sentences optionally tagged param fileid the name of the underlying file param bracket_sent if true include sentence bracketing param tag the name of the tagset to use or none for no tags param strip_space if true strip spaces from word tokens param stem if true then substitute stems for words fixes issue 337 a list of words augmented by an attribute num used to record the sentence identifier the n attribute from the xml a stream backed corpus view specialized for use with the bnc corpus these tags are ignored for their description refer to the technical documentation for example http www natcorp ox ac uk docs urg ref vocal html param fileid the name of the underlying file param sent if true include sentence bracketing param tag the name of the tagset to use or none for no tags param strip_space if true strip spaces from word tokens param stem if true then substitute stems for words title of the document of the document editor statement of responsibility read in a tasty header reset tag context set up some metadata fixes issue 337
from nltk.corpus.reader.util import concat
from nltk.corpus.reader.xmldocs import ElementTree, XMLCorpusReader, XMLCorpusView


class BNCCorpusReader(XMLCorpusReader):
    r

    def __init__(self, root, fileids, lazy=True):
        XMLCorpusReader.__init__(self, root, fileids)
        self._lazy = lazy

    def words(self, fileids=None, strip_space=True, stem=False):
        
        return self._views(fileids, False, None, strip_space, stem)

    def tagged_words(self, fileids=None, c5=False, strip_space=True, stem=False):
        
        tag = "c5" if c5 else "pos"
        return self._views(fileids, False, tag, strip_space, stem)

    def sents(self, fileids=None, strip_space=True, stem=False):
        
        return self._views(fileids, True, None, strip_space, stem)

    def tagged_sents(self, fileids=None, c5=False, strip_space=True, stem=False):
        
        tag = "c5" if c5 else "pos"
        return self._views(
            fileids, sent=True, tag=tag, strip_space=strip_space, stem=stem
        )

    def _views(self, fileids=None, sent=False, tag=False, strip_space=True, stem=False):
        
        f = BNCWordView if self._lazy else self._words
        return concat(
            [
                f(fileid, sent, tag, strip_space, stem)
                for fileid in self.abspaths(fileids)
            ]
        )

    def _words(self, fileid, bracket_sent, tag, strip_space, stem):
        
        result = []

        xmldoc = ElementTree.parse(fileid).getroot()
        for xmlsent in xmldoc.findall(".//s"):
            sent = []
            for xmlword in _all_xmlwords_in(xmlsent):
                word = xmlword.text
                if not word:
                    word = ""  
                if strip_space or stem:
                    word = word.strip()
                if stem:
                    word = xmlword.get("hw", word)
                if tag == "c5":
                    word = (word, xmlword.get("c5"))
                elif tag == "pos":
                    word = (word, xmlword.get("pos", xmlword.get("c5")))
                sent.append(word)
            if bracket_sent:
                result.append(BNCSentence(xmlsent.attrib["n"], sent))
            else:
                result.extend(sent)

        assert None not in result
        return result


def _all_xmlwords_in(elt, result=None):
    if result is None:
        result = []
    for child in elt:
        if child.tag in ("c", "w"):
            result.append(child)
        else:
            _all_xmlwords_in(child, result)
    return result


class BNCSentence(list):
    

    def __init__(self, num, items):
        self.num = num
        list.__init__(self, items)


class BNCWordView(XMLCorpusView):
    

    tags_to_ignore = {
        "pb",
        "gap",
        "vocal",
        "event",
        "unclear",
        "shift",
        "pause",
        "align",
    }
    

    def __init__(self, fileid, sent, tag, strip_space, stem):
        
        if sent:
            tagspec = ".*/s"
        else:
            tagspec = ".*/s/(.*/)?(c|w)"
        self._sent = sent
        self._tag = tag
        self._strip_space = strip_space
        self._stem = stem

        self.title = None  
        self.author = None  
        self.editor = None  
        self.resps = None  

        XMLCorpusView.__init__(self, fileid, tagspec)

        
        self._open()
        self.read_block(self._stream, ".*/teiHeader$", self.handle_header)
        self.close()

        
        self._tag_context = {0: ()}

    def handle_header(self, elt, context):
        
        titles = elt.findall("titleStmt/title")
        if titles:
            self.title = "\n".join(title.text.strip() for title in titles)

        authors = elt.findall("titleStmt/author")
        if authors:
            self.author = "\n".join(author.text.strip() for author in authors)

        editors = elt.findall("titleStmt/editor")
        if editors:
            self.editor = "\n".join(editor.text.strip() for editor in editors)

        resps = elt.findall("titleStmt/respStmt")
        if resps:
            self.resps = "\n\n".join(
                "\n".join(resp_elt.text.strip() for resp_elt in resp) for resp in resps
            )

    def handle_elt(self, elt, context):
        if self._sent:
            return self.handle_sent(elt)
        else:
            return self.handle_word(elt)

    def handle_word(self, elt):
        word = elt.text
        if not word:
            word = ""  
        if self._strip_space or self._stem:
            word = word.strip()
        if self._stem:
            word = elt.get("hw", word)
        if self._tag == "c5":
            word = (word, elt.get("c5"))
        elif self._tag == "pos":
            word = (word, elt.get("pos", elt.get("c5")))
        return word

    def handle_sent(self, elt):
        sent = []
        for child in elt:
            if child.tag in ("mw", "hi", "corr", "trunc"):
                sent += [self.handle_word(w) for w in child]
            elif child.tag in ("w", "c"):
                sent.append(self.handle_word(child))
            elif child.tag not in self.tags_to_ignore:
                raise ValueError("Unexpected element %s" % child.tag)
        return BNCSentence(elt.attrib["n"], sent)