natural language toolkit nkjp corpus reader c 20012023 nltk project gabriela kaczka url https www nltk org for license information see license txt wraps function arguments if fileids not specified then function set nkjpcorpusreader paths corpus reader designed to work with national corpus of polish see http nkjp pl for more details about nkjp use example import nltk import nkjp from nkjp import nkjpcorpusreader x nkjpcorpusreaderroot homeusernltkdatacorporankjp fileids obtain the whole corpus x header x raw x words x taggedwordstags subst comp link to find more tags nkjp plpoliqarphelpense2 html x sents x nkjpcorpusreaderroot homeusernltkdatacorporankjp fileids wilk obtain particular files x headerfileids wilkdom homeusernltkdatacorporankjpwilkwilczy x taggedwordsfileids wilkdom homeusernltkdatacorporankjpwilkwilczy tags subst comp returns a list of file identifiers for the fileids that make up this corpus returns a view specialised for use with particular corpus file add root if necessary to specified fileid returns headers of specified fileids returns sentences in specified fileids returns words in specified fileids call with specified tags as a list e g tags subst comp returns tagged words in specified fileids returns words in specified fileids headermode a stream backed corpus view specialized for use with header xml files in nkjp corpus helper class creating xml file to one without references to nkjp namespace that s needed because the xmlcorpusview assumes that one can find short substrings of xml that are valid xml which is not true if a namespace is declared at top level a stream backed corpus view specialized for use with annsegmentation xml files in nkjp corpus intersperse nkjpcorpustextview xml preprocessing base class init returns index of beginning letter in sentence returns index of end letter in sentence returns one sentence get increasing sequence of ids in case of choice get first possibility a stream backed corpus view specialized for use with text xml files in nkjp corpus xml preprocessing base class init returns text as a list of sentences fill dictionary to use later in sents mode a stream backed corpus view specialized for use with annmorphosyntax xml files in nkjp corpus if tags not specified then always return word get word natural language toolkit nkjp corpus reader c 2001 2023 nltk project gabriela kaczka url https www nltk org for license information see license txt wraps function arguments if fileids not specified then function set nkjpcorpusreader paths corpus reader designed to work with national corpus of polish see http nkjp pl for more details about nkjp use example import nltk import nkjp from nkjp import nkjpcorpusreader x nkjpcorpusreader root home user nltk_data corpora nkjp fileids obtain the whole corpus x header x raw x words x tagged_words tags subst comp link to find more tags nkjp pl poliqarp help ense2 html x sents x nkjpcorpusreader root home user nltk_data corpora nkjp fileids wilk obtain particular file s x header fileids wilkdom home user nltk_data corpora nkjp wilkwilczy x tagged_words fileids wilkdom home user nltk_data corpora nkjp wilkwilczy tags subst comp returns a list of file identifiers for the fileids that make up this corpus returns a view specialised for use with particular corpus file add root if necessary to specified fileid returns header s of specified fileids returns sentences in specified fileids returns words in specified fileids call with specified tags as a list e g tags subst comp returns tagged words in specified fileids returns words in specified fileids header_mode a stream backed corpus view specialized for use with header xml files in nkjp corpus helper class creating xml file to one without references to nkjp namespace that s needed because the xmlcorpusview assumes that one can find short substrings of xml that are valid xml which is not true if a namespace is declared at top level in all files in ann_segmentation xml in ann_segmentation xml in ann_segmentation xml in ann_segmentation xml a stream backed corpus view specialized for use with ann_segmentation xml files in nkjp corpus intersperse nkjpcorpus_text_view xml preprocessing base class init returns index of beginning letter in sentence returns index of end letter in sentence returns one sentence text segment get increasing sequence of ids in case of choice get first possibility a stream backed corpus view specialized for use with text xml files in nkjp corpus xml preprocessing base class init returns text as a list of sentences fill dictionary to use later in sents mode a stream backed corpus view specialized for use with ann_morphosyntax xml files in nkjp corpus if tags not specified then always return word get word
import functools
import os
import re
import tempfile

from nltk.corpus.reader.util import concat
from nltk.corpus.reader.xmldocs import XMLCorpusReader, XMLCorpusView


def _parse_args(fun):
    

    @functools.wraps(fun)
    def decorator(self, fileids=None, **kwargs):
        if not fileids:
            fileids = self._paths
        return fun(self, fileids, **kwargs)

    return decorator


class NKJPCorpusReader(XMLCorpusReader):
    WORDS_MODE = 0
    SENTS_MODE = 1
    HEADER_MODE = 2
    RAW_MODE = 3

    def __init__(self, root, fileids=".*"):
        
        if isinstance(fileids, str):
            XMLCorpusReader.__init__(self, root, fileids + ".*/header.xml")
        else:
            XMLCorpusReader.__init__(
                self, root, [fileid + "/header.xml" for fileid in fileids]
            )
        self._paths = self.get_paths()

    def get_paths(self):
        return [
            os.path.join(str(self._root), f.split("header.xml")[0])
            for f in self._fileids
        ]

    def fileids(self):
        
        return [f.split("header.xml")[0] for f in self._fileids]

    def _view(self, filename, tags=None, **kwargs):
        
        mode = kwargs.pop("mode", NKJPCorpusReader.WORDS_MODE)
        if mode is NKJPCorpusReader.WORDS_MODE:
            return NKJPCorpus_Morph_View(filename, tags=tags)
        elif mode is NKJPCorpusReader.SENTS_MODE:
            return NKJPCorpus_Segmentation_View(filename, tags=tags)
        elif mode is NKJPCorpusReader.HEADER_MODE:
            return NKJPCorpus_Header_View(filename, tags=tags)
        elif mode is NKJPCorpusReader.RAW_MODE:
            return NKJPCorpus_Text_View(
                filename, tags=tags, mode=NKJPCorpus_Text_View.RAW_MODE
            )

        else:
            raise NameError("No such mode!")

    def add_root(self, fileid):
        
        if self.root in fileid:
            return fileid
        return self.root + fileid

    @_parse_args
    def header(self, fileids=None, **kwargs):
        
        return concat(
            [
                self._view(
                    self.add_root(fileid), mode=NKJPCorpusReader.HEADER_MODE, **kwargs
                ).handle_query()
                for fileid in fileids
            ]
        )

    @_parse_args
    def sents(self, fileids=None, **kwargs):
        
        return concat(
            [
                self._view(
                    self.add_root(fileid), mode=NKJPCorpusReader.SENTS_MODE, **kwargs
                ).handle_query()
                for fileid in fileids
            ]
        )

    @_parse_args
    def words(self, fileids=None, **kwargs):
        

        return concat(
            [
                self._view(
                    self.add_root(fileid), mode=NKJPCorpusReader.WORDS_MODE, **kwargs
                ).handle_query()
                for fileid in fileids
            ]
        )

    @_parse_args
    def tagged_words(self, fileids=None, **kwargs):
        
        tags = kwargs.pop("tags", [])
        return concat(
            [
                self._view(
                    self.add_root(fileid),
                    mode=NKJPCorpusReader.WORDS_MODE,
                    tags=tags,
                    **kwargs
                ).handle_query()
                for fileid in fileids
            ]
        )

    @_parse_args
    def raw(self, fileids=None, **kwargs):
        
        return concat(
            [
                self._view(
                    self.add_root(fileid), mode=NKJPCorpusReader.RAW_MODE, **kwargs
                ).handle_query()
                for fileid in fileids
            ]
        )


class NKJPCorpus_Header_View(XMLCorpusView):
    def __init__(self, filename, **kwargs):
        
        self.tagspec = ".*/sourceDesc$"
        XMLCorpusView.__init__(self, filename + "header.xml", self.tagspec)

    def handle_query(self):
        self._open()
        header = []
        while True:
            segm = XMLCorpusView.read_block(self, self._stream)
            if len(segm) == 0:
                break
            header.extend(segm)
        self.close()
        return header

    def handle_elt(self, elt, context):
        titles = elt.findall("bibl/title")
        title = []
        if titles:
            title = "\n".join(title.text.strip() for title in titles)

        authors = elt.findall("bibl/author")
        author = []
        if authors:
            author = "\n".join(author.text.strip() for author in authors)

        dates = elt.findall("bibl/date")
        date = []
        if dates:
            date = "\n".join(date.text.strip() for date in dates)

        publishers = elt.findall("bibl/publisher")
        publisher = []
        if publishers:
            publisher = "\n".join(publisher.text.strip() for publisher in publishers)

        idnos = elt.findall("bibl/idno")
        idno = []
        if idnos:
            idno = "\n".join(idno.text.strip() for idno in idnos)

        notes = elt.findall("bibl/note")
        note = []
        if notes:
            note = "\n".join(note.text.strip() for note in notes)

        return {
            "title": title,
            "author": author,
            "date": date,
            "publisher": publisher,
            "idno": idno,
            "note": note,
        }


class XML_Tool:
    

    def __init__(self, root, filename):
        self.read_file = os.path.join(root, filename)
        self.write_file = tempfile.NamedTemporaryFile(delete=False)

    def build_preprocessed_file(self):
        try:
            fr = open(self.read_file)
            fw = self.write_file
            line = " "
            while len(line):
                line = fr.readline()
                x = re.split(r"nkjp:[^ ]* ", line)  
                ret = " ".join(x)
                x = re.split("<nkjp:paren>", ret)  
                ret = " ".join(x)
                x = re.split("</nkjp:paren>", ret)  
                ret = " ".join(x)
                x = re.split("<choice>", ret)  
                ret = " ".join(x)
                x = re.split("</choice>", ret)  
                ret = " ".join(x)
                fw.write(ret)
            fr.close()
            fw.close()
            return self.write_file.name
        except Exception as e:
            self.remove_preprocessed_file()
            raise Exception from e

    def remove_preprocessed_file(self):
        os.remove(self.write_file.name)


class NKJPCorpus_Segmentation_View(XMLCorpusView):
    

    def __init__(self, filename, **kwargs):
        self.tagspec = ".*p/.*s"
        
        self.text_view = NKJPCorpus_Text_View(
            filename, mode=NKJPCorpus_Text_View.SENTS_MODE
        )
        self.text_view.handle_query()
        
        self.xml_tool = XML_Tool(filename, "ann_segmentation.xml")
        
        XMLCorpusView.__init__(
            self, self.xml_tool.build_preprocessed_file(), self.tagspec
        )

    def get_segm_id(self, example_word):
        return example_word.split("(")[1].split(",")[0]

    def get_sent_beg(self, beg_word):
        
        return int(beg_word.split(",")[1])

    def get_sent_end(self, end_word):
        
        splitted = end_word.split(")")[0].split(",")
        return int(splitted[1]) + int(splitted[2])

    def get_sentences(self, sent_segm):
        
        id = self.get_segm_id(sent_segm[0])
        segm = self.text_view.segm_dict[id]  
        beg = self.get_sent_beg(sent_segm[0])
        end = self.get_sent_end(sent_segm[len(sent_segm) - 1])
        return segm[beg:end]

    def remove_choice(self, segm):
        ret = []
        prev_txt_end = -1
        prev_txt_nr = -1
        for word in segm:
            txt_nr = self.get_segm_id(word)
            
            if self.get_sent_beg(word) > prev_txt_end - 1 or prev_txt_nr != txt_nr:
                ret.append(word)
                prev_txt_end = self.get_sent_end(word)
            prev_txt_nr = txt_nr

        return ret

    def handle_query(self):
        try:
            self._open()
            sentences = []
            while True:
                sent_segm = XMLCorpusView.read_block(self, self._stream)
                if len(sent_segm) == 0:
                    break
                for segm in sent_segm:
                    segm = self.remove_choice(segm)
                    sentences.append(self.get_sentences(segm))
            self.close()
            self.xml_tool.remove_preprocessed_file()
            return sentences
        except Exception as e:
            self.xml_tool.remove_preprocessed_file()
            raise Exception from e

    def handle_elt(self, elt, context):
        ret = []
        for seg in elt:
            ret.append(seg.get("corresp"))
        return ret


class NKJPCorpus_Text_View(XMLCorpusView):
    

    SENTS_MODE = 0
    RAW_MODE = 1

    def __init__(self, filename, **kwargs):
        self.mode = kwargs.pop("mode", 0)
        self.tagspec = ".*/div/ab"
        self.segm_dict = dict()
        
        self.xml_tool = XML_Tool(filename, "text.xml")
        
        XMLCorpusView.__init__(
            self, self.xml_tool.build_preprocessed_file(), self.tagspec
        )

    def handle_query(self):
        try:
            self._open()
            x = self.read_block(self._stream)
            self.close()
            self.xml_tool.remove_preprocessed_file()
            return x
        except Exception as e:
            self.xml_tool.remove_preprocessed_file()
            raise Exception from e

    def read_block(self, stream, tagspec=None, elt_handler=None):
        
        txt = []
        while True:
            segm = XMLCorpusView.read_block(self, stream)
            if len(segm) == 0:
                break
            for part in segm:
                txt.append(part)

        return [" ".join([segm for segm in txt])]

    def get_segm_id(self, elt):
        for attr in elt.attrib:
            if attr.endswith("id"):
                return elt.get(attr)

    def handle_elt(self, elt, context):
        
        if self.mode is NKJPCorpus_Text_View.SENTS_MODE:
            self.segm_dict[self.get_segm_id(elt)] = elt.text
        return elt.text


class NKJPCorpus_Morph_View(XMLCorpusView):
    

    def __init__(self, filename, **kwargs):
        self.tags = kwargs.pop("tags", None)
        self.tagspec = ".*/seg/fs"
        self.xml_tool = XML_Tool(filename, "ann_morphosyntax.xml")
        XMLCorpusView.__init__(
            self, self.xml_tool.build_preprocessed_file(), self.tagspec
        )

    def handle_query(self):
        try:
            self._open()
            words = []
            while True:
                segm = XMLCorpusView.read_block(self, self._stream)
                if len(segm) == 0:
                    break
                for part in segm:
                    if part is not None:
                        words.append(part)
            self.close()
            self.xml_tool.remove_preprocessed_file()
            return words
        except Exception as e:
            self.xml_tool.remove_preprocessed_file()
            raise Exception from e

    def handle_elt(self, elt, context):
        word = ""
        flag = False
        is_not_interp = True
        
        if self.tags is None:
            flag = True

        for child in elt:

            
            if "name" in child.keys() and child.attrib["name"] == "orth":
                for symbol in child:
                    if symbol.tag == "string":
                        word = symbol.text
            elif "name" in child.keys() and child.attrib["name"] == "interps":
                for symbol in child:
                    if "type" in symbol.keys() and symbol.attrib["type"] == "lex":
                        for symbol2 in symbol:
                            if (
                                "name" in symbol2.keys()
                                and symbol2.attrib["name"] == "ctag"
                            ):
                                for symbol3 in symbol2:
                                    if (
                                        "value" in symbol3.keys()
                                        and self.tags is not None
                                        and symbol3.attrib["value"] in self.tags
                                    ):
                                        flag = True
                                    elif (
                                        "value" in symbol3.keys()
                                        and symbol3.attrib["value"] == "interp"
                                    ):
                                        is_not_interp = False
        if flag and is_not_interp:
            return word