natural language toolkit toolbox reader c 20012023 nltk project greg aumann gregaumannsil org url https www nltk org for license information see license txt module for reading writing and manipulating toolbox databases and settings files class for reading and processing standard format marker files and strings open a standard format marker file for sequential reading param sfmfile name of the standard format marker input file type sfmfile str open a standard format marker string for sequential reading param s string to parse as a standard format marker input file type s str return an iterator that returns the next field in a marker value tuple linebreaks and trailing white space are preserved except for the final newline in each field rtype itertuplestr str discard a bom in the first line need to get first line outside the loop for correct handling of the first marker if it spans multiple lines pep 479 prevent runtimeerror when stopiteration is raised inside generator no more data is available terminate the generator return an iterator that returns the next field in a marker value tuple where marker and value are unicode strings if an encoding was specified in the fields method otherwise they are nonunicode strings param strip strip trailing whitespace from the last line of each field type strip bool param unwrap convert newlines in a field to spaces type unwrap bool param encoding name of an encoding to use if it is specified then the fields method returns unicode strings rather than non unicode strings type encoding str or none param errors error handling scheme for codec same as the decode builtin string method type errors str param unicodefields set of marker names whose values are utf8 encoded ignored if encoding is none if the whole file is utf8 encoded set encoding utf8 and leave unicodefields with its default value of none type unicodefields sequence rtype itertuplestr str close a previously opened standard format marker file or string self file close try del self linenum except attributeerror pass class toolboxdatastandardformat def parseself grammarnone kwargs if grammar return self chunkparsegrammargrammar kwargs else return self recordparsekwargs def recordparseself keynone kwargs r returns an element tree structure corresponding to a toolbox data file with all markers at the same level thus the following toolbox database sh v3 0 400 rotokas dictionary datestamphasfourdigityear lx kaa ps v a ge gag gp nek i pas lx kaa ps v b ge strangle gp pasim nek after parsing will end up with the same structure ignoring the extra whitespace as the following xml fragment after being parsed by elementtree toolboxdata header shv3 0 400 rotokas dictionarysh datestamphasfourdigityear header record lxkaalx psv aps gegagge gpnek i pasgp record record lxkaalx psv bps gestranglege gppasim nekgp record toolboxdata param key name of key marker at the start of each record if set to none the default value the first marker that doesn t begin with an underscore is assumed to be the key type key str param kwargs keyword arguments passed to standardformat fields type kwargs dict rtype elementtree elementinterface return contents of toolbox data divided into header and records returns an element tree structure corresponding to a toolbox data file parsed according to the chunk grammar type grammar str param grammar contains the chunking rules used to parse the database see chunk regexp for documentation type rootlabel str param rootlabel the node value that should be used for the top node of the chunk structure type trace int param trace the level of tracing that should be used when parsing a text 0 will generate no tracing output 1 will generate normal tracing output and 2 or higher will generate verbose tracing output type kwargs dict param kwargs keyword arguments passed to toolbox standardformat fields rtype elementtree elementinterface return a string with a standard format representation of the toolbox data in tree tree can be a toolbox database or a single record param tree flat representation of toolbox data whole database or single record type tree elementtree elementinterface param encoding name of an encoding to use type encoding str param errors error handling scheme for codec same as the encode builtin string method type errors str param unicodefields type unicodefields dictstr or setstr rtype str this class is the base class for settings files def initself super init def parseself encodingnone errorsstrict kwargs builder treebuilder for mkr value in self fieldsencodingencoding errorserrors kwargs check whether the first char of the field marker indicates a block start or end block mkr0 if block in mkr mkr1 else block none build tree on the basis of block char if block builder startmkr builder datavalue elif block builder endmkr else builder startmkr builder datavalue builder endmkr return builder close def tosettingsstringtree encodingnone errorsstrict unicodefieldsnone write xml to file l list tosettingsstring tree getroot l encodingencoding errorserrors unicodefieldsunicodefields return joinl def tosettingsstringnode l kwargs write xml to file tag node tag text node text if lennode 0 if text l appendftag textn else l appendsn tag else if text l appendftag textn else l appendsn tag for n in node tosettingsstringn l kwargs l appendsn tag return def removeblankselem out list for child in elem removeblankschild if child text or lenchild 0 out appendchild elem out def adddefaultfieldselem defaultfields for field in defaultfields getelem tag if elem findfield is none subelementelem field for child in elem adddefaultfieldschild defaultfields def sortfieldselem fieldorders orderdicts dict for field order in fieldorders items orderdictsfield orderkey dict for i subfield in enumerateorder orderkeysubfield i sortfieldselem orderdicts def sortfieldselem ordersdicts add blank lines before all elements and subelements specified in blankbefore param elem toolbox data in an elementtree structure type elem elementtree elementinterface param blankbefore elements and subelements to add blank lines before type blankbefore dicttuple zippath find corporatoolbox zip lexicon toolboxdatazipfilepathpointerzippath toolboxrotokas dic parse settings openzipfilepathpointerzippath entry toolboxmdfmdfalth typ natural language toolkit toolbox reader c 2001 2023 nltk project greg aumann greg_aumann sil org url https www nltk org for license information see license txt module for reading writing and manipulating toolbox databases and settings files class for reading and processing standard format marker files and strings open a standard format marker file for sequential reading param sfm_file name of the standard format marker input file type sfm_file str open a standard format marker string for sequential reading param s string to parse as a standard format marker input file type s str return an iterator that returns the next field in a marker value tuple linebreaks and trailing white space are preserved except for the final newline in each field rtype iter tuple str str discard a bom in the first line need to get first line outside the loop for correct handling of the first marker if it spans multiple lines pep 479 prevent runtimeerror when stopiteration is raised inside generator no more data is available terminate the generator return an iterator that returns the next field in a marker value tuple where marker and value are unicode strings if an encoding was specified in the fields method otherwise they are non unicode strings param strip strip trailing whitespace from the last line of each field type strip bool param unwrap convert newlines in a field to spaces type unwrap bool param encoding name of an encoding to use if it is specified then the fields method returns unicode strings rather than non unicode strings type encoding str or none param errors error handling scheme for codec same as the decode builtin string method type errors str param unicode_fields set of marker names whose values are utf 8 encoded ignored if encoding is none if the whole file is utf 8 encoded set encoding utf8 and leave unicode_fields with its default value of none type unicode_fields sequence rtype iter tuple str str close a previously opened standard format marker file or string returns an element tree structure corresponding to a toolbox data file with all markers at the same level thus the following toolbox database _sh v3 0 400 rotokas dictionary _datestamphasfourdigityear lx kaa ps v a ge gag gp nek i pas lx kaa ps v b ge strangle gp pasim nek after parsing will end up with the same structure ignoring the extra whitespace as the following xml fragment after being parsed by elementtree toolbox_data header _sh v3 0 400 rotokas dictionary _sh _datestamphasfourdigityear header record lx kaa lx ps v a ps ge gag ge gp nek i pas gp record record lx kaa lx ps v b ps ge strangle ge gp pasim nek gp record toolbox_data param key name of key marker at the start of each record if set to none the default value the first marker that doesn t begin with an underscore is assumed to be the key type key str param kwargs keyword arguments passed to standardformat fields type kwargs dict rtype elementtree _elementinterface return contents of toolbox data divided into header and records returns an element tree structure corresponding to a toolbox data file parsed according to the chunk grammar type grammar str param grammar contains the chunking rules used to parse the database see chunk regexp for documentation type root_label str param root_label the node value that should be used for the top node of the chunk structure type trace int param trace the level of tracing that should be used when parsing a text 0 will generate no tracing output 1 will generate normal tracing output and 2 or higher will generate verbose tracing output type kwargs dict param kwargs keyword arguments passed to toolbox standardformat fields rtype elementtree _elementinterface return a string with a standard format representation of the toolbox data in tree tree can be a toolbox database or a single record param tree flat representation of toolbox data whole database or single record type tree elementtree _elementinterface param encoding name of an encoding to use type encoding str param errors error handling scheme for codec same as the encode builtin string method type errors str param unicode_fields type unicode_fields dict str or set str rtype str this class is the base class for settings files return the contents of toolbox settings file with a nested structure param encoding encoding used by settings file type encoding str param errors error handling scheme for codec same as decode builtin method type errors str param kwargs keyword arguments passed to standardformat fields type kwargs dict rtype elementtree _elementinterface check whether the first char of the field marker indicates a block start or end build tree on the basis of block char write xml to file write xml to file remove all elements and subelements with no text and no child elements param elem toolbox data in an elementtree structure type elem elementtree _elementinterface add blank elements and subelements specified in default_fields param elem toolbox data in an elementtree structure type elem elementtree _elementinterface param default_fields fields to add to each type of element and subelement type default_fields dict tuple sort the elements and subelements in order specified in field_orders param elem toolbox data in an elementtree structure type elem elementtree _elementinterface param field_orders order of fields for each type of element and subelement type field_orders dict tuple sort the children of elem add blank lines before all elements and subelements specified in blank_before param elem toolbox data in an elementtree structure type elem elementtree _elementinterface param blank_before elements and subelements to add blank lines before type blank_before dict tuple zip_path find corpora toolbox zip lexicon toolboxdata zipfilepathpointer zip_path toolbox rotokas dic parse settings open zipfilepathpointer zip_path entry toolbox mdf mdf_alth typ
import codecs
import re
from io import StringIO
from xml.etree.ElementTree import Element, ElementTree, SubElement, TreeBuilder

from nltk.data import PathPointer, find


class StandardFormat:
    

    def __init__(self, filename=None, encoding=None):
        self._encoding = encoding
        if filename is not None:
            self.open(filename)

    def open(self, sfm_file):
        
        if isinstance(sfm_file, PathPointer):
            self._file = sfm_file.open(self._encoding)
        else:
            self._file = codecs.open(sfm_file, "r", self._encoding)

    def open_string(self, s):
        
        self._file = StringIO(s)

    def raw_fields(self):
        
        join_string = "\n"
        line_regexp = r"^%s(?:\\(\S+)\s*)?(.*)$"
        
        first_line_pat = re.compile(line_regexp % "(?:\xef\xbb\xbf)?")
        line_pat = re.compile(line_regexp % "")
        
        
        file_iter = iter(self._file)
        
        try:
            line = next(file_iter)
        except StopIteration:
            
            return
        mobj = re.match(first_line_pat, line)
        mkr, line_value = mobj.groups()
        value_lines = [line_value]
        self.line_num = 0
        for line in file_iter:
            self.line_num += 1
            mobj = re.match(line_pat, line)
            line_mkr, line_value = mobj.groups()
            if line_mkr:
                yield (mkr, join_string.join(value_lines))
                mkr = line_mkr
                value_lines = [line_value]
            else:
                value_lines.append(line_value)
        self.line_num += 1
        yield (mkr, join_string.join(value_lines))

    def fields(
        self,
        strip=True,
        unwrap=True,
        encoding=None,
        errors="strict",
        unicode_fields=None,
    ):
        
        if encoding is None and unicode_fields is not None:
            raise ValueError("unicode_fields is set but not encoding.")
        unwrap_pat = re.compile(r"\n+")
        for mkr, val in self.raw_fields():
            if unwrap:
                val = unwrap_pat.sub(" ", val)
            if strip:
                val = val.rstrip()
            yield (mkr, val)

    def close(self):
        
        self._file.close()
        try:
            del self.line_num
        except AttributeError:
            pass


class ToolboxData(StandardFormat):
    def parse(self, grammar=None, **kwargs):
        if grammar:
            return self._chunk_parse(grammar=grammar, **kwargs)
        else:
            return self._record_parse(**kwargs)

    def _record_parse(self, key=None, **kwargs):
        r
        builder = TreeBuilder()
        builder.start("toolbox_data", {})
        builder.start("header", {})
        in_records = False
        for mkr, value in self.fields(**kwargs):
            if key is None and not in_records and mkr[0] != "_":
                key = mkr
            if mkr == key:
                if in_records:
                    builder.end("record")
                else:
                    builder.end("header")
                    in_records = True
                builder.start("record", {})
            builder.start(mkr, {})
            builder.data(value)
            builder.end(mkr)
        if in_records:
            builder.end("record")
        else:
            builder.end("header")
        builder.end("toolbox_data")
        return builder.close()

    def _tree2etree(self, parent):
        from nltk.tree import Tree

        root = Element(parent.label())
        for child in parent:
            if isinstance(child, Tree):
                root.append(self._tree2etree(child))
            else:
                text, tag = child
                e = SubElement(root, tag)
                e.text = text
        return root

    def _chunk_parse(self, grammar=None, root_label="record", trace=0, **kwargs):
        
        from nltk import chunk
        from nltk.tree import Tree

        cp = chunk.RegexpParser(grammar, root_label=root_label, trace=trace)
        db = self.parse(**kwargs)
        tb_etree = Element("toolbox_data")
        header = db.find("header")
        tb_etree.append(header)
        for record in db.findall("record"):
            parsed = cp.parse([(elem.text, elem.tag) for elem in record])
            tb_etree.append(self._tree2etree(parsed))
        return tb_etree


_is_value = re.compile(r"\S")


def to_sfm_string(tree, encoding=None, errors="strict", unicode_fields=None):
    
    if tree.tag == "record":
        root = Element("toolbox_data")
        root.append(tree)
        tree = root

    if tree.tag != "toolbox_data":
        raise ValueError("not a toolbox_data element structure")
    if encoding is None and unicode_fields is not None:
        raise ValueError(
            "if encoding is not specified then neither should unicode_fields"
        )
    l = []
    for rec in tree:
        l.append("\n")
        for field in rec:
            mkr = field.tag
            value = field.text
            if encoding is not None:
                if unicode_fields is not None and mkr in unicode_fields:
                    cur_encoding = "utf8"
                else:
                    cur_encoding = encoding
                if re.search(_is_value, value):
                    l.append((f"\\{mkr} {value}\n").encode(cur_encoding, errors))
                else:
                    l.append((f"\\{mkr}{value}\n").encode(cur_encoding, errors))
            else:
                if re.search(_is_value, value):
                    l.append(f"\\{mkr} {value}\n")
                else:
                    l.append(f"\\{mkr}{value}\n")
    return "".join(l[1:])


class ToolboxSettings(StandardFormat):
    

    def __init__(self):
        super().__init__()

    def parse(self, encoding=None, errors="strict", **kwargs):
        
        builder = TreeBuilder()
        for mkr, value in self.fields(encoding=encoding, errors=errors, **kwargs):
            
            
            block = mkr[0]
            if block in ("+", "-"):
                mkr = mkr[1:]
            else:
                block = None
            
            if block == "+":
                builder.start(mkr, {})
                builder.data(value)
            elif block == "-":
                builder.end(mkr)
            else:
                builder.start(mkr, {})
                builder.data(value)
                builder.end(mkr)
        return builder.close()


def to_settings_string(tree, encoding=None, errors="strict", unicode_fields=None):
    
    l = list()
    _to_settings_string(
        tree.getroot(),
        l,
        encoding=encoding,
        errors=errors,
        unicode_fields=unicode_fields,
    )
    return "".join(l)


def _to_settings_string(node, l, **kwargs):
    
    tag = node.tag
    text = node.text
    if len(node) == 0:
        if text:
            l.append(f"\\{tag} {text}\n")
        else:
            l.append("\\%s\n" % tag)
    else:
        if text:
            l.append(f"\\+{tag} {text}\n")
        else:
            l.append("\\+%s\n" % tag)
        for n in node:
            _to_settings_string(n, l, **kwargs)
        l.append("\\-%s\n" % tag)
    return


def remove_blanks(elem):
    
    out = list()
    for child in elem:
        remove_blanks(child)
        if child.text or len(child) > 0:
            out.append(child)
    elem[:] = out


def add_default_fields(elem, default_fields):
    
    for field in default_fields.get(elem.tag, []):
        if elem.find(field) is None:
            SubElement(elem, field)
    for child in elem:
        add_default_fields(child, default_fields)


def sort_fields(elem, field_orders):
    
    order_dicts = dict()
    for field, order in field_orders.items():
        order_dicts[field] = order_key = dict()
        for i, subfield in enumerate(order):
            order_key[subfield] = i
    _sort_fields(elem, order_dicts)


def _sort_fields(elem, orders_dicts):
    
    try:
        order = orders_dicts[elem.tag]
    except KeyError:
        pass
    else:
        tmp = sorted(
            ((order.get(child.tag, 1e9), i), child) for i, child in enumerate(elem)
        )
        elem[:] = [child for key, child in tmp]
    for child in elem:
        if len(child):
            _sort_fields(child, orders_dicts)


def add_blank_lines(tree, blanks_before, blanks_between):
    
    try:
        before = blanks_before[tree.tag]
        between = blanks_between[tree.tag]
    except KeyError:
        for elem in tree:
            if len(elem):
                add_blank_lines(elem, blanks_before, blanks_between)
    else:
        last_elem = None
        for elem in tree:
            tag = elem.tag
            if last_elem is not None and last_elem.tag != tag:
                if tag in before and last_elem is not None:
                    e = last_elem.getiterator()[-1]
                    e.text = (e.text or "") + "\n"
            else:
                if tag in between:
                    e = last_elem.getiterator()[-1]
                    e.text = (e.text or "") + "\n"
            if len(elem):
                add_blank_lines(elem, blanks_before, blanks_between)
            last_elem = elem


def demo():
    from itertools import islice

    
    
    file_path = find("corpora/toolbox/rotokas.dic")
    lexicon = ToolboxData(file_path).parse()
    print("first field in fourth record:")
    print(lexicon[3][0].tag)
    print(lexicon[3][0].text)

    print("\nfields in sequential order:")
    for field in islice(lexicon.find("record"), 10):
        print(field.tag, field.text)

    print("\nlx fields:")
    for field in islice(lexicon.findall("record/lx"), 10):
        print(field.text)

    settings = ToolboxSettings()
    file_path = find("corpora/toolbox/MDF/MDF_AltH.typ")
    settings.open(file_path)
    
    tree = settings.parse(unwrap=False, encoding="cp1252")
    print(tree.find("expset/expMDF/rtfPageSetup/paperSize").text)
    settings_tree = ElementTree(tree)
    print(to_settings_string(settings_tree).encode("utf8"))


if __name__ == "__main__":
    demo()