natural language toolkit toolbox reader c 20012023 nltk project greg aumann gregaumannsil org stuart robinson stuart robinsonmpi nl steven bird stevenbird1gmail com url https www nltk org for license information see license txt module for reading writing and manipulating toolbox databases and settings fileids should probably be done lazily natural language toolkit toolbox reader c 2001 2023 nltk project greg aumann greg_aumann sil org stuart robinson stuart robinson mpi nl steven bird stevenbird1 gmail com url https www nltk org for license information see license txt module for reading writing and manipulating toolbox databases and settings fileids should probably be done lazily the default key in mdf
from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.toolbox import ToolboxData


class ToolboxCorpusReader(CorpusReader):
    def xml(self, fileids, key=None):
        return concat(
            [
                ToolboxData(path, enc).parse(key=key)
                for (path, enc) in self.abspaths(fileids, True)
            ]
        )

    def fields(
        self,
        fileids,
        strip=True,
        unwrap=True,
        encoding="utf8",
        errors="strict",
        unicode_fields=None,
    ):
        return concat(
            [
                list(
                    ToolboxData(fileid, enc).fields(
                        strip, unwrap, encoding, errors, unicode_fields
                    )
                )
                for (fileid, enc) in self.abspaths(fileids, include_encoding=True)
            ]
        )

    
    def entries(self, fileids, **kwargs):
        if "key" in kwargs:
            key = kwargs["key"]
            del kwargs["key"]
        else:
            key = "lx"  
        entries = []
        for marker, contents in self.fields(fileids, **kwargs):
            if marker == key:
                entries.append((contents, []))
            else:
                try:
                    entries[-1][-1].append((marker, contents))
                except IndexError:
                    pass
        return entries

    def words(self, fileids, key="lx"):
        return [contents for marker, contents in self.fields(fileids) if marker == key]


def demo():
    pass


if __name__ == "__main__":
    demo()