natural language toolkit corpus reader utility functions c 20012023 nltk project edward loper edlopergmail com url https www nltk org for license information see license txt lazy corpus loader to see the api documentation for this lazily loaded corpus first run corpus ensureloaded and then run helpthiscorpus lazycorpusloader is a proxy object which is used to stand in for a corpus object before the corpus is loaded this allows nltk to create an object for each corpus but defer the costs associated with loading those corpora until the first time that they re actually accessed the first time this object is accessed in any way it will load the corresponding corpus and transform itself into that corpus by modifying its own class and dict attributes if the corpus can not be found then accessing this object will raise an exception displaying installation instructions for the nltk data package once they ve properly installed the data package or modified nltk data path to point to its location they can then use the corpus object without restarting python param name the name of the corpus type name str param readercls the specific corpusreader class e g plaintextcorpusreader wordlistcorpusreader type reader nltk corpus reader api corpusreader param nltkdatasubdir the subdirectory where the corpus is stored type nltkdatasubdir str param args any other nonkeywords arguments that readercls might need param kwargs any other keywords arguments that readercls might need if nltkdatasubdir is set explicitly use the specified subdirectory path pops the nltkdatasubdir argument we don t need it anymore find the corpus root directory load the corpus this is where the magic happens transform ourselves into the corpus by modifying our own dict and class to match that of the corpus unload support assign dict and class back then do gc after reassigning dict there shouldn t be any references to corpus data so the memory should be deallocated after gc collect fix for inspect isclass under python 2 6 see https bugs python orgissue1225107 without this fix tests may take extra 1 5gb ram because all corpora gets loaded during test collection this looks circular but its not since load changes our class to something new if an exception occurs during corpus loading then unload method may be unattached so getattr can be called we shouldn t trigger corpus loading again in this case magic for creating bound methods used for unload natural language toolkit corpus reader utility functions c 2001 2023 nltk project edward loper edloper gmail com url https www nltk org for license information see license txt lazy corpus loader to see the api documentation for this lazily loaded corpus first run corpus ensure_loaded and then run help this_corpus lazycorpusloader is a proxy object which is used to stand in for a corpus object before the corpus is loaded this allows nltk to create an object for each corpus but defer the costs associated with loading those corpora until the first time that they re actually accessed the first time this object is accessed in any way it will load the corresponding corpus and transform itself into that corpus by modifying its own __class__ and __dict__ attributes if the corpus can not be found then accessing this object will raise an exception displaying installation instructions for the nltk data package once they ve properly installed the data package or modified nltk data path to point to its location they can then use the corpus object without restarting python param name the name of the corpus type name str param reader_cls the specific corpusreader class e g plaintextcorpusreader wordlistcorpusreader type reader nltk corpus reader api corpusreader param nltk_data_subdir the subdirectory where the corpus is stored type nltk_data_subdir str param args any other non keywords arguments that reader_cls might need param kwargs any other keywords arguments that reader_cls might need if nltk_data_subdir is set explicitly use the specified subdirectory path pops the nltk_data_subdir argument we don t need it anymore otherwise use nltk_data corpora find the corpus root directory load the corpus this is where the magic happens transform ourselves into the corpus by modifying our own __dict__ and __class__ to match that of the corpus _unload support assign __dict__ and __class__ back then do gc after reassigning __dict__ there shouldn t be any references to corpus data so the memory should be deallocated after gc collect fix for inspect isclass under python 2 6 see https bugs python org issue1225107 without this fix tests may take extra 1 5gb ram because all corpora gets loaded during test collection this looks circular but its not since __load changes our __class__ to something new if an exception occurs during corpus loading then _unload method may be unattached so __getattr__ can be called we shouldn t trigger corpus loading again in this case magic for creating bound methods used for _unload python3
import gc
import re

import nltk

TRY_ZIPFILE_FIRST = False


class LazyCorpusLoader:
    

    def __init__(self, name, reader_cls, *args, **kwargs):
        from nltk.corpus.reader.api import CorpusReader

        assert issubclass(reader_cls, CorpusReader)
        self.__name = self.__name__ = name
        self.__reader_cls = reader_cls
        
        if "nltk_data_subdir" in kwargs:
            
            self.subdir = kwargs["nltk_data_subdir"]
            
            kwargs.pop("nltk_data_subdir", None)
        else:  
            self.subdir = "corpora"
        self.__args = args
        self.__kwargs = kwargs

    def __load(self):
        
        zip_name = re.sub(r"(([^/]+)(/.*)?)", r"\2.zip/\1/", self.__name)
        if TRY_ZIPFILE_FIRST:
            try:
                root = nltk.data.find(f"{self.subdir}/{zip_name}")
            except LookupError as e:
                try:
                    root = nltk.data.find(f"{self.subdir}/{self.__name}")
                except LookupError:
                    raise e
        else:
            try:
                root = nltk.data.find(f"{self.subdir}/{self.__name}")
            except LookupError as e:
                try:
                    root = nltk.data.find(f"{self.subdir}/{zip_name}")
                except LookupError:
                    raise e

        
        corpus = self.__reader_cls(root, *self.__args, **self.__kwargs)

        
        
        

        args, kwargs = self.__args, self.__kwargs
        name, reader_cls = self.__name, self.__reader_cls

        self.__dict__ = corpus.__dict__
        self.__class__ = corpus.__class__

        
        
        
        def _unload(self):
            lazy_reader = LazyCorpusLoader(name, reader_cls, *args, **kwargs)
            self.__dict__ = lazy_reader.__dict__
            self.__class__ = lazy_reader.__class__
            gc.collect()

        self._unload = _make_bound_method(_unload, self)

    def __getattr__(self, attr):

        
        
        
        
        if attr == "__bases__":
            raise AttributeError("LazyCorpusLoader object has no attribute '__bases__'")

        self.__load()
        
        
        return getattr(self, attr)

    def __repr__(self):
        return "<{} in {!r} (not loaded yet)>".format(
            self.__reader_cls.__name__,
            ".../corpora/" + self.__name,
        )

    def _unload(self):
        
        
        
        pass


def _make_bound_method(func, self):
    

    class Foo:
        def meth(self):
            pass

    f = Foo()
    bound_method = type(f.meth)

    try:
        return bound_method(func, self, self.__class__)
    except TypeError:  
        return bound_method(func, self)