# Path: cablemap.nlp/cablemap/nlp/corpus.py
# class WordCorpus(BaseCorpus):
#     """\
#     Wrapper around a `gensim.corpora.dictionary.Dictionary`.
# 
#     This is a light-weight alternative to `CableCorpus` to create an initial
#     word dictionary::
# 
#         wd = WordCorpus()
#         wd.add_text('ref-1', 'bla bla')
#         # add more texts
#         wd.dct.filter_extremes()
# 
#         corpus = CableCorpus('/my/directory/', wd.dct)
#         corpus.add_text('ref-1', 'bla bla')
#         # add more texts
#         corpus.close()
#     """
#     def __init__(self, dct=None, tokenizer=None):
#         """\
#         Initializes the wrapper.
# 
#         `dct`
#             An existing Dictionary or ``None`` if a new Dictionary should be
#             created (default)
#         `tokenizer`
#             A tokenizer function or ``None``, see `BaseCorpus`
#         """
#         super(WordCorpus, self).__init__(tokenizer)
#         self.dct = Dictionary() if dct is None else dct
# 
#     def add_words(self, reference_id, words):
#         self.dct.doc2bow(words, True)
# 
# class CableCorpus(BaseCorpus):
#     """\
#     The cable corpus consists of several files which are written into a directory.
# 
#     * a dictionary with a ``<word id> <word> <frequency>`` mapping saved under "wordids.pickle"
#     * a JSON file with a ``<cable reference id> <document number>`` mapping under "id2docid.json"
#     * a `Market Matrix format <http://math.nist.gov/MatrixMarket/formats.html>` vector space model file "bow.mm"
# 
#     CAUTION: The corpus overrides any existing files with the same file name in the specified directory.
# 
#     By default, the corpus creates the word dictionary and the vector space model which
#     may lead into an unuseful vector space model. To filter certain words, the corpus may be
#     initialized with a pre-generated word dictionary. To make the dictionary immutable, the property
#     ``allow_dict_updates`` should be set to ``False`` (updates are allowed by default).
#     The resulting vector space model contains only words which are in the word dictionary then.
# 
#     Example to reduce the clutter::
# 
#         corpus = CableCorpus('/my/directory/')
#         # Add some texts here
#         corpus.add_text('ref-1', u'bla bla bla')
#         corpus.add_text('ref-2', u'bla bla blub')
#         ...
#         corpus.dct.filter_extremes()
#         corpus.close()
# 
#         from gensim.corpora.dictionary import Dictionary
# 
#         # Load previously created dict
#         dct = Dictionary.load_from_text('/my/directory/cables_wordids.txt')
#         # Create another corpus with the previously word dict
#         corpus = CableCorpus('/my/directory/', dct, allow_dict_updates=False)
#         # Add some texts
#         ....
#         corpus.close()
#     """
#     def __init__(self, path, dct=None, tokenizer=None, allow_dict_updates=True, prefix=None):
#         """\
#         Initializes the cable corpus.
#         
#         `path`
#             Directory where the generated files are stored.
#         `dct`
#             An existing `gensim.corpora.dictionary.Dictionary`
#             If it's ``None`` (default) a dictionary will be created.
#         `tokenizer`
#             A function to tokenize/normalize/clean-up/remove stop words from strings.
#             If it's ``None`` (default), a default function will be used to tokenize texts.
#         `allow_dict_updates`
#             Indicats if unknown words should be added to the dictionary (default ``True``).
#         `prefix`
#             A prefix for the generated file names.
#         """
#         super(CableCorpus, self).__init__(tokenizer)
#         if not os.path.isdir(path):
#             raise IOError('Expected a directory path')
#         self.dct = Dictionary() if dct is None else dct
#         self._path = path
#         self._prefix = prefix or 'cables_'
#         self._mw = IncrementalMmWriter(os.path.join(path, self._prefix + 'bow.mm'))
#         self.allow_dict_updates = allow_dict_updates
#         self._cables = []
# 
#     def add_words(self, reference_id, words):
#         self._cables.append(reference_id)
#         self._mw.add_vector(self.dct.doc2bow(words, self.allow_dict_updates))
# 
#     def close(self):
#         self._mw.close()
#         self.dct.save(os.path.join(self._path, self._prefix + 'wordids.pickle'))
#         json_filename = os.path.join(self._path, self._prefix + 'id2docid.json')
#         json.dump(dict(zip(self._cables, count())), open(json_filename, 'wb'))


from cablemap.core.handler import NoopCableHandler, DelegatingCableHandler
from .corpus import WordCorpus, CableCorpus
            closed.
        """
        super(DictionaryHandler, self).__init__(WordCorpus(dct, tokenizer), before_close)

class CorpusHandler(NLPCableHandler):
    """\
    Creates a `cablemap.nlp.corpus.CableCorpus` instance.
    """
    def __init__(self, path, dct=None, tokenizer=None, allow_dict_updates=True, prefix=None, before_close=None):
        """\
        Initializes the corpus writer which creates a new `CableCorpus`.

        `path`
            Directory where the generated files are stored.
        `dct`
            An existing `gensim.corpora.dictionary.Dictionary`
            If it's ``None`` (default) a dictionary will be created.
        `tokenizer`
            A function to tokenize/normalize/clean-up strings.
            If it's ``None`` (default), a default function will be used to tokenize
            texts.
        `allow_dict_updates`
            Indicats if unknown words should be added to the dictionary (default ``True``).
        `prefix`
            A prefix for the generated file names.
        `before_close`
            An optional function which is called with the underlying corpus before it is
            closed. May be useful to modify the corpus or the Dictionary before changes are
            written to disk.
        """

# Path: cablemap.nlp/cablemap/nlp/corpus.py
# class WordCorpus(BaseCorpus):
#     """\
#     Wrapper around a `gensim.corpora.dictionary.Dictionary`.
# 
#     This is a light-weight alternative to `CableCorpus` to create an initial
#     word dictionary::
# 
#         wd = WordCorpus()
#         wd.add_text('ref-1', 'bla bla')
#         # add more texts
#         wd.dct.filter_extremes()
# 
#         corpus = CableCorpus('/my/directory/', wd.dct)
#         corpus.add_text('ref-1', 'bla bla')
#         # add more texts
#         corpus.close()
#     """
#     def __init__(self, dct=None, tokenizer=None):
#         """\
#         Initializes the wrapper.
# 
#         `dct`
#             An existing Dictionary or ``None`` if a new Dictionary should be
#             created (default)
#         `tokenizer`
#             A tokenizer function or ``None``, see `BaseCorpus`
#         """
#         super(WordCorpus, self).__init__(tokenizer)
#         self.dct = Dictionary() if dct is None else dct
# 
#     def add_words(self, reference_id, words):
#         self.dct.doc2bow(words, True)
# 
# class CableCorpus(BaseCorpus):
#     """\
#     The cable corpus consists of several files which are written into a directory.
# 
#     * a dictionary with a ``<word id> <word> <frequency>`` mapping saved under "wordids.pickle"
#     * a JSON file with a ``<cable reference id> <document number>`` mapping under "id2docid.json"
#     * a `Market Matrix format <http://math.nist.gov/MatrixMarket/formats.html>` vector space model file "bow.mm"
# 
#     CAUTION: The corpus overrides any existing files with the same file name in the specified directory.
# 
#     By default, the corpus creates the word dictionary and the vector space model which
#     may lead into an unuseful vector space model. To filter certain words, the corpus may be
#     initialized with a pre-generated word dictionary. To make the dictionary immutable, the property
#     ``allow_dict_updates`` should be set to ``False`` (updates are allowed by default).
#     The resulting vector space model contains only words which are in the word dictionary then.
# 
#     Example to reduce the clutter::
# 
#         corpus = CableCorpus('/my/directory/')
#         # Add some texts here
#         corpus.add_text('ref-1', u'bla bla bla')
#         corpus.add_text('ref-2', u'bla bla blub')
#         ...
#         corpus.dct.filter_extremes()
#         corpus.close()
# 
#         from gensim.corpora.dictionary import Dictionary
# 
#         # Load previously created dict
#         dct = Dictionary.load_from_text('/my/directory/cables_wordids.txt')
#         # Create another corpus with the previously word dict
#         corpus = CableCorpus('/my/directory/', dct, allow_dict_updates=False)
#         # Add some texts
#         ....
#         corpus.close()
#     """
#     def __init__(self, path, dct=None, tokenizer=None, allow_dict_updates=True, prefix=None):
#         """\
#         Initializes the cable corpus.
#         
#         `path`
#             Directory where the generated files are stored.
#         `dct`
#             An existing `gensim.corpora.dictionary.Dictionary`
#             If it's ``None`` (default) a dictionary will be created.
#         `tokenizer`
#             A function to tokenize/normalize/clean-up/remove stop words from strings.
#             If it's ``None`` (default), a default function will be used to tokenize texts.
#         `allow_dict_updates`
#             Indicats if unknown words should be added to the dictionary (default ``True``).
#         `prefix`
#             A prefix for the generated file names.
#         """
#         super(CableCorpus, self).__init__(tokenizer)
#         if not os.path.isdir(path):
#             raise IOError('Expected a directory path')
#         self.dct = Dictionary() if dct is None else dct
#         self._path = path
#         self._prefix = prefix or 'cables_'
#         self._mw = IncrementalMmWriter(os.path.join(path, self._prefix + 'bow.mm'))
#         self.allow_dict_updates = allow_dict_updates
#         self._cables = []
# 
#     def add_words(self, reference_id, words):
#         self._cables.append(reference_id)
#         self._mw.add_vector(self.dct.doc2bow(words, self.allow_dict_updates))
# 
#     def close(self):
#         self._mw.close()
#         self.dct.save(os.path.join(self._path, self._prefix + 'wordids.pickle'))
#         json_filename = os.path.join(self._path, self._prefix + 'id2docid.json')
#         json.dump(dict(zip(self._cables, count())), open(json_filename, 'wb'))


# Path: cablemap.nlp/cablemap/nlp/handler.py
from cablemap.core.handler import NoopCableHandler, DelegatingCableHandler
from .corpus import WordCorpus, CableCorpus

            closed.
        """
        super(DictionaryHandler, self).__init__(WordCorpus(dct, tokenizer), before_close)

class CorpusHandler(NLPCableHandler):
    """\
    Creates a `cablemap.nlp.corpus.CableCorpus` instance.
    """
    def __init__(self, path, dct=None, tokenizer=None, allow_dict_updates=True, prefix=None, before_close=None):
        """\
        Initializes the corpus writer which creates a new `CableCorpus`.

        `path`
            Directory where the generated files are stored.
        `dct`
            An existing `gensim.corpora.dictionary.Dictionary`
            If it's ``None`` (default) a dictionary will be created.
        `tokenizer`
            A function to tokenize/normalize/clean-up strings.
            If it's ``None`` (default), a default function will be used to tokenize
            texts.
        `allow_dict_updates`
            Indicats if unknown words should be added to the dictionary (default ``True``).
        `prefix`
            A prefix for the generated file names.
        `before_close`
            An optional function which is called with the underlying corpus before it is
            closed. May be useful to modify the corpus or the Dictionary before changes are
            written to disk.
        """

        super(CorpusHandler, self).__init__(CableCorpus(path, dct, tokenizer, allow_dict_updates, prefix),