# Path: cablemap.nlp/cablemap/nlp/corpus.py
# class WordCorpus(BaseCorpus):
#     """\
#     Wrapper around a `gensim.corpora.dictionary.Dictionary`.
# 
#     This is a light-weight alternative to `CableCorpus` to create an initial
#     word dictionary::
# 
#         wd = WordCorpus()
#         wd.add_text('ref-1', 'bla bla')
#         # add more texts
#         wd.dct.filter_extremes()
# 
#         corpus = CableCorpus('/my/directory/', wd.dct)
#         corpus.add_text('ref-1', 'bla bla')
#         # add more texts
#         corpus.close()
#     """
#     def __init__(self, dct=None, tokenizer=None):
#         """\
#         Initializes the wrapper.
# 
#         `dct`
#             An existing Dictionary or ``None`` if a new Dictionary should be
#             created (default)
#         `tokenizer`
#             A tokenizer function or ``None``, see `BaseCorpus`
#         """
#         super(WordCorpus, self).__init__(tokenizer)
#         self.dct = Dictionary() if dct is None else dct
# 
#     def add_words(self, reference_id, words):
#         self.dct.doc2bow(words, True)
# 
# class CableCorpus(BaseCorpus):
#     """\
#     The cable corpus consists of several files which are written into a directory.
# 
#     * a dictionary with a ``<word id> <word> <frequency>`` mapping saved under "wordids.pickle"
#     * a JSON file with a ``<cable reference id> <document number>`` mapping under "id2docid.json"
#     * a `Market Matrix format <http://math.nist.gov/MatrixMarket/formats.html>` vector space model file "bow.mm"
# 
#     CAUTION: The corpus overrides any existing files with the same file name in the specified directory.
# 
#     By default, the corpus creates the word dictionary and the vector space model which
#     may lead into an unuseful vector space model. To filter certain words, the corpus may be
#     initialized with a pre-generated word dictionary. To make the dictionary immutable, the property
#     ``allow_dict_updates`` should be set to ``False`` (updates are allowed by default).
#     The resulting vector space model contains only words which are in the word dictionary then.
# 
#     Example to reduce the clutter::
# 
#         corpus = CableCorpus('/my/directory/')
#         # Add some texts here
#         corpus.add_text('ref-1', u'bla bla bla')
#         corpus.add_text('ref-2', u'bla bla blub')
#         ...
#         corpus.dct.filter_extremes()
#         corpus.close()
# 
#         from gensim.corpora.dictionary import Dictionary
# 
#         # Load previously created dict
#         dct = Dictionary.load_from_text('/my/directory/cables_wordids.txt')
#         # Create another corpus with the previously word dict
#         corpus = CableCorpus('/my/directory/', dct, allow_dict_updates=False)
#         # Add some texts
#         ....
#         corpus.close()
#     """
#     def __init__(self, path, dct=None, tokenizer=None, allow_dict_updates=True, prefix=None):
#         """\
#         Initializes the cable corpus.
#         
#         `path`
#             Directory where the generated files are stored.
#         `dct`
#             An existing `gensim.corpora.dictionary.Dictionary`
#             If it's ``None`` (default) a dictionary will be created.
#         `tokenizer`
#             A function to tokenize/normalize/clean-up/remove stop words from strings.
#             If it's ``None`` (default), a default function will be used to tokenize texts.
#         `allow_dict_updates`
#             Indicats if unknown words should be added to the dictionary (default ``True``).
#         `prefix`
#             A prefix for the generated file names.
#         """
#         super(CableCorpus, self).__init__(tokenizer)
#         if not os.path.isdir(path):
#             raise IOError('Expected a directory path')
#         self.dct = Dictionary() if dct is None else dct
#         self._path = path
#         self._prefix = prefix or 'cables_'
#         self._mw = IncrementalMmWriter(os.path.join(path, self._prefix + 'bow.mm'))
#         self.allow_dict_updates = allow_dict_updates
#         self._cables = []
# 
#     def add_words(self, reference_id, words):
#         self._cables.append(reference_id)
#         self._mw.add_vector(self.dct.doc2bow(words, self.allow_dict_updates))
# 
#     def close(self):
#         self._mw.close()
#         self.dct.save(os.path.join(self._path, self._prefix + 'wordids.pickle'))
#         json_filename = os.path.join(self._path, self._prefix + 'id2docid.json')
#         json.dump(dict(zip(self._cables, count())), open(json_filename, 'wb'))


from cablemap.core.handler import NoopCableHandler, DelegatingCableHandler
from .corpus import WordCorpus, CableCorpus
    Note: This handler won't attempt to persist the Dictionary. The caller
    should handle over an existing Dictionary or should save it in the `before_close`
    callback::

        from gensim.corpora.dictionary import Dictionary
        from cablemap.core import handle_source

        dct = Dictionary()
        handler = DictionaryHandler(dct)

        handle_source('cables.csv', handler)

        # Now save the dict:
        dct.save_as_text('/path/wordids.txt')
        
    """
    def __init__(self, dct=None, tokenizer=None, before_close=None):
        """\

        `dct`
            An existing `gensim.corpora.dictionary.Dictionary`
            If it's ``None`` (default) a dictionary will be created.
        `tokenizer`
            A function to tokenize/normalize/clean-up strings.
            If it's ``None`` (default), a default function will be used to tokenize
            texts.
        `before_close`
            An optional function which is called with the underlying corpus before it is
            closed.
        """

# Path: cablemap.nlp/cablemap/nlp/corpus.py
# class WordCorpus(BaseCorpus):
#     """\
#     Wrapper around a `gensim.corpora.dictionary.Dictionary`.
# 
#     This is a light-weight alternative to `CableCorpus` to create an initial
#     word dictionary::
# 
#         wd = WordCorpus()
#         wd.add_text('ref-1', 'bla bla')
#         # add more texts
#         wd.dct.filter_extremes()
# 
#         corpus = CableCorpus('/my/directory/', wd.dct)
#         corpus.add_text('ref-1', 'bla bla')
#         # add more texts
#         corpus.close()
#     """
#     def __init__(self, dct=None, tokenizer=None):
#         """\
#         Initializes the wrapper.
# 
#         `dct`
#             An existing Dictionary or ``None`` if a new Dictionary should be
#             created (default)
#         `tokenizer`
#             A tokenizer function or ``None``, see `BaseCorpus`
#         """
#         super(WordCorpus, self).__init__(tokenizer)
#         self.dct = Dictionary() if dct is None else dct
# 
#     def add_words(self, reference_id, words):
#         self.dct.doc2bow(words, True)
# 
# class CableCorpus(BaseCorpus):
#     """\
#     The cable corpus consists of several files which are written into a directory.
# 
#     * a dictionary with a ``<word id> <word> <frequency>`` mapping saved under "wordids.pickle"
#     * a JSON file with a ``<cable reference id> <document number>`` mapping under "id2docid.json"
#     * a `Market Matrix format <http://math.nist.gov/MatrixMarket/formats.html>` vector space model file "bow.mm"
# 
#     CAUTION: The corpus overrides any existing files with the same file name in the specified directory.
# 
#     By default, the corpus creates the word dictionary and the vector space model which
#     may lead into an unuseful vector space model. To filter certain words, the corpus may be
#     initialized with a pre-generated word dictionary. To make the dictionary immutable, the property
#     ``allow_dict_updates`` should be set to ``False`` (updates are allowed by default).
#     The resulting vector space model contains only words which are in the word dictionary then.
# 
#     Example to reduce the clutter::
# 
#         corpus = CableCorpus('/my/directory/')
#         # Add some texts here
#         corpus.add_text('ref-1', u'bla bla bla')
#         corpus.add_text('ref-2', u'bla bla blub')
#         ...
#         corpus.dct.filter_extremes()
#         corpus.close()
# 
#         from gensim.corpora.dictionary import Dictionary
# 
#         # Load previously created dict
#         dct = Dictionary.load_from_text('/my/directory/cables_wordids.txt')
#         # Create another corpus with the previously word dict
#         corpus = CableCorpus('/my/directory/', dct, allow_dict_updates=False)
#         # Add some texts
#         ....
#         corpus.close()
#     """
#     def __init__(self, path, dct=None, tokenizer=None, allow_dict_updates=True, prefix=None):
#         """\
#         Initializes the cable corpus.
#         
#         `path`
#             Directory where the generated files are stored.
#         `dct`
#             An existing `gensim.corpora.dictionary.Dictionary`
#             If it's ``None`` (default) a dictionary will be created.
#         `tokenizer`
#             A function to tokenize/normalize/clean-up/remove stop words from strings.
#             If it's ``None`` (default), a default function will be used to tokenize texts.
#         `allow_dict_updates`
#             Indicats if unknown words should be added to the dictionary (default ``True``).
#         `prefix`
#             A prefix for the generated file names.
#         """
#         super(CableCorpus, self).__init__(tokenizer)
#         if not os.path.isdir(path):
#             raise IOError('Expected a directory path')
#         self.dct = Dictionary() if dct is None else dct
#         self._path = path
#         self._prefix = prefix or 'cables_'
#         self._mw = IncrementalMmWriter(os.path.join(path, self._prefix + 'bow.mm'))
#         self.allow_dict_updates = allow_dict_updates
#         self._cables = []
# 
#     def add_words(self, reference_id, words):
#         self._cables.append(reference_id)
#         self._mw.add_vector(self.dct.doc2bow(words, self.allow_dict_updates))
# 
#     def close(self):
#         self._mw.close()
#         self.dct.save(os.path.join(self._path, self._prefix + 'wordids.pickle'))
#         json_filename = os.path.join(self._path, self._prefix + 'id2docid.json')
#         json.dump(dict(zip(self._cables, count())), open(json_filename, 'wb'))


# Path: cablemap.nlp/cablemap/nlp/handler.py
from cablemap.core.handler import NoopCableHandler, DelegatingCableHandler
from .corpus import WordCorpus, CableCorpus

    Note: This handler won't attempt to persist the Dictionary. The caller
    should handle over an existing Dictionary or should save it in the `before_close`
    callback::

        from gensim.corpora.dictionary import Dictionary
        from cablemap.core import handle_source

        dct = Dictionary()
        handler = DictionaryHandler(dct)

        handle_source('cables.csv', handler)

        # Now save the dict:
        dct.save_as_text('/path/wordids.txt')
        
    """
    def __init__(self, dct=None, tokenizer=None, before_close=None):
        """\

        `dct`
            An existing `gensim.corpora.dictionary.Dictionary`
            If it's ``None`` (default) a dictionary will be created.
        `tokenizer`
            A function to tokenize/normalize/clean-up strings.
            If it's ``None`` (default), a default function will be used to tokenize
            texts.
        `before_close`
            An optional function which is called with the underlying corpus before it is
            closed.
        """

        super(DictionaryHandler, self).__init__(WordCorpus(dct, tokenizer), before_close)