# Path: cablemap.nlp/cablemap/nlp/utils.py
# class IncrementalMmWriter(object):
#     """\
#     Writer for the `Matrix Market Exchange Format <http://math.nist.gov/MatrixMarket/formats.html>`_
#     
#     Wraps a `gensim.matutils.MmWriter` to write a file incrementally: The total number
#     of vectors could be unknown in advance.
# 
#     Its purpose is somewhat similar to
#     `gensim.matutils.MmWriter.write_corpus(fname, corpus, progress_cnt=1000, index=False)` but
#     it doesn't expect that all vectors are available. The vectors are added incrementally via
#     `add_vector`.
#     """
#     def __init__(self, filename):
#         """\
# 
#         `filename`
#             The name of the target file.
#         """
#         mmw = MmWriter(filename)
#         # write empty headers to the file (with enough space to be overwritten later)
#         mmw.write_headers(-1, -1, -1) # will print 50 spaces followed by newline on the stats line
#         self._mmw = mmw
#         self._num_docs = -1
#         self._num_terms = 0
#         self._num_nnz = 0 # number of non-zeroes in the sparse corpus matrix
# 
#     def add_vector(self, vector):
#         """\
#         Writes the provided vector to the matrix.
# 
#         `vector`
#             An iterable of ``(word-id, word-frequency)`` tuples
#         """
#         self._num_docs+=1
#         max_id, veclen = self._mmw.write_vector(self._num_docs, vector)
#         self._num_terms = max(self._num_terms, 1 + max_id)
#         self._num_nnz += veclen
# 
#     def close(self):
#         """\
#         Closes the writer.
# 
#         This method MUST be called once all vectors are added.
#         """
#         self._mmw.fake_headers(self._num_docs+1, self._num_terms, self._num_nnz)
#         self._mmw.close()


import os
    import json
    import simplejson as json
from itertools import chain, count
from gensim.corpora.dictionary import Dictionary
from gensim import utils
from .utils import IncrementalMmWriter
        dct = Dictionary.load_from_text('/my/directory/cables_wordids.txt')
        # Create another corpus with the previously word dict
        corpus = CableCorpus('/my/directory/', dct, allow_dict_updates=False)
        # Add some texts
        ....
        corpus.close()
    """
    def __init__(self, path, dct=None, tokenizer=None, allow_dict_updates=True, prefix=None):
        """\
        Initializes the cable corpus.
        
        `path`
            Directory where the generated files are stored.
        `dct`
            An existing `gensim.corpora.dictionary.Dictionary`
            If it's ``None`` (default) a dictionary will be created.
        `tokenizer`
            A function to tokenize/normalize/clean-up/remove stop words from strings.
            If it's ``None`` (default), a default function will be used to tokenize texts.
        `allow_dict_updates`
            Indicats if unknown words should be added to the dictionary (default ``True``).
        `prefix`
            A prefix for the generated file names.
        """
        super(CableCorpus, self).__init__(tokenizer)
        if not os.path.isdir(path):
            raise IOError('Expected a directory path')
        self.dct = Dictionary() if dct is None else dct
        self._path = path
        self._prefix = prefix or 'cables_'

# Path: cablemap.nlp/cablemap/nlp/utils.py
# class IncrementalMmWriter(object):
#     """\
#     Writer for the `Matrix Market Exchange Format <http://math.nist.gov/MatrixMarket/formats.html>`_
#     
#     Wraps a `gensim.matutils.MmWriter` to write a file incrementally: The total number
#     of vectors could be unknown in advance.
# 
#     Its purpose is somewhat similar to
#     `gensim.matutils.MmWriter.write_corpus(fname, corpus, progress_cnt=1000, index=False)` but
#     it doesn't expect that all vectors are available. The vectors are added incrementally via
#     `add_vector`.
#     """
#     def __init__(self, filename):
#         """\
# 
#         `filename`
#             The name of the target file.
#         """
#         mmw = MmWriter(filename)
#         # write empty headers to the file (with enough space to be overwritten later)
#         mmw.write_headers(-1, -1, -1) # will print 50 spaces followed by newline on the stats line
#         self._mmw = mmw
#         self._num_docs = -1
#         self._num_terms = 0
#         self._num_nnz = 0 # number of non-zeroes in the sparse corpus matrix
# 
#     def add_vector(self, vector):
#         """\
#         Writes the provided vector to the matrix.
# 
#         `vector`
#             An iterable of ``(word-id, word-frequency)`` tuples
#         """
#         self._num_docs+=1
#         max_id, veclen = self._mmw.write_vector(self._num_docs, vector)
#         self._num_terms = max(self._num_terms, 1 + max_id)
#         self._num_nnz += veclen
# 
#     def close(self):
#         """\
#         Closes the writer.
# 
#         This method MUST be called once all vectors are added.
#         """
#         self._mmw.fake_headers(self._num_docs+1, self._num_terms, self._num_nnz)
#         self._mmw.close()


# Path: cablemap.nlp/cablemap/nlp/corpus.py
import os
    import json
    import simplejson as json
from itertools import chain, count
from gensim.corpora.dictionary import Dictionary
from gensim import utils
from .utils import IncrementalMmWriter

        dct = Dictionary.load_from_text('/my/directory/cables_wordids.txt')
        # Create another corpus with the previously word dict
        corpus = CableCorpus('/my/directory/', dct, allow_dict_updates=False)
        # Add some texts
        ....
        corpus.close()
    """
    def __init__(self, path, dct=None, tokenizer=None, allow_dict_updates=True, prefix=None):
        """\
        Initializes the cable corpus.
        
        `path`
            Directory where the generated files are stored.
        `dct`
            An existing `gensim.corpora.dictionary.Dictionary`
            If it's ``None`` (default) a dictionary will be created.
        `tokenizer`
            A function to tokenize/normalize/clean-up/remove stop words from strings.
            If it's ``None`` (default), a default function will be used to tokenize texts.
        `allow_dict_updates`
            Indicats if unknown words should be added to the dictionary (default ``True``).
        `prefix`
            A prefix for the generated file names.
        """
        super(CableCorpus, self).__init__(tokenizer)
        if not os.path.isdir(path):
            raise IOError('Expected a directory path')
        self.dct = Dictionary() if dct is None else dct
        self._path = path
        self._prefix = prefix or 'cables_'

        self._mw = IncrementalMmWriter(os.path.join(path, self._prefix + 'bow.mm'))