# Path: streamcorpus_pipeline/_clean_visible.py
# def cleanse(span, lower=True):
#     '''Convert a unicode string into a lowercase string with no
# punctuation and only spaces for whitespace.
# 
# Replace PennTreebank escaped brackets with ' ':
# -LRB- -RRB- -RSB- -RSB- -LCB- -RCB-
# (The acronyms stand for (Left|Right) (Round|Square|Curly) Bracket.)
# http://www.cis.upenn.edu/~treebank/tokenization.html
# 
# :param span: string
#     '''
#     assert isinstance(span, unicode), \
#         'got non-unicode string %r' % span
#     # lowercase, strip punctuation, and shrink all whitespace
#     span = penn_treebank_brackets.sub(' ', span)
#     if lower:
#         span = span.lower()
#     span = span.translate(strip_punctuation)
#     span = whitespace.sub(' ', span)
#     # trim any leading or trailing whitespace
#     return span.strip()
# 
# Path: streamcorpus_pipeline/_kvlayer_table_names.py
# HASH_TF_INDEX_TABLE = 'sitf'
# 
# HASH_FREQUENCY_TABLE = 'sihf'
# 
# HASH_KEYWORD_INDEX_TABLE = 'sihk'
# 
# def kvlayer_key_to_stream_id(k):
#     '''Convert a kvlayer key to a text stream ID.
# 
#     `k` should be of the same form produced by
#     :func:`stream_id_to_kvlayer_key`.
# 
#     :param k: :mod:`kvlayer` key tuple
#     :return: converted stream ID
#     :returntype str:
# 
#     '''
#     abs_url_hash, epoch_ticks = k
#     return '{0}-{1}'.format(epoch_ticks,
#                             base64.b16encode(abs_url_hash).lower())
# 
# def key_for_stream_item(si):
#     '''Get a kvlayer key from a stream item.
# 
#     The return tuple can be used directly as a key in the
#     :data:`STREAM_ITEMS_TABLE` table.  Note that this recalculates the
#     stream ID, and if the internal data on the stream item is inconsistent
#     then this could return a different result from
#     :func:`stream_id_to_kvlayer_key`.
# 
#     :param si: stream item to get key for
#     :return: :mod:`kvlayer` key tuple
# 
#     '''
#     # get binary 16 byte digest
#     urlhash = hashlib.md5(si.abs_url).digest()
#     return (urlhash, int(si.stream_time.epoch_ticks))


from collections import Counter, defaultdict
from many_stop_words import get_stop_words
from streamcorpus_pipeline._clean_visible import cleanse
from streamcorpus_pipeline._kvlayer_table_names import \
    HASH_TF_INDEX_TABLE, HASH_FREQUENCY_TABLE, HASH_KEYWORD_INDEX_TABLE, \
    kvlayer_key_to_stream_id, key_for_stream_item
import logging
import mmh3
        '''Get strings that correspond to some hash.

        No string will correspond to :data:`DOCUMENT_HASH_KEY`; use
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param int tok_hash: Murmur hash to query
        :return: list of :class:`unicode` strings

        '''
        return [tok_encoded.decode('utf8')
                for (_, tok_encoded) in
                self.client.scan_keys(HASH_KEYWORD_INDEX_TABLE,
                                      ((tok_hash,), (tok_hash,)))]

    def document_frequencies(self, hashes):
        '''Get document frequencies for a list of hashes.

        This will return all zeros unless the index was written with
        `hash_frequencies` set.  If :data:`DOCUMENT_HASH_KEY` is
        included in `hashes`, that value will be returned with the
        total number of documents indexed.  If you are looking for
        documents with that hash, pass
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param hashes: hashes to query
        :paramtype hashes: list of :class:`int`
        :return: map from hash to document frequency

        '''
        result = {}

# Path: streamcorpus_pipeline/_clean_visible.py
# def cleanse(span, lower=True):
#     '''Convert a unicode string into a lowercase string with no
# punctuation and only spaces for whitespace.
# 
# Replace PennTreebank escaped brackets with ' ':
# -LRB- -RRB- -RSB- -RSB- -LCB- -RCB-
# (The acronyms stand for (Left|Right) (Round|Square|Curly) Bracket.)
# http://www.cis.upenn.edu/~treebank/tokenization.html
# 
# :param span: string
#     '''
#     assert isinstance(span, unicode), \
#         'got non-unicode string %r' % span
#     # lowercase, strip punctuation, and shrink all whitespace
#     span = penn_treebank_brackets.sub(' ', span)
#     if lower:
#         span = span.lower()
#     span = span.translate(strip_punctuation)
#     span = whitespace.sub(' ', span)
#     # trim any leading or trailing whitespace
#     return span.strip()
# 
# Path: streamcorpus_pipeline/_kvlayer_table_names.py
# HASH_TF_INDEX_TABLE = 'sitf'
# 
# HASH_FREQUENCY_TABLE = 'sihf'
# 
# HASH_KEYWORD_INDEX_TABLE = 'sihk'
# 
# def kvlayer_key_to_stream_id(k):
#     '''Convert a kvlayer key to a text stream ID.
# 
#     `k` should be of the same form produced by
#     :func:`stream_id_to_kvlayer_key`.
# 
#     :param k: :mod:`kvlayer` key tuple
#     :return: converted stream ID
#     :returntype str:
# 
#     '''
#     abs_url_hash, epoch_ticks = k
#     return '{0}-{1}'.format(epoch_ticks,
#                             base64.b16encode(abs_url_hash).lower())
# 
# def key_for_stream_item(si):
#     '''Get a kvlayer key from a stream item.
# 
#     The return tuple can be used directly as a key in the
#     :data:`STREAM_ITEMS_TABLE` table.  Note that this recalculates the
#     stream ID, and if the internal data on the stream item is inconsistent
#     then this could return a different result from
#     :func:`stream_id_to_kvlayer_key`.
# 
#     :param si: stream item to get key for
#     :return: :mod:`kvlayer` key tuple
# 
#     '''
#     # get binary 16 byte digest
#     urlhash = hashlib.md5(si.abs_url).digest()
#     return (urlhash, int(si.stream_time.epoch_ticks))


# Path: streamcorpus_pipeline/_kvlayer_keyword_search.py
from collections import Counter, defaultdict
from many_stop_words import get_stop_words
from streamcorpus_pipeline._clean_visible import cleanse
from streamcorpus_pipeline._kvlayer_table_names import \
    HASH_TF_INDEX_TABLE, HASH_FREQUENCY_TABLE, HASH_KEYWORD_INDEX_TABLE, \
    kvlayer_key_to_stream_id, key_for_stream_item
import logging
import mmh3

        '''Get strings that correspond to some hash.

        No string will correspond to :data:`DOCUMENT_HASH_KEY`; use
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param int tok_hash: Murmur hash to query
        :return: list of :class:`unicode` strings

        '''
        return [tok_encoded.decode('utf8')
                for (_, tok_encoded) in
                self.client.scan_keys(HASH_KEYWORD_INDEX_TABLE,
                                      ((tok_hash,), (tok_hash,)))]

    def document_frequencies(self, hashes):
        '''Get document frequencies for a list of hashes.

        This will return all zeros unless the index was written with
        `hash_frequencies` set.  If :data:`DOCUMENT_HASH_KEY` is
        included in `hashes`, that value will be returned with the
        total number of documents indexed.  If you are looking for
        documents with that hash, pass
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param hashes: hashes to query
        :paramtype hashes: list of :class:`int`
        :return: map from hash to document frequency

        '''
        result = {}

        for (k, v) in self.client.get(HASH_FREQUENCY_TABLE,