# Path: streamcorpus_pipeline/text_profile_signature.py
# def tps(text, min_token_len=2, quant_rate=0.01):
#     '''
#     :param text: tag-free UTF-8 string of free text
#     :returns string: 32-character md5 hash in hexadecimal
# 
#     Python implementation of the TextProfileSignature provided in
#     SOLR.  Unlike most other locality sensitive hashes, a TPS can be
#     indexed as a searchable property of each document that does not
#     require n^2 comparisons to find duplicates.
# 
#     http://wiki.apache.org/solr/TextProfileSignature
#     '''
#     
#     counts = Counter(
#         ifilter(lambda x: len(x) >= min_token_len, 
#                 imap(cleanse, text.split())))
# 
#     max_freq = counts.most_common(1)[0][1]
#     if max_freq <= 1:
#         quant = 1
#     else:
#         quant = max(2, round(max_freq * quant_rate))
# 
#     to_hash = []
#     for word, count in counts.most_common():
#         if count <= quant:
#             break
#         to_hash += [word, str(int(math.floor(count / quant)))]
# 
#     to_hash = u' '.join(to_hash)
#     return hashlib.md5(to_hash.encode('utf8')).hexdigest()


import os
import hashlib
from streamcorpus_pipeline.text_profile_signature import tps

from __future__ import absolute_import

def test_tps(test_data_dir):
    path = os.path.join(test_data_dir, 'test')
    text1 = open(os.path.join(path, 'nytimes-index-clean-visible.html')).read().decode('utf8')
    print hashlib.md5(repr(text1)).hexdigest(), tps(text1)
    text2 = open(os.path.join(path, 'nytimes-index-clean-visible-dup.html')).read().decode('utf8')

# Path: streamcorpus_pipeline/text_profile_signature.py
# def tps(text, min_token_len=2, quant_rate=0.01):
#     '''
#     :param text: tag-free UTF-8 string of free text
#     :returns string: 32-character md5 hash in hexadecimal
# 
#     Python implementation of the TextProfileSignature provided in
#     SOLR.  Unlike most other locality sensitive hashes, a TPS can be
#     indexed as a searchable property of each document that does not
#     require n^2 comparisons to find duplicates.
# 
#     http://wiki.apache.org/solr/TextProfileSignature
#     '''
#     
#     counts = Counter(
#         ifilter(lambda x: len(x) >= min_token_len, 
#                 imap(cleanse, text.split())))
# 
#     max_freq = counts.most_common(1)[0][1]
#     if max_freq <= 1:
#         quant = 1
#     else:
#         quant = max(2, round(max_freq * quant_rate))
# 
#     to_hash = []
#     for word, count in counts.most_common():
#         if count <= quant:
#             break
#         to_hash += [word, str(int(math.floor(count / quant)))]
# 
#     to_hash = u' '.join(to_hash)
#     return hashlib.md5(to_hash.encode('utf8')).hexdigest()


# Path: streamcorpus_pipeline/tests/test_text_profile_signature.py
import os
import hashlib
from streamcorpus_pipeline.text_profile_signature import tps


from __future__ import absolute_import

def test_tps(test_data_dir):
    path = os.path.join(test_data_dir, 'test')
    text1 = open(os.path.join(path, 'nytimes-index-clean-visible.html')).read().decode('utf8')
    print hashlib.md5(repr(text1)).hexdigest(), tps(text1)
    text2 = open(os.path.join(path, 'nytimes-index-clean-visible-dup.html')).read().decode('utf8')

    print hashlib.md5(repr(text2)).hexdigest(), tps(text2)