<commit_before>#!/usr/bin/python
import lsh
import Levenshtein

import argparse
import sys
import time
import pickle

import MySQLdb as sql
import pymongo

import cProfile
import pstats


"""
This provides a CLI for the lsh module, which implements a Locality-Sensitive Hashing system to efficiently detect
duplicate reports.  It is currently designed to operate like a periodic background job.  To create a new cache and
index all of the documents in the Report collection added after <unix_time>, call:

    lsh_app.py -save <save_name> -start <unix_time>

then, to update the cache with any reports added to the db since the last document added to the lsh cache, call:

    lsh_app.py -cache <save_name>

"""

def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument('-cache', type=unicode, default=None, help='The location of the cache file to use. If provided, the program will run in worker mode, waiting for duplicate check requests against the specified cache file.')
    parser.add_argument('-save', type=unicode, default=u'anon_'+unicode(int(time.time())), help='Save name for cache file. If loading from saved cache, updated cache will be saved to same location.  Otherwise, default is anon_<unix_time>.cache')
    parser.add_argument('-start', type=int, default=None, help='If cache file not supplied, this arg specifies the earliest time from which to build a new cache.  value is given as unix time')
    parser.add_argument('-passive', action='store_true', default=False,  help='checks docs against cache without adding those docs to cache')
    parser.add_argument('-tag', type=str, default=unicode(int(time.time())), help='uses this name')
    parser.add_argument('-n', type=int, default=100, help='Number of hashes per document. Note, b*n==r.')
    parser.add_argument('-b', type=int, default=20, help='Number of bins in cache. Note, b*n==r.')
    parser.add_argument('-r', type=int, default=5, help='Number of rows per bin. Note, b*n==r.')
    parser.add_argument('-max_shingle', type=int, default=3, help='Maximum shingle size.)')
    parser.add_argument('-min_jaccard', type=float, default=0.8, help='minimum jaccard similarity to judge as duplicate')
    parser.add_argument('-max_edit_rate', type=float, default=.1, help='Maximum edit distance rate (edit_dist(doc1,doc2)/min(len(doc1),len(doc2))) to judge as a duplicate')
    parser.add_argument('-min_doc_len', type=int, default=100, help='Minimum doc length for fuzzy matching.')
    parser.add_argument('-db_host', type=unicode, default="10.172.126.80",help='mongodb hostname/ip')
    parser.add_argument('-db_name', type=unicode, default="epidemiciq",help='mongodb db name')
    parser.add_argument('-coll_name', type=unicode, default="Report",help='mongodb collection name with db specified by -db_name arg')

    args = parser.parse_args()
    print args
    return args

"""
Returns a mongo collection object for the db_host/db_name/coll_name collections
"""
def get_coll(db_host,db_name,coll_name):
    conn = pymongo.Connection(db_host)
    print '[get_coll]\tgot connection db_host: %s' % (db_host)
    db = conn[db_name]
    coll = db[coll_name]
    print '[get_coll]\tgot collection: epidemiciq.Report, count({})=%d' % (coll.count())
    return coll

"""
Computes the jaccard SIMILARITY between two tokenized lists<commit_msg>removed old application specific things ;<commit_after>#!/usr/bin/python
import lsh
import Levenshtein

import argparse
import sys
import time
import pickle

import MySQLdb as sql
import pymongo

import cProfile
import pstats


"""
This provides a CLI for the lsh module, which implements a Locality-Sensitive Hashing system to efficiently detect
duplicate reports.  It is currently designed to operate like a periodic background job.  To create a new cache and
index all of the documents in the Report collection added after <unix_time>, call:

    lsh_app.py -save <save_name> -start <unix_time>

then, to update the cache with any reports added to the db since the last document added to the lsh cache, call:

    lsh_app.py -cache <save_name>

"""

def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument('-cache', type=unicode, default=None, help='The location of the cache file to use. If provided, the program will run in worker mode, waiting for duplicate check requests against the specified cache file.')
    parser.add_argument('-save', type=unicode, default=u'anon_'+unicode(int(time.time())), help='Save name for cache file. If loading from saved cache, updated cache will be saved to same location.  Otherwise, default is anon_<unix_time>.cache')
    parser.add_argument('-start', type=int, default=None, help='If cache file not supplied, this arg specifies the earliest time from which to build a new cache.  value is given as unix time')
    parser.add_argument('-passive', action='store_true', default=False,  help='checks docs against cache without adding those docs to cache')
    parser.add_argument('-tag', type=str, default=unicode(int(time.time())), help='uses this name')
    parser.add_argument('-n', type=int, default=100, help='Number of hashes per document. Note, b*n==r.')
    parser.add_argument('-b', type=int, default=20, help='Number of bins in cache. Note, b*n==r.')
    parser.add_argument('-r', type=int, default=5, help='Number of rows per bin. Note, b*n==r.')
    parser.add_argument('-max_shingle', type=int, default=3, help='Maximum shingle size.)')
    parser.add_argument('-min_jaccard', type=float, default=0.8, help='minimum jaccard similarity to judge as duplicate')
    parser.add_argument('-max_edit_rate', type=float, default=.1, help='Maximum edit distance rate (edit_dist(doc1,doc2)/min(len(doc1),len(doc2))) to judge as a duplicate')
    parser.add_argument('-min_doc_len', type=int, default=100, help='Minimum doc length for fuzzy matching.')
    parser.add_argument('-db_host', type=unicode, default="localhost",help='mongodb hostname/ip')
    parser.add_argument('-db_name', type=unicode, default=None,help='mongodb db name')
    parser.add_argument('-coll_name', type=unicode, default="Report",help='mongodb collection name with db specified by -db_name arg')

    args = parser.parse_args()
    print args
    return args

"""
Returns a mongo collection object for the db_host/db_name/coll_name collections
"""
def get_coll(db_host,db_name,coll_name):
    conn = pymongo.Connection(db_host)
    print '[get_coll]\tgot connection db_host: %s' % (db_host)
    db = conn[db_name]
    coll = db[coll_name]
    print '[get_coll]\tgot collection: epidemiciq.Report, count({})=%d' % (coll.count())
    return coll

"""
Computes the jaccard SIMILARITY between two tokenized lists
{'fraction_non_alphanumeric': 0.11659192825112108, 'fraction_numerical': 0.006576980568011958, 'mean_word_length': 5.044263775971093, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '294850', 'n_tokens_mistral': 2092, 'n_tokens_neox': 2007, 'n_words': 684}