<commit_before>#! /usr/bin/env python

from detector import Detector

if __name__ == "__main__":
    # run the program
    detector = Detector('./test')
    print "Checking for duplicates using NDD..."
    print detector.check_for_duplicates()<commit_msg>Printing three nearest neighbors of first 10 docs<commit_after>#! /usr/bin/env python

import operator, copy
from detector import Detector

if __name__ == "__main__":
    # run the program
    detector = Detector('./test')
    print "Checking for duplicates using NDD..."
    duplicates = detector.check_for_duplicates()
    if duplicates:
        print "Duplicates found (Jaccard coefficient > 0.5):"
        print duplicates
    
    print "Printing three nearest neighbors of the first 10 files..."
    filenames_of_first_ten = [detector.filename("file%02d.txt") % (f,) for f in xrange(10)]
    for index1, j in enumerate(filenames_of_first_ten):
        jaccard_coefficients = [0,0,0,0,0,0,0,0,0,0]
        for index2, d in enumerate(filenames_of_first_ten):
            if index1 != index2:
                jaccard_coefficients[index2] = detector.get_jaccard(j, d)
        three_nearest = []
        nearest_count = -1
        jcos = copy.deepcopy(jaccard_coefficients)
        while len(three_nearest) < 3:
            index,coefficient = max(enumerate(jcos), key=operator.itemgetter(1))
            del jcos[index]
            # put the index back where it was in the original jaccard_coefficients
            if nearest_count == 0 and index >= three_nearest[0][0]:
                index += 1
            if nearest_count == 1:
                if index >= three_nearest[0][0]:
                    index += 1
                if index >= three_nearest[1][0]:
                    index += 1
            three_nearest.append((index,coefficient))
            nearest_count += 1
        print "Three nearest neighbors to %s:" % ("file%02d.txt" % index1)
        for near in sorted(three_nearest, key=operator.itemgetter(1), reverse=True):
            print "\t%s with Jaccard coefficient of %0.3f" % ("file%02d.txt" % near[0], near[1])
{'fraction_non_alphanumeric': 0.10856049736967958, 'fraction_numerical': 0.02391200382592061, 'mean_word_length': 2.557823129251701, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1572969', 'n_tokens_mistral': 668, 'n_tokens_neox': 636, 'n_words': 172}