<commit_before>from collections import defaultdict

from kolekto.printer import printer
from kolekto.commands import Command


class FindDuplicates(Command):

    """ Find duplicate movies.
    """

    help = 'find duplicate movies in collection'

    def run(self, args, config):
        mdb = self.get_metadata_db(args.tree)
        hash_by_title = defaultdict(lambda: [])
        for movie_hash, movie in mdb.itermovies():
            hash_by_title[movie.get('title', None), movie.get('year', None)].append(movie_hash)

        for (title, year), hashs in hash_by_title.iteritems():
            if len(hashs) > 1:
                printer.p('<b>{title}</b> ({year}): {hashs}', title=title,
                          year=year, hashs=' '.join(hashs))<commit_msg>Now show quality on output of find-duplicates<commit_after>from collections import defaultdict

from kolekto.printer import printer
from kolekto.commands import Command
from kolekto.datasources import MovieDatasource


class FindDuplicates(Command):

    """ Find duplicate movies.
    """

    help = 'find duplicate movies in collection'

    def run(self, args, config):
        mds = MovieDatasource(config.subsections('datasource'), args.tree, self.profile.object_class)
        mdb = self.get_metadata_db(args.tree)
        hash_by_title = defaultdict(lambda: [])
        for movie_hash, movie in mdb.itermovies():
            movie = mds.attach(movie_hash, movie)
            hash_info = '<inv> %s </inv> (%s/%s)' % (movie_hash, movie.get('quality'), movie.get('ext'))
            hash_by_title[movie.get('title', None), movie.get('year', None)].append(hash_info)

        for (title, year), hashs in hash_by_title.iteritems():
            if len(hashs) > 1:
                printer.p('<b>{title}</b> ({year}): {hashs}', title=title,
                          year=year, hashs=' '.join(hashs))
{'fraction_non_alphanumeric': 0.13251072961373392, 'fraction_numerical': 0.001072961373390558, 'mean_word_length': 3.010752688172043, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4022769', 'n_tokens_mistral': 579, 'n_tokens_neox': 564, 'n_words': 136}