<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from collections import defaultdict
from pprint import pprint

def print_report(target_dict):

    len_pairs = len(target_dict)
    print 'Pairs Number: {:>6,}'.format(len_pairs)
    print

    print 'Count Each Lengths of Values:'

    lenv_count_map = defaultdict(int)
    for k, v in target_dict.iteritems():
        lenv_count_map[len(v)] += 1
    total_count = sum(lenv_count_map.itervalues())

    cum_pct = .0
    for lenv, count in sorted(lenv_count_map.iteritems(), key=lambda k: k[1], reverse=True):
        pct = 100.*count/total_count
        cum_pct += pct
        print ' {:>2,} | {:>7,} | {:>6.2f}% | {:>6.2f}%'.format(lenv, count, pct, cum_pct)
    print

    print 'Total  : {:>6,}'.format(total_count)
    print 'Average: {:>9,.2f}'.format(1.*total_count/len(lenv_count_map))

if __name__ == '__main__':

    from time import time

    start = time()
    import zipcodetw
    end = time()

    print '# Tokens -> Gradual Zipcode'
    print
    print_report(zipcodetw._dir.tokens_gzipcode_map)
    print
    print

    print '# Tokens -> Rule Str and Zipcode Pairs'
    print
    print_report(zipcodetw._dir.tokens_rzpairs_map)
    print<commit_msg>more meaningful stat<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from collections import defaultdict
from pprint import pprint

def print_report(target_dict):

    len_pairs = len(target_dict)
    print 'Pairs Number: {:>6,}'.format(len_pairs)
    print

    print 'Count Length of Each Key and Value Pair:'

    lenkv_count_map = defaultdict(int)
    for k, v in target_dict.iteritems():
        lenkv_count_map[(len(k), len(v))] += 1
    total_count = sum(lenkv_count_map.itervalues())

    cum_pct = .0
    for lenkv, count in sorted(lenkv_count_map.iteritems(), key=lambda k: k[1], reverse=True):
        pct = 100.*count/total_count
        cum_pct += pct
        print ' {:7} | {:>7,} | {:>6.2f}% | {:>6.2f}%'.format(lenkv, count, pct, cum_pct)
    print

    print 'Total  : {:>6,}'.format(total_count)
    print 'Average: {:>9,.2f}'.format(1.*total_count/len(lenkv_count_map))

if __name__ == '__main__':

    from time import time

    start = time()
    import zipcodetw
    end = time()

    print '# Tokens -> Gradual Zipcode'
    print
    print_report(zipcodetw._dir.tokens_gzipcode_map)
    print
    print

    print '# Tokens -> Rule Str and Zipcode Pairs'
    print
    print_report(zipcodetw._dir.tokens_rzpairs_map)
    print
{'fraction_non_alphanumeric': 0.13700281237444756, 'fraction_numerical': 0.01446364001607071, 'mean_word_length': 3.1848739495798317, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7437986', 'n_tokens_mistral': 952, 'n_tokens_neox': 890, 'n_words': 203}