<commit_before>import khmer, sys, os
import gc
import glob

K = 32

filename=sys.argv[1]
subset_filenames=sys.argv[2:]

if not os.path.exists(filename):
    print '%s doesn\'t exist! dying.' % filename
    sys.exit(0)

# create a fake-ish ht; K matters, but not hashtable size.
ht = khmer.new_hashbits(32, 1, 1)
 
# load & merge
for subset_file in subset_filenames:
    print '<-', subset_file
    ht.merge_subset_from_disk(subset_file)

# save merged partitionmap
ht.save_partitionmap(filename + '.pmap.merged')

# partition!
n_partitions = ht.output_partitions(filename, filename + '.part')
print n_partitions
print ht.count_partitions()<commit_msg>cleaned up output display on partition load<commit_after>import khmer, sys, os
import gc
import glob

K = 32

filename=sys.argv[1]
subset_filenames=sys.argv[2:]

if not os.path.exists(filename):
    print '%s doesn\'t exist! dying.' % filename
    sys.exit(0)

# create a fake-ish ht; K matters, but not hashtable size.
ht = khmer.new_hashbits(32, 1, 1)
 
# load & merge
for subset_file in subset_filenames:
    print '<-', subset_file
    ht.merge_subset_from_disk(subset_file)

# save merged partitionmap
ht.save_partitionmap(filename + '.pmap.merged')

# partition!
n_partitions = ht.output_partitions(filename, filename + '.part')
(n_partitions, n_singletons) = ht.count_partitions()
print 'output partitions:', n_partitions
print 'pmap partitions:', n_partitions
print 'singletons:', n_singletons
{'fraction_non_alphanumeric': 0.11570247933884298, 'fraction_numerical': 0.012396694214876033, 'mean_word_length': 5.0041322314049586, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7031221', 'n_tokens_mistral': 534, 'n_tokens_neox': 504, 'n_words': 133}