<commit_before>#!/usr/bin/env python3

# PHMMER does not let you report only the best hit.
# I have 11GB Phmmer files :(

# Austin


import sys

lastquery = False

with open(sys.argv[1]) as handle:
    for line in handle:
        if line.startswith('#'):
            print(line.strip())
            continue
        line = line.split()
        query = line[2]
        if lastquery and query != lastquery:
            print(' '.join(line))
        lastquery = query
<commit_msg>bumped down to python 2-y code<commit_after>#!/usr/bin/env python

# PHMMER does not let you report only the best hit.
# I have 11GB Phmmer files :(

# Austin


import sys

lastquery = False

with open(sys.argv[1]) as handle:
    for line in handle:
        if line.startswith('#'):
            print line.strip()
            continue
        line = line.split()
        query = line[2]
        if lastquery and query != lastquery:
            print ' '.join(line)
        lastquery = query

{'fraction_non_alphanumeric': 0.10433884297520661, 'fraction_numerical': 0.010330578512396695, 'mean_word_length': 2.1157556270096465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3911986', 'n_tokens_mistral': 304, 'n_tokens_neox': 292, 'n_words': 100}