<commit_before>import hashlib


def record_is_valid(record):
    "Checks if a record is valid for processing."

    # No random contigs
    if record.CHROM.startswith('GL'):
        return False

    # Skip results with a read depth < 5
    if record.INFO['DP'] < 5:
        return False

    return True


def file_md5(filename, block_size=2**16):
    md5 = hashlib.md5()
    with open(filename, 'rb') as fin:
        while True:
            data = fin.read(block_size)
            if not data:
                break
            md5.update(data)
    return md5.hexdigest()
<commit_msg>Do not assume read depth is present in pipeline checks<commit_after>import hashlib


def record_is_valid(record):
    "Checks if a record is valid for processing."

    # No random contigs
    if record.CHROM.startswith('GL'):
        return False

    # Skip results with a read depth < 5. If no read depth is specified then
    # we have no choice but to consider this record as being valid.
    if 'DP' in record.INFO and record.INFO['DP'] < 5:
        return False

    return True


def file_md5(filename, block_size=2**16):
    md5 = hashlib.md5()
    with open(filename, 'rb') as fin:
        while True:
            data = fin.read(block_size)
            if not data:
                break
            md5.update(data)
    return md5.hexdigest()

{'fraction_non_alphanumeric': 0.08650260999254288, 'fraction_numerical': 0.014914243102162566, 'mean_word_length': 2.233734939759036, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2942224', 'n_tokens_mistral': 418, 'n_tokens_neox': 397, 'n_words': 140}