<commit_before>"""Functions for checking files"""
import os
import stat
import mimetypes

from .checker import is_link
from .config import CONFIG

def is_fastq(fi):
    """Check whether a given file is a fastq file."""
    path = fi['path']
    if os.path.splitext(path)[1] == ".fastq":
        if not is_link(path):
            return 'PROB_FILE_IS_FASTQ'

def sam_should_compress(fi):
    """Check if a *.SAM file should be compressed or deleted"""
    path = fi['path']
    name, ext = os.path.splitext(path)
    if ext == '.sam':
        if os.path.isfile('.'.join((name, 'bam'))):
            return 'PROB_SAM_AND_BAM_EXIST'
        else:
            return 'PROB_SAM_SHOULD_COMPRESS'

def is_large_plaintext(fi):
    """Detect if a file plaintext and >100MB"""
    guess = mimetypes.guess_type(fi['path'])
    mod_days = fi['lastmod'] / (24 * 60 * 60) # Days since last modification

    large = CONFIG.get('checks', 'largesize',
                       fallback=100000000) # Default to 100MB
    old = CONFIG.get('checks', 'oldage',
                     fallback=30) # Default to one month
    if guess == 'text/plain' and fi['size'] > large and mod_days >= old:
        return 'PROB_OLD_LARGE_PLAINTEXT'<commit_msg>Fix SAM checker to for better coverage<commit_after>"""Functions for checking files"""
import os
import stat
import mimetypes

from .checker import is_link
from .config import CONFIG

def is_fastq(fi):
    """Check whether a given file is a fastq file."""
    path = fi['path']
    if os.path.splitext(path)[1] == ".fastq":
        if not is_link(path):
            return 'PROB_FILE_IS_FASTQ'

def sam_should_compress(fi):
    """Check if a *.SAM file should be compressed or deleted"""
    path = fi['path']
    name, ext = os.path.splitext(path)
    if ext == '.sam':
        if os.path.isfile('.'.join((name, 'bam'))):
            return 'PROB_SAM_AND_BAM_EXIST'
        else:
            return 'PROB_SAM_SHOULD_COMPRESS'
    elif ext == '.bam':
        if os.path.isfile('.'.join((name, 'sam'))):
            return 'PROB_SAM_SHOULD_COMPRESS'

def is_large_plaintext(fi):
    """Detect if a file plaintext and >100MB"""
    guess = mimetypes.guess_type(fi['path'])
    mod_days = fi['lastmod'] / (24 * 60 * 60) # Days since last modification

    large = CONFIG.get('checks', 'largesize',
                       fallback=100000000) # Default to 100MB
    old = CONFIG.get('checks', 'oldage',
                     fallback=30) # Default to one month
    if guess == 'text/plain' and fi['size'] > large and mod_days >= old:
        return 'PROB_OLD_LARGE_PLAINTEXT'
{'fraction_non_alphanumeric': 0.1327672713238132, 'fraction_numerical': 0.018525665766113468, 'mean_word_length': 2.806167400881057, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7009835', 'n_tokens_mistral': 910, 'n_tokens_neox': 843, 'n_words': 239}