<commit_before>#!/usr/bin/env python

import os
import time


class Utils(object):
    """
    A few things we'll (eventually) use.
    """

    def __init__(self):
        # These are file types we do not want to send to DocumentCloud.
        self.file_excludes = (
            'aiff',
            'DS_Store',
            'flac',
            'mid',
            'mdb',
            'mp3',
            'ogg',
            'pst',
            'wav',
            'wma'
        )

    def file_directory_check(self, file_directory):
        dir_exists = os.path.isdir(file_directory)
        return dir_exists

    def sanitize_uploads(self, doc_list):
        """
        Flag prohibited file types and files over 400MB upload size limit.
        """
        for x in doc_list:
            file_split = x['name'].split('.')
            if file_split[-1] in self.file_excludes:
                x['exclude_flag'] = True
                x['exclude_reason'] = 'Prohibited file type'
            elif os.path.getsize(x['full_path']) > 400000000:
                x['exclude_flag'] = True
                x['exclude_reason'] = 'File over 400MB'
            else:
                x['exclude_flag'] = False
        return doc_list

    def timestamp(self):
        self.time = time.strftime("%Y%m%dT%H%M%S")
        return self.time
<commit_msg>Better test for extension on excluded files.<commit_after>#!/usr/bin/env python

import os
import time


class Utils(object):
    """
    A few things we'll (eventually) use.
    """

    def __init__(self):
        # These are file types we do not want to send to DocumentCloud.
        self.file_excludes = (
            'accdb',
            'aiff',
            'ds_store',
            'exe',
            'flac',
            'mid',
            'mdb',
            'mov',
            'mp3',
            'mpa',
            'ogg',
            'pst',
            'wav',
            'wma',
            'zip'
        )

    def file_directory_check(self, file_directory):
        dir_exists = os.path.isdir(file_directory)
        return dir_exists

    def sanitize_uploads(self, doc_list):
        """
        Flag prohibited file types and files over 400MB upload size limit.
        """
        for x in doc_list:
            file_split = x['name'].split('.')
            if file_split[-1].lower() in self.file_excludes:
                x['exclude_flag'] = True
                x['exclude_reason'] = 'Prohibited file type'
            elif os.path.getsize(x['full_path']) > 400000000:
                x['exclude_flag'] = True
                x['exclude_reason'] = 'File over 400MB'
            else:
                x['exclude_flag'] = False
        return doc_list

    def timestamp(self):
        self.time = time.strftime("%Y%m%dT%H%M%S")
        return self.time

{'fraction_non_alphanumeric': 0.11615439599714081, 'fraction_numerical': 0.012151536812008578, 'mean_word_length': 1.5585009140767825, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3468762', 'n_tokens_mistral': 886, 'n_tokens_neox': 811, 'n_words': 203}