<commit_before>#!/usr/bin/env python

import copy
from cStringIO import StringIO
from fnmatch import fnmatch
import gzip
import hashlib
import mimetypes
import os

import boto
from boto.s3.key import Key

import app_config

GZIP_FILE_TYPES = ['.html', '.js', '.json', '.css', '.xml']

class FakeTime:
    def time(self):
        return 1261130520.0

# Hack to override gzip's time implementation
# See: http://stackoverflow.com/questions/264224/setting-the-gzip-timestamp-from-python
gzip.time = FakeTime()

def deploy_file(connection, src, dst, headers={}):
    """
    Deploy a single file to S3, if the local version is different.
    """
    bucket = connection.get_bucket(app_config.S3_BUCKET['bucket_name'])

    k = bucket.get_key(dst)
    s3_md5 = None

    if k:
        s3_md5 = k.etag.strip('"')
    else:
        k = Key(bucket)
        k.key = dst

    file_headers = copy.copy(headers)

    if 'Content-Type' not in headers:
        file_headers['Content-Type'] = mimetypes.guess_type(src)[0],

    # Gzip file
    if os.path.splitext(src)[1].lower() in GZIP_FILE_TYPES:
        file_headers['Content-Encoding'] = 'gzip'

        with open(src, 'rb') as f_in:
            contents = f_in.read()

        output = StringIO()<commit_msg>backport mimetype fix, thx @tylrfishr<commit_after>#!/usr/bin/env python

import copy
from cStringIO import StringIO
from fnmatch import fnmatch
import gzip
import hashlib
import mimetypes
import os

import boto
from boto.s3.key import Key

import app_config

GZIP_FILE_TYPES = ['.html', '.js', '.json', '.css', '.xml']

class FakeTime:
    def time(self):
        return 1261130520.0

# Hack to override gzip's time implementation
# See: http://stackoverflow.com/questions/264224/setting-the-gzip-timestamp-from-python
gzip.time = FakeTime()

def deploy_file(connection, src, dst, headers={}):
    """
    Deploy a single file to S3, if the local version is different.
    """
    bucket = connection.get_bucket(app_config.S3_BUCKET['bucket_name'])

    k = bucket.get_key(dst)
    s3_md5 = None

    if k:
        s3_md5 = k.etag.strip('"')
    else:
        k = Key(bucket)
        k.key = dst

    file_headers = copy.copy(headers)

    if 'Content-Type' not in headers:
        file_headers['Content-Type'] = mimetypes.guess_type(src)[0]

    # Gzip file
    if os.path.splitext(src)[1].lower() in GZIP_FILE_TYPES:
        file_headers['Content-Encoding'] = 'gzip'

        with open(src, 'rb') as f_in:
            contents = f_in.read()

        output = StringIO()
{'fraction_non_alphanumeric': 0.12336374454581515, 'fraction_numerical': 0.020626735422451407, 'mean_word_length': 3.087520259319287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '178808', 'n_tokens_mistral': 913, 'n_tokens_neox': 872, 'n_words': 219}