<commit_before>import os
import hashlib
import httplib
import requests
import json
import urllib
import urlparse
import tempfile
from datetime import datetime
from ckan.lib.celery_app import celery
from webstore_upload import upload_content
from webstore_upload import DATA_FORMATS as WEBSTORE_DATA_FORMATS

try:
    from ckanext.archiver import settings
except ImportError:
    from ckanext.archiver import default_settings as settings

HTTP_ERROR_CODES = {
    httplib.MULTIPLE_CHOICES: "300 Multiple Choices not implemented",
    httplib.USE_PROXY: "305 Use Proxy not implemented",
    httplib.INTERNAL_SERVER_ERROR: "Internal server error on the remote server",
    httplib.BAD_GATEWAY: "Bad gateway",
    httplib.SERVICE_UNAVAILABLE: "Service unavailable",
    httplib.GATEWAY_TIMEOUT: "Gateway timeout",
    httplib.METHOD_NOT_ALLOWED: "405 Method Not Allowed"
}

DATA_FORMATS = [ 
    'csv',
    'text/csv',
    'txt',
    'text/plain',
    'text/html',
    'html',
    'rdf',
    'text/rdf',
    'xml',
    'xls',
    'application/ms-excel',
    'application/vnd.ms-excel',    
    'application/xls',
    'text/xml',
    'tar',
    'application/x-tar',
    'zip',
    'application/zip'
    'gz',
    'application/gzip',
    'application/x-gzip',
    'application/octet-stream'
]

class ArchiverError(Exception):
    pass
class DownloadError(Exception):
    pass
class LinkCheckerError(Exception):
    pass
class CkanError(Exception):
    pass


def download(context, resource, url_timeout=30,
             max_content_length=settings.MAX_CONTENT_LENGTH,
             data_formats=DATA_FORMATS):

    link_context = "{}"
    link_data = json.dumps({
        'url': resource['url'],
        'url_timeout': url_timeout
    })
    headers = json.loads(link_checker(link_context, link_data))

    resource_format = resource['format'].lower()
    ct = headers.get('content-type', '').lower()
    cl = headers.get('content-length')

    resource_changed = (resource.get('mimetype') != ct) or (resource.get('size') != cl)
    if resource_changed:
        resource['mimetype'] = ct
        resource['size'] = cl

    # make sure resource content-length does not exceed our maximum
    if cl and int(cl) >= max_content_length:
        if resource_changed: 
            _update_resource(context, resource) 
        # record fact that resource is too large to archive
        raise DownloadError("Content-length %s exceeds maximum allowed value %s" %
            (cl, max_content_length))                                                      

    # check that resource is a data file
    if not (resource_format in data_formats or ct.lower() in data_formats):
        if resource_changed: 
            _update_resource(context, resource) 
        raise DownloadError("Of content type %s, not downloading" % ct) 

    # get the resource and archive it
    res = requests.get(resource['url'], timeout = url_timeout)
    length, hash, saved_file = _save_resource(resource, res, max_content_length)

    # check that resource did not exceed maximum size when being saved
    # (content-length header could have been invalid/corrupted, or not accurate
    # if resource was streamed)
    #
    # TODO: remove partially archived file in this case
    if length >= max_content_length:
        if resource_changed: 
            _update_resource(context, resource) 
        # record fact that resource is too large to archive
        raise DownloadError("Content-length after streaming reached maximum allowed value of %s" % 
            max_content_length) 

    # update the resource metadata in CKAN if the resource has changed
    if resource.get('hash') != hash:
        resource['hash'] = hash
        try:
            # This may fail for archiver.update() as a result of the resource
            # not yet existing.
            _update_resource(context, resource)
        except:
            pass
        

    return {'length': length,
            'hash' : hash,
            'headers': headers,
            'saved_file': saved_file}


@celery.task(name = "archiver.clean")
def clean():
    """
    Remove all archived resources.
    """
    logger = clean.get_logger()
    logger.error("clean task not implemented yet")

@celery.task(name = "archiver.update")
def update(context, data):
    try:
        data = json.loads(data)
        context = json.loads(context)
        result = _update(context, data) 
        # Decide whether to trigger the upload task here in response to the update
        return result<commit_msg>Make sure we do the content type comparison in lower case so it has at least a fighting change of matching<commit_after>import os
import hashlib
import httplib
import requests
import json
import urllib
import urlparse
import tempfile
from datetime import datetime
from ckan.lib.celery_app import celery
from webstore_upload import upload_content
from webstore_upload import DATA_FORMATS as WEBSTORE_DATA_FORMATS

try:
    from ckanext.archiver import settings
except ImportError:
    from ckanext.archiver import default_settings as settings

HTTP_ERROR_CODES = {
    httplib.MULTIPLE_CHOICES: "300 Multiple Choices not implemented",
    httplib.USE_PROXY: "305 Use Proxy not implemented",
    httplib.INTERNAL_SERVER_ERROR: "Internal server error on the remote server",
    httplib.BAD_GATEWAY: "Bad gateway",
    httplib.SERVICE_UNAVAILABLE: "Service unavailable",
    httplib.GATEWAY_TIMEOUT: "Gateway timeout",
    httplib.METHOD_NOT_ALLOWED: "405 Method Not Allowed"
}

DATA_FORMATS = [ 
    'csv',
    'text/csv',
    'txt',
    'text/plain',
    'text/html',
    'html',
    'rdf',
    'text/rdf',
    'xml',
    'xls',
    'application/ms-excel',
    'application/vnd.ms-excel',    
    'application/xls',
    'text/xml',
    'tar',
    'application/x-tar',
    'zip',
    'application/zip'
    'gz',
    'application/gzip',
    'application/x-gzip',
    'application/octet-stream'
]

class ArchiverError(Exception):
    pass
class DownloadError(Exception):
    pass
class LinkCheckerError(Exception):
    pass
class CkanError(Exception):
    pass


def download(context, resource, url_timeout=30,
             max_content_length=settings.MAX_CONTENT_LENGTH,
             data_formats=DATA_FORMATS):

    link_context = "{}"
    link_data = json.dumps({
        'url': resource['url'],
        'url_timeout': url_timeout
    })
    headers = json.loads(link_checker(link_context, link_data))

    resource_format = resource['format'].lower()
    ct = headers.get('content-type', '').lower()
    cl = headers.get('content-length')

    resource_changed = (resource.get('mimetype', '').lower() != ct) or (resource.get('size') != cl)
    if resource_changed:
        resource['mimetype'] = ct
        resource['size'] = cl

    # make sure resource content-length does not exceed our maximum
    if cl and int(cl) >= max_content_length:
        if resource_changed: 
            _update_resource(context, resource) 
        # record fact that resource is too large to archive
        raise DownloadError("Content-length %s exceeds maximum allowed value %s" %
            (cl, max_content_length))                                                      

    # check that resource is a data file
    if not (resource_format in data_formats or ct.lower() in data_formats):
        if resource_changed: 
            _update_resource(context, resource) 
        raise DownloadError("Of content type %s, not downloading" % ct) 

    # get the resource and archive it
    res = requests.get(resource['url'], timeout = url_timeout)
    length, hash, saved_file = _save_resource(resource, res, max_content_length)

    # check that resource did not exceed maximum size when being saved
    # (content-length header could have been invalid/corrupted, or not accurate
    # if resource was streamed)
    #
    # TODO: remove partially archived file in this case
    if length >= max_content_length:
        if resource_changed: 
            _update_resource(context, resource) 
        # record fact that resource is too large to archive
        raise DownloadError("Content-length after streaming reached maximum allowed value of %s" % 
            max_content_length) 

    # update the resource metadata in CKAN if the resource has changed
    if resource.get('hash') != hash:
        resource['hash'] = hash
        try:
            # This may fail for archiver.update() as a result of the resource
            # not yet existing.
            _update_resource(context, resource)
        except:
            pass
        

    return {'length': length,
            'hash' : hash,
            'headers': headers,
            'saved_file': saved_file}


@celery.task(name = "archiver.clean")
def clean():
    """
    Remove all archived resources.
    """
    logger = clean.get_logger()
    logger.error("clean task not implemented yet")

@celery.task(name = "archiver.update")
def update(context, data):
    try:
        data = json.loads(data)
        context = json.loads(context)
        result = _update(context, data) 
        return result
{'fraction_non_alphanumeric': 0.09370860927152318, 'fraction_numerical': 0.0024282560706401766, 'mean_word_length': 2.837780601440068, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6405178', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2579, 'n_words': 765}