<commit_before>import os
import cgi
import pylons
import logging
import datetime
import mimetypes

import boto3
import botocore

import ckan.model as model
import ckan.lib.munge as munge


config = pylons.config
log = logging.getLogger(__name__)

_storage_path = None
_max_resource_size = None
_max_image_size = None


class S3FileStoreException(Exception):
    pass


class BaseS3Uploader(object):

    def __init__(self):
        self.bucket_name = config.get('ckanext.s3filestore.aws_bucket_name')
        self.p_key = config.get('ckanext.s3filestore.aws_access_key_id')
        self.s_key = config.get('ckanext.s3filestore.aws_secret_access_key')
        self.region = config.get('ckanext.s3filestore.region_name')
        self.signature = config.get('ckanext.s3filestore.sigranure_version')
        self.host_name = config.get('ckanext.s3filestore.host_name')
        self.bucket = self.get_s3_bucket(self.bucket_name)

    def get_directory(self, id, storage_path):
        directory = os.path.join(storage_path, id)
        return directory

    def get_s3_bucket(self, bucket_name):
        '''Return a boto bucket, creating it if it doesn't exist.'''

        # make s3 connection using boto3
        session = boto3.session.Session(aws_access_key_id=self.p_key,
                                        aws_secret_access_key=self.s_key,
                                        region_name=self.region)
        s3 = session.resource('s3', endpoint_url=self.host_name,
                              config=botocore.client.Config(signature_version=self.signature))
        bucket = s3.Bucket(bucket_name)
        try:
            if s3.Bucket(bucket.name) in s3.buckets.all():
                log.info('Bucket {0} found!'.format(bucket_name))

            else:
                log.warning(
                    'Bucket {0} could not be found, attempting to create it...'.format(bucket_name))
                try:
                    bucket = s3.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={
                        'LocationConstraint': 'us-east-1'})
                    log.info(
                        'Bucket {0} succesfully created'.format(bucket_name))
                except botocore.exceptions.ClientError as e:
                    log.warning('Could not create bucket {0}: {1}'.format(
                        bucket_name, str(e)))
        except botocore.exceptions.ClientError as e:
            error_code = int(e.response['Error']['Code'])
            if error_code == 404:
                log.warning('Bucket {0} could not be found, ' +
                            'attempting to create it...'.format(bucket_name))
                try:
                    bucket = s3.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={
                        'LocationConstraint': self.region})
                    log.info(
                        'Bucket {0} succesfully created'.format(bucket_name))
                except botocore.exceptions.ClientError as e:
                    log.warning('Could not create bucket {0}: {1}'.format(
                        bucket_name, str(e)))
            elif error_code == 403:
                raise S3FileStoreException(
                    'Access to bucket {0} denied'.format(bucket_name))
            else:
                raise S3FileStoreException(
                    'Something went wrong for bucket {0}'.format(bucket_name))

        return bucket

    def upload_to_key(self, filepath, upload_file, make_public=False):
        '''Uploads the `upload_file` to `filepath` on `self.bucket`.'''
        upload_file.seek(0)
        content_type, x = mimetypes.guess_type(filepath)
        headers = {}
        if content_type:
            headers.update({'Content-Type': content_type})

            session = boto3.session.Session(aws_access_key_id=self.p_key,
                                            aws_secret_access_key=self.s_key,
                                            region_name=self.region)
            s3 = session.resource('s3', endpoint_url=self.host_name,
                                  config=botocore.client.Config(signature_version=self.signature))
            try:
                s3.Object(self.bucket_name, filepath).put(
                    Body=upload_file.read(), ACL='public-read')
                log.info("Succesfully uploaded {0} to S3!".format(filepath))
            except Exception as e:
                log.error(
                    'Something went very very wrong for {0}'.format(str(e)))
                raise e

    def clear_key(self, filepath):
        '''Deletes the contents of the key at `filepath` on `self.bucket`.'''
        session = boto3.session.Session(aws_access_key_id=self.p_key,
                                        aws_secret_access_key=self.s_key,
                                        region_name=self.region)
        s3 = session.resource('s3', endpoint_url=self.host_name, config=botocore.client.Config(
                signature_version=self.signature))
        try:
            s3.Object(self.bucket_name, filepath).delete()
        except Exception as e:
                raise e


class S3Uploader(BaseS3Uploader):

    '''
    An uploader class to replace local file storage with Amazon Web Services
    S3 for general files (e.g. Group cover images).
    '''

    def __init__(self, upload_to, old_filename=None):
        '''Setup the uploader. Additional setup is performed by
        update_data_dict(), and actual uploading performed by `upload()`.

        Create a storage path in the format:
        <ckanext.s3filestore.aws_storage_path>/storage/uploads/<upload_to>/
        '''
<commit_msg>content type check removed when uploading resources<commit_after>import os
import cgi
import pylons
import logging
import datetime
import mimetypes

import boto3
import botocore

import ckan.model as model
import ckan.lib.munge as munge


config = pylons.config
log = logging.getLogger(__name__)

_storage_path = None
_max_resource_size = None
_max_image_size = None


class S3FileStoreException(Exception):
    pass


class BaseS3Uploader(object):

    def __init__(self):
        self.bucket_name = config.get('ckanext.s3filestore.aws_bucket_name')
        self.p_key = config.get('ckanext.s3filestore.aws_access_key_id')
        self.s_key = config.get('ckanext.s3filestore.aws_secret_access_key')
        self.region = config.get('ckanext.s3filestore.region_name')
        self.signature = config.get('ckanext.s3filestore.sigranure_version')
        self.host_name = config.get('ckanext.s3filestore.host_name')
        self.bucket = self.get_s3_bucket(self.bucket_name)

    def get_directory(self, id, storage_path):
        directory = os.path.join(storage_path, id)
        return directory

    def get_s3_bucket(self, bucket_name):
        '''Return a boto bucket, creating it if it doesn't exist.'''

        # make s3 connection using boto3
        session = boto3.session.Session(aws_access_key_id=self.p_key,
                                        aws_secret_access_key=self.s_key,
                                        region_name=self.region)
        s3 = session.resource('s3', endpoint_url=self.host_name,
                              config=botocore.client.Config(signature_version=self.signature))
        bucket = s3.Bucket(bucket_name)
        try:
            if s3.Bucket(bucket.name) in s3.buckets.all():
                log.info('Bucket {0} found!'.format(bucket_name))

            else:
                log.warning(
                    'Bucket {0} could not be found, attempting to create it...'.format(bucket_name))
                try:
                    bucket = s3.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={
                        'LocationConstraint': 'us-east-1'})
                    log.info(
                        'Bucket {0} succesfully created'.format(bucket_name))
                except botocore.exceptions.ClientError as e:
                    log.warning('Could not create bucket {0}: {1}'.format(
                        bucket_name, str(e)))
        except botocore.exceptions.ClientError as e:
            error_code = int(e.response['Error']['Code'])
            if error_code == 404:
                log.warning('Bucket {0} could not be found, ' +
                            'attempting to create it...'.format(bucket_name))
                try:
                    bucket = s3.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={
                        'LocationConstraint': self.region})
                    log.info(
                        'Bucket {0} succesfully created'.format(bucket_name))
                except botocore.exceptions.ClientError as e:
                    log.warning('Could not create bucket {0}: {1}'.format(
                        bucket_name, str(e)))
            elif error_code == 403:
                raise S3FileStoreException(
                    'Access to bucket {0} denied'.format(bucket_name))
            else:
                raise S3FileStoreException(
                    'Something went wrong for bucket {0}'.format(bucket_name))

        return bucket

    def upload_to_key(self, filepath, upload_file, make_public=False):
        '''Uploads the `upload_file` to `filepath` on `self.bucket`.'''
        upload_file.seek(0)

        session = boto3.session.Session(aws_access_key_id=self.p_key,
                                        aws_secret_access_key=self.s_key,
                                        region_name=self.region)
        s3 = session.resource('s3', endpoint_url=self.host_name,
                              config=botocore.client.Config(signature_version=self.signature))
        try:
            s3.Object(self.bucket_name, filepath).put(
                Body=upload_file.read(), ACL='public-read')
            log.info("Succesfully uploaded {0} to S3!".format(filepath))
        except Exception as e:
            log.error('Something went very very wrong for {0}'.format(str(e)))
            raise e

    def clear_key(self, filepath):
        '''Deletes the contents of the key at `filepath` on `self.bucket`.'''
        session = boto3.session.Session(aws_access_key_id=self.p_key,
                                    aws_secret_access_key=self.s_key,
                                    region_name=self.region)
        s3 = session.resource('s3', endpoint_url=self.host_name, config=botocore.client.Config(
                             signature_version=self.signature))
        try:
            s3.Object(self.bucket_name, filepath).delete()
        except Exception as e:
            raise e


class S3Uploader(BaseS3Uploader):

    '''
    An uploader class to replace local file storage with Amazon Web Services
    S3 for general files (e.g. Group cover images).
    '''

    def __init__(self, upload_to, old_filename=None):
        '''Setup the uploader. Additional setup is performed by
        update_data_dict(), and actual uploading performed by `upload()`.

        Create a storage path in the format:
        <ckanext.s3filestore.aws_storage_path>/storage/uploads/<upload_to>/
        '''

{'fraction_non_alphanumeric': 0.09575233981281497, 'fraction_numerical': 0.010259179265658747, 'mean_word_length': 1.9946106170843438, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '644186', 'n_tokens_mistral': 3165, 'n_tokens_neox': 3042, 'n_words': 659}