<commit_before>import os
import six
import logging
import tempfile

from boto3.session import Session
from botocore.exceptions import ClientError

from aleph.archive.archive import Archive
from aleph.util import make_filename

log = logging.getLogger(__name__)


class S3Archive(Archive):  # pragma: no cover

    def __init__(self, config):
        self.local_base = six.text_type(tempfile.gettempdir())
        self.key_id = config.get('ARCHIVE_AWS_KEY_ID')
        self.secret = config.get('ARCHIVE_AWS_SECRET')
        self.region = config.get('ARCHIVE_AWS_REGION', 'eu-west-1')
        self.bucket_name = config.get('ARCHIVE_BUCKET')

        self.session = Session(aws_access_key_id=self.key_id,
                               aws_secret_access_key=self.secret,
                               region_name=self.region)
        self.s3 = self.session.resource('s3')
        self.client = self.session.client('s3')
        log.info("Using archive: s3://%s", self.bucket_name)
        self.bucket = self.s3.Bucket(self.bucket_name)

        try:
            self.bucket.load()
        except ClientError as e:
            error_code = int(e.response['Error']['Code'])
            if error_code == 404:
                self.bucket.create(CreateBucketConfiguration={
                    'LocationConstraint': self.region
                })
            else:
                raise

    def upgrade(self):
        """Make sure bucket policy is set correctly."""
        cors = self.bucket.Cors()
        config = {
            'CORSRules': [
                {
                    'AllowedMethods': ['GET'],
                    'AllowedOrigins': ['*'],
                    'AllowedHeaders': ['*'],
                    'MaxAgeSeconds': 84600 * 14
                }
            ]
        }
        cors.put(CORSConfiguration=config)

    def _locate_key(self, meta):
        key = self._get_file_path(meta)
        prefix = os.path.dirname(key)
        for obj in self.bucket.objects.filter(MaxKeys=1, Prefix=prefix):
            return obj

    def archive_file(self, filename, meta, move=False):
        meta = self._update_metadata(filename, meta)
        path = self._get_file_path(meta)
        obj = self._locate_key(meta)
        if obj is None:
            self.bucket.upload_file(filename, path)
<commit_msg>expose headers in s3<commit_after>import os
import six
import logging
import tempfile

from boto3.session import Session
from botocore.exceptions import ClientError

from aleph.archive.archive import Archive
from aleph.util import make_filename

log = logging.getLogger(__name__)


class S3Archive(Archive):  # pragma: no cover

    def __init__(self, config):
        self.local_base = six.text_type(tempfile.gettempdir())
        self.key_id = config.get('ARCHIVE_AWS_KEY_ID')
        self.secret = config.get('ARCHIVE_AWS_SECRET')
        self.region = config.get('ARCHIVE_AWS_REGION', 'eu-west-1')
        self.bucket_name = config.get('ARCHIVE_BUCKET')

        self.session = Session(aws_access_key_id=self.key_id,
                               aws_secret_access_key=self.secret,
                               region_name=self.region)
        self.s3 = self.session.resource('s3')
        self.client = self.session.client('s3')
        log.info("Using archive: s3://%s", self.bucket_name)
        self.bucket = self.s3.Bucket(self.bucket_name)

        try:
            self.bucket.load()
        except ClientError as e:
            error_code = int(e.response['Error']['Code'])
            if error_code == 404:
                self.bucket.create(CreateBucketConfiguration={
                    'LocationConstraint': self.region
                })
            else:
                raise

    def upgrade(self):
        """Make sure bucket policy is set correctly."""
        cors = self.bucket.Cors()
        config = {
            'CORSRules': [
                {
                    'AllowedMethods': ['GET'],
                    'AllowedOrigins': ['*'],
                    'AllowedHeaders': ['*'],
                    'ExposedHeaders': ['Accept-Ranges', 'Content-Encoding',
                                       'Content-Length', 'Content-Range'],
                    'MaxAgeSeconds': 84600 * 14
                }
            ]
        }
        cors.put(CORSConfiguration=config)

    def _locate_key(self, meta):
        key = self._get_file_path(meta)
        prefix = os.path.dirname(key)
        for obj in self.bucket.objects.filter(MaxKeys=1, Prefix=prefix):
            return obj

    def archive_file(self, filename, meta, move=False):
        meta = self._update_metadata(filename, meta)
        path = self._get_file_path(meta)
        obj = self._locate_key(meta)
        if obj is None:
            self.bucket.upload_file(filename, path)

{'fraction_non_alphanumeric': 0.10853199498117942, 'fraction_numerical': 0.008155583437892095, 'mean_word_length': 1.952469135802469, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6468168', 'n_tokens_mistral': 1409, 'n_tokens_neox': 1356, 'n_words': 264}