<commit_before>from portality.core import app
from portality.lib import plugin

import os, shutil, boto3
from urllib.parse import quote_plus
from boto3.s3.transfer import TransferConfig

class StoreException(Exception):
    pass

class StoreFactory(object):

    @classmethod
    def get(cls, scope):
        """
        Returns an implementation of the base Store class
        """
        si = app.config.get("STORE_IMPL")
        sm = plugin.load_class(si)
        return sm(scope)

    @classmethod
    def tmp(cls):
        """
        Returns an implementation of the base Store class which should be able
        to provide local temp storage to the app.  In addition to the methods supplied
        by Store, it must also provide a "path" function to give the path on-disk to
        the file
        """
        si = app.config.get("STORE_TMP_IMPL")
        sm = plugin.load_class(si)
        return sm()

class Store(object):
    """
    ~~FileStore:Feature~~
    """
    def __init__(self, scope):
        pass

    def store(self, container_id, target_name, source_path=None, source_stream=None):
        pass

    def exists(self, container_id):
        return False

    def list(self, container_id):
        pass

    def get(self, container_id, target_name, encoding=None):
        return None

    def url(self, container_id, target_name):
        pass

    def delete_container(self, container_id):
        pass

    def delete_file(self, container_id, target_name):
        pass


class StoreS3(Store):
    """
    Primitive local storage system.  Use this for testing in place of remote store
    ~~->FileStoreS3:Feature~~
    ~~!FileStoreS3:Feature->S3:Technology~~
    """
    def __init__(self, scope):
        cfg = app.config.get("STORE_S3_SCOPES", {}).get(scope)
        multipart_threshold = app.config.get("STORE_S3_MULTIPART_THRESHOLD", 5 * 1024**3)
        access_key = cfg.get("aws_access_key_id")
        secret = cfg.get("aws_secret_access_key")
        if access_key is None or secret is None:
            raise StoreException("'aws_access_key_id' and 'aws_secret_access_key' must be set in STORE_S3_SCOPE for scope '{x}'".format(x=scope))

        self.client = boto3.client(
            's3',
            aws_access_key_id=access_key,
            aws_secret_access_key=secret
        )
        self.config = TransferConfig(multipart_threshold=multipart_threshold)

    def store(self, container_id, target_name, source_path=None, source_stream=None):
        # Note that this assumes the container (bucket) exists
        if source_path is not None:
            with open(source_path, "rb") as f:
                self.client.upload_fileobj(f, Bucket=container_id, Key=target_name, Config=self.config)
        elif source_stream is not None:
            self.client.upload_fileobj(source_stream, Bucket=container_id, Key=target_name, Config=self.config)

    def exists(self, container_id):
        pass

    def list(self, container_id):
        all_keys = []
        r = self.client.list_objects_v2(Bucket=container_id)
        while r.get('Contents', None):
            all_keys += [key["Key"] for key in r['Contents']]

            if r.get('NextContinuationToken', None):
                r = self.client.list_objects_v2(Bucket=container_id, ContinuationToken=r['NextContinuationToken'])
            else:
                break
        return all_keys

    def get(self, container_id, target_name, encoding=None):
        try:
            obj = self.client.get_object(Bucket=container_id, Key=target_name)
        except self.client.exceptions.NoSuchKey:
            return None<commit_msg>Don't use threads on multipart uploads https://github.com/DOAJ/doajPM/issues/3223<commit_after>from portality.core import app
from portality.lib import plugin

import os, shutil, boto3
from urllib.parse import quote_plus
from boto3.s3.transfer import TransferConfig

class StoreException(Exception):
    pass

class StoreFactory(object):

    @classmethod
    def get(cls, scope):
        """
        Returns an implementation of the base Store class
        """
        si = app.config.get("STORE_IMPL")
        sm = plugin.load_class(si)
        return sm(scope)

    @classmethod
    def tmp(cls):
        """
        Returns an implementation of the base Store class which should be able
        to provide local temp storage to the app.  In addition to the methods supplied
        by Store, it must also provide a "path" function to give the path on-disk to
        the file
        """
        si = app.config.get("STORE_TMP_IMPL")
        sm = plugin.load_class(si)
        return sm()

class Store(object):
    """
    ~~FileStore:Feature~~
    """
    def __init__(self, scope):
        pass

    def store(self, container_id, target_name, source_path=None, source_stream=None):
        pass

    def exists(self, container_id):
        return False

    def list(self, container_id):
        pass

    def get(self, container_id, target_name, encoding=None):
        return None

    def url(self, container_id, target_name):
        pass

    def delete_container(self, container_id):
        pass

    def delete_file(self, container_id, target_name):
        pass


class StoreS3(Store):
    """
    Primitive local storage system.  Use this for testing in place of remote store
    ~~->FileStoreS3:Feature~~
    ~~!FileStoreS3:Feature->S3:Technology~~
    """
    def __init__(self, scope):
        cfg = app.config.get("STORE_S3_SCOPES", {}).get(scope)
        multipart_threshold = app.config.get("STORE_S3_MULTIPART_THRESHOLD", 5 * 1024**3)
        access_key = cfg.get("aws_access_key_id")
        secret = cfg.get("aws_secret_access_key")
        if access_key is None or secret is None:
            raise StoreException("'aws_access_key_id' and 'aws_secret_access_key' must be set in STORE_S3_SCOPE for scope '{x}'".format(x=scope))

        self.client = boto3.client(
            's3',
            aws_access_key_id=access_key,
            aws_secret_access_key=secret
        )
        self.config = TransferConfig(multipart_threshold=multipart_threshold, use_threads=False)

    def store(self, container_id, target_name, source_path=None, source_stream=None):
        # Note that this assumes the container (bucket) exists
        if source_path is not None:
            with open(source_path, "rb") as f:
                self.client.upload_fileobj(f, Bucket=container_id, Key=target_name, Config=self.config)
        elif source_stream is not None:
            self.client.upload_fileobj(source_stream, Bucket=container_id, Key=target_name, Config=self.config)

    def exists(self, container_id):
        pass

    def list(self, container_id):
        all_keys = []
        r = self.client.list_objects_v2(Bucket=container_id)
        while r.get('Contents', None):
            all_keys += [key["Key"] for key in r['Contents']]

            if r.get('NextContinuationToken', None):
                r = self.client.list_objects_v2(Bucket=container_id, ContinuationToken=r['NextContinuationToken'])
            else:
                break
        return all_keys

    def get(self, container_id, target_name, encoding=None):
        try:
            obj = self.client.get_object(Bucket=container_id, Key=target_name)
        except self.client.exceptions.NoSuchKey:
            return None
{'fraction_non_alphanumeric': 0.0943498978897209, 'fraction_numerical': 0.005990469707283867, 'mean_word_length': 2.715730905412241, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '612388', 'n_tokens_mistral': 2220, 'n_tokens_neox': 2157, 'n_words': 558}