<commit_before>"""
AWS S3 operations
"""
from io import BytesIO
import minio   # https://docs.minio.io/docs/python-client-api-reference
import warnings
import uuid
from pathlib import Path
from . import errors


class Folder:
    """
    A Folder instance manipulates a flat folder of objects within an S3-compatible object store
    """
    def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False, **_):
        self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key,
                                  secure=secure)
        self.bucket = bucket
        if not self.client.bucket_exists(bucket):
            raise errors.BucketInaccessible('Inaccessible s3 bucket %s' % bucket) from None

    def put(self, name, buffer):
        return self.client.put_object(
            self.bucket, str(name), BytesIO(buffer), length=len(buffer))

    def fput(self, local_file, name, metadata=None):
        return self.client.fput_object(
            self.bucket, str(name), str(local_file), metadata=metadata)

    def get(self, name):
        try:
            return self.client.get_object(self.bucket, str(name)).data
        except minio.error.NoSuchKey:
            raise errors.MissingExternalFile('Missing s3 key %s' % name) from None

    def fget(self, name, local_filepath):
        """get file from object name to local filepath"""
        name = str(name)
        stat = self.client.stat_object(self.bucket, name)
        meta = {k.lower().lstrip('x-amz-meta'): v for k, v in stat.metadata.items()}
        data = self.client.get_object(self.bucket, name)
        local_filepath = Path(local_filepath)
        local_filepath.parent.mkdir(parents=True, exist_ok=True)
        with local_filepath.open('wb') as f:
            for d in data.stream(1 << 16):
                f.write(d)<commit_msg>[wip] try for gwdg hpc<commit_after>"""
AWS S3 operations
"""
from io import BytesIO
import minio   # https://docs.minio.io/docs/python-client-api-reference
import urllib3
import warnings
import uuid
from pathlib import Path
from . import errors


class Folder:
    """
    A Folder instance manipulates a flat folder of objects within an S3-compatible object store
    """
    def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False, proxy_server=None, **_):
        # proxy_server: string, like "https://PROXYSERVER:PROXYPORT/"
        # self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key,
        #                           secure=secure)
        # TODO implement proxy_server argument
        http_client = urllib3.ProxyManager(
                "http://www-cache.gwdg.de:3128",
                timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
                cert_reqs="CERT_REQUIRED",
                retries=urllib3.Retry(
                    total=5,
                    backoff_factor=0.2,
                    status_forcelist=[500, 502, 503, 504],
                )
            )
        self.client = minio.Minio(
            endpoint,
            access_key=access_key,
            secret_key=secret_key,
            secure=secure,
            http_client=http_client,
            )
        self.bucket = bucket
        if not self.client.bucket_exists(bucket):
            raise errors.BucketInaccessible('Inaccessible s3 bucket %s' % bucket) from None

    def put(self, name, buffer):
        return self.client.put_object(
            self.bucket, str(name), BytesIO(buffer), length=len(buffer))

    def fput(self, local_file, name, metadata=None):
        return self.client.fput_object(
            self.bucket, str(name), str(local_file), metadata=metadata)

    def get(self, name):
        try:
            return self.client.get_object(self.bucket, str(name)).data
        except minio.error.NoSuchKey:
            raise errors.MissingExternalFile('Missing s3 key %s' % name) from None

    def fget(self, name, local_filepath):
        """get file from object name to local filepath"""
        name = str(name)
        stat = self.client.stat_object(self.bucket, name)
        meta = {k.lower().lstrip('x-amz-meta'): v for k, v in stat.metadata.items()}
        data = self.client.get_object(self.bucket, name)
        local_filepath = Path(local_filepath)
        local_filepath.parent.mkdir(parents=True, exist_ok=True)
        with local_filepath.open('wb') as f:
            for d in data.stream(1 << 16):
                f.write(d)
{'fraction_non_alphanumeric': 0.10731158605174353, 'fraction_numerical': 0.008323959505061868, 'mean_word_length': 2.4438419829589466, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1303032', 'n_tokens_mistral': 1344, 'n_tokens_neox': 1307, 'n_words': 319}