<commit_before>"""
AWS S3 operations
"""
from io import BytesIO
import minio   # https://docs.minio.io/docs/python-client-api-reference
import warnings
import itertools


class Folder:
    """
    A Folder instance manipulates a flat folder of objects within an S3-compatible object store
    """
    def __init__(self, endpoint, bucket, access_key, secret_key, location, database, **_):
        self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=False)
        self.bucket = bucket
        self.remote_path = '/'.join((location.lstrip('/'), database))

    def make_bucket(self):
        self.client.make_bucket(self.bucket)

    def put(self, blob_hash, blob):
        try:
            self.client.put_object(self.bucket, '/'.join((self.remote_path, blob_hash)), BytesIO(blob), len(blob))
        except minio.error.NoSuchBucket:
            warnings.warn('Creating bucket "%s"' % self.bucket)
            self.client.make_bucket(self.bucket)
            self.put(blob_hash, blob)

    def get(self, blob_hash):
        return self.client.get_object(self.bucket, '/'.join((self.remote_path, blob_hash))).data

    def clean(self, except_list, max_count=None):
        """
        Delete all objects except for those in the except_list
        :param except_list: a list of blob_hashes to skip.
        :param max_count: maximum number of object to delete
        :return: a generator of objects that failed to delete
        """
        return self.client.remove_objects(itertools.islice(
            (x for x in self.client.list_object(self.bucket, self.client.remote_path + '/') if x not in except_list),
            max_count))<commit_msg>rename except_list into exclude in s3.Folder.clean<commit_after>"""
AWS S3 operations
"""
from io import BytesIO
import minio   # https://docs.minio.io/docs/python-client-api-reference
import warnings
import itertools


class Folder:
    """
    A Folder instance manipulates a flat folder of objects within an S3-compatible object store
    """
    def __init__(self, endpoint, bucket, access_key, secret_key, location, database, **_):
        self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=False)
        self.bucket = bucket
        self.remote_path = '/'.join((location.lstrip('/'), database))

    def make_bucket(self):
        self.client.make_bucket(self.bucket)

    def put(self, blob_hash, blob):
        try:
            self.client.put_object(self.bucket, '/'.join((self.remote_path, blob_hash)), BytesIO(blob), len(blob))
        except minio.error.NoSuchBucket:
            warnings.warn('Creating bucket "%s"' % self.bucket)
            self.client.make_bucket(self.bucket)
            self.put(blob_hash, blob)

    def get(self, blob_hash):
        return self.client.get_object(self.bucket, '/'.join((self.remote_path, blob_hash))).data

    def clean(self, exclude, max_count=None):
        """
        Delete all objects except for those in the exclude
        :param exclude: a list of blob_hashes to skip.
        :param max_count: maximum number of object to delete
        :return: generator of objects that failed to delete
        """
        return self.client.remove_objects(itertools.islice(
            (x for x in self.client.list_object(self.bucket, self.client.remote_path + '/') if x not in exclude),
            max_count))
{'fraction_non_alphanumeric': 0.11291752882057346, 'fraction_numerical': 0.0014779781259237363, 'mean_word_length': 3.01423487544484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7171201', 'n_tokens_mistral': 998, 'n_tokens_neox': 988, 'n_words': 263}