<commit_before>"""
An in-memory implementation of the S3 client interface as an aid
to unit testing.
"""

__all__ = [
    "MemoryS3",
]

from datetime import datetime

import attr

from dateutil.tz import tzutc

from twisted.internet.defer import succeed, fail

from txaws.s3.model import Bucket, BucketListing, BucketItem
from txaws.s3.exception import S3Error
from txaws.testing.base import MemoryClient, MemoryService

def _rate_limited(f):
    def g(self, *a, **kw):
        if self._state.get_rate_limit_exceeded():
            return fail(S3Error("<slowdown/>", 400))
        return f(self, *a, **kw)
    return g


class MemoryS3(MemoryService):
    """
    ``MemoryS3`` is a factory for new S3 clients.
    """
    def __init__(self):
        super(MemoryS3, self).__init__(
            client_factory=_MemoryS3Client,
            state_factory=S3ClientState,
        )

class S3ClientState(object):
    """
    ``S3ClientState`` instances hold the ``_MemoryS3Client`` instance
    state that is specific to testing and does not exist on
    ``txaws.s3.S3Client`` instances.
    """
    from time import time

    rate_limit_exceeded = False

    def __init__(self):
        self.buckets = {}
        self.objects = {}

    def set_rate_limit_exceeded(self):
        self.rate_limit_exceeded = True

    def clear_rate_limit_exceeded(self):
        self.rate_limit_exceeded = False

    def get_rate_limit_exceeded(self,):
        return self.rate_limit_exceeded


class _MemoryS3Client(MemoryClient):
    def __init__(self, controller, creds, endpoint):
        super(_MemoryS3Client, self).__init__(controller=controller, creds=creds)
        self.endpoint = endpoint

    @_rate_limited
    def list_buckets(self):
        return succeed(list(
            item["bucket"]
            for item
            in self._state.buckets.itervalues()
        ))

    @_rate_limited
    def create_bucket(self, bucket):
        assert bucket not in self._state.buckets
        self._state.buckets[bucket] = dict(
            bucket=Bucket(bucket, self._state.time()),
            listing=BucketListing(bucket, None, None, None, False),
        )
        return succeed(None)

    @_rate_limited
    def delete_bucket(self, bucket):
        if self._state.buckets[bucket]["listing"].contents:
            return fail(S3Error("<notempty/>", 400))
        del self._state.buckets[bucket]
        return succeed(None)

    @_rate_limited
    def get_bucket(self, bucket):
        try:
            pieces = self._state.buckets[bucket]
        except KeyError:
            return fail(S3Error("<nosuchbucket/>", 400))
        return succeed(pieces["listing"])

    @_rate_limited
    def get_bucket_location(self, bucket):<commit_msg>implement prefix for the in-memory double<commit_after>"""
An in-memory implementation of the S3 client interface as an aid
to unit testing.
"""

__all__ = [
    "MemoryS3",
]

from datetime import datetime

import attr

from dateutil.tz import tzutc

from twisted.internet.defer import succeed, fail

from txaws.s3.model import Bucket, BucketListing, BucketItem
from txaws.s3.exception import S3Error
from txaws.testing.base import MemoryClient, MemoryService

def _rate_limited(f):
    def g(self, *a, **kw):
        if self._state.get_rate_limit_exceeded():
            return fail(S3Error("<slowdown/>", 400))
        return f(self, *a, **kw)
    return g


class MemoryS3(MemoryService):
    """
    ``MemoryS3`` is a factory for new S3 clients.
    """
    def __init__(self):
        super(MemoryS3, self).__init__(
            client_factory=_MemoryS3Client,
            state_factory=S3ClientState,
        )

class S3ClientState(object):
    """
    ``S3ClientState`` instances hold the ``_MemoryS3Client`` instance
    state that is specific to testing and does not exist on
    ``txaws.s3.S3Client`` instances.

    @ivar buckets: A ``dict`` mapping bucket identifiers to ``dict`` of
        ``Bucket`` and ``BucketListing`` details.
    """
    from time import time

    rate_limit_exceeded = False

    def __init__(self):
        self.buckets = {}
        self.objects = {}

    def set_rate_limit_exceeded(self):
        self.rate_limit_exceeded = True

    def clear_rate_limit_exceeded(self):
        self.rate_limit_exceeded = False

    def get_rate_limit_exceeded(self,):
        return self.rate_limit_exceeded


class _MemoryS3Client(MemoryClient):
    def __init__(self, controller, creds, endpoint):
        super(_MemoryS3Client, self).__init__(controller=controller, creds=creds)
        self.endpoint = endpoint

    @_rate_limited
    def list_buckets(self):
        return succeed(list(
            item["bucket"]
            for item
            in self._state.buckets.itervalues()
        ))

    @_rate_limited
    def create_bucket(self, bucket):
        assert bucket not in self._state.buckets
        self._state.buckets[bucket] = dict(
            bucket=Bucket(bucket, self._state.time()),
            listing=BucketListing(bucket, None, None, None, False),
        )
        return succeed(None)

    @_rate_limited
    def delete_bucket(self, bucket):
        if self._state.buckets[bucket]["listing"].contents:
            return fail(S3Error("<notempty/>", 400))
        del self._state.buckets[bucket]
        return succeed(None)

    @_rate_limited
    def get_bucket(self, bucket, prefix=None):
        try:
            pieces = self._state.buckets[bucket]
        except KeyError:
            return fail(S3Error("<nosuchbucket/>", 400))
        listing = pieces["listing"]
        if prefix is not None:
            listing = attr.assoc(
                listing,
                contents=list(
                    content
                    for content
                    in listing.contents
                    if content.key.startswith(prefix)
                ),
                prefix=prefix,
            )
        return succeed(listing)

    @_rate_limited
    def get_bucket_location(self, bucket):
{'fraction_non_alphanumeric': 0.0986127360855758, 'fraction_numerical': 0.010028413839211098, 'mean_word_length': 2.5725373134328358, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5789859', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1737, 'n_words': 430}