<commit_before>import datetime
import mimetypes
import posixpath
import urllib
from logging import getLogger
from time import mktime
from urllib.parse import urlparse

import minio
import minio.error as merr
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import Storage
from django.utils import timezone
from django.utils.deconstruct import deconstructible
from minio.helpers import get_target_url
from minio_storage import policy
from minio_storage.errors import minio_error
from minio_storage.files import ReadOnlySpooledTemporaryFile

logger = getLogger("minio_storage")


@deconstructible
class MinioStorage(Storage):
    """An implementation of Django's file storage using the minio client.

    The implementation should comply with
    https://docs.djangoproject.com/en/dev/ref/files/storage/.

    """

    file_class = ReadOnlySpooledTemporaryFile

    def __init__(
        self,
        minio_client,
        bucket_name,
        *,
        base_url=None,
        file_class=None,
        auto_create_bucket=False,
        presign_urls=False,
        auto_create_policy: False,
        backup_format=None,
        backup_bucket=None,
        **kwargs,
    ):
        self.client = minio_client
        self.bucket_name = bucket_name
        self.base_url = base_url

        self.backup_format = backup_format
        self.backup_bucket = backup_bucket
        if bool(self.backup_format) != bool(self.backup_bucket):
            raise ImproperlyConfigured(
                "To enable backups, make sure to set both backup format "
                "and backup format"
            )

        if file_class is not None:
            self.file_class = file_class
        self.auto_create_bucket = auto_create_bucket

        self.presign_urls = presign_urls

        if auto_create_bucket and not self.client.bucket_exists(self.bucket_name):

            self.client.make_bucket(self.bucket_name)

            if auto_create_policy:
                self.client.set_bucket_policy(
                    self.bucket_name, self._policy("READ_ONLY")
                )

        elif not self.client.bucket_exists(self.bucket_name):
            raise OSError(f"The bucket {bucket_name} does not exist")

        super().__init__()

    def _sanitize_path(self, name):
        v = posixpath.normpath(name).replace("\\", "/")
        if v == ".":
            v = ""
        if name.endswith("/") and not v.endswith("/"):
            v += "/"
        return v

    def _examine_file(self, name, content):
        """Examines a file and produces information necessary for upload.

        Returns a tuple of the form (content_size, content_type,
        sanitized_name)

        """
        content_size = content.size
        content_type = mimetypes.guess_type(name, strict=False)
        content_type = content_type[0] or "application/octet-stream"
        sane_name = self._sanitize_path(name)
        return (content_size, content_type, sane_name)

    def _open(self, name, mode="rb"):
        try:
            f = self.file_class(self._sanitize_path(name), mode, self)
        except merr.MinioError as e:
            raise minio_error("File {} could not be saved: {}".format(name, str(e)), e)
        return f

    def _policy(self, policy_type: str):
        """
        Dictionary containing basic AWS Policies in JSON format
        Policies: READ_ONLY, WRITE_ONLY, READ_WRITE
        """
        return policy.Policy(policy_type).bucket(self.bucket_name)

    def _save(self, name, content):
        # (str, bytes) -> str
        try:
            if hasattr(content, "seek") and callable(content.seek):
                content.seek(0)
            content_size, content_type, sane_name = self._examine_file(name, content)
            self.client.put_object(
                self.bucket_name, sane_name, content, content_size, content_type
            )
            return sane_name
        except merr.ResponseError as error:
            raise minio_error(f"File {name} could not be saved", error)

    def delete(self, name):
        # type: (str) -> None
        if self.backup_format and self.backup_bucket:
            try:
                obj = self.client.get_object(self.bucket_name, name)
            except merr.ResponseError as error:
                raise minio_error(
                    "Could not obtain file {} " "to make a copy of it".format(name),
                    error,
                )

            try:
                content_length = int(obj.getheader("Content-Length"))
            except ValueError as error:
                raise minio_error(f"Could not backup removed file {name}", error)

            # Creates the backup filename
            target_name = "{}{}".format(
                timezone.now().strftime(self.backup_format), name
            )
            try:
                self.client.put_object(
                    self.backup_bucket, target_name, obj, content_length
                )
            except merr.ResponseError as error:
                raise minio_error(
                    "Could not make a copy of file "
                    "{} before removing it".format(name),
                    error,
                )

        try:
            self.client.remove_object(self.bucket_name, name)
        except merr.ResponseError as error:
            raise minio_error(f"Could not remove file {name}", error)

    def exists(self, name):
        # type: (str) -> bool
        try:
            self.client.stat_object(self.bucket_name, self._sanitize_path(name))
            return True
        except merr.ResponseError as error:
            # TODO - deprecate
            if error.code == "NoSuchKey":
                return False
            else:
                raise minio_error(f"Could not stat file {name}", error)
        except merr.NoSuchKey:
            return False
        except merr.NoSuchBucket:
            raise
        except Exception as error:
            logger.error(error)

    def listdir(self, path):
        #  [None, "", "."] is supported to mean the configured root among various
        #  implementations of Storage implementations so we copy that behaviour even if
        #  maybe None should raise an exception instead.
        #
        #  If the path prefix does not match anything full prefix that does exist this
        #  function will just return empty results, this is different from
        #  FileSystemStorage where an invalid directory would raise an OSError.

        if path in [None, "", "."]:
            path = ""
        else:
            path += "/"

        dirs = []
        files = []
        try:
            objects = self.client.list_objects_v2(self.bucket_name, prefix=path)
            for o in objects:
                p = posixpath.relpath(o.object_name, path)
                if o.is_dir:
                    dirs.append(p)
                else:
                    files.append(p)
            return dirs, files
        except merr.NoSuchBucket:
            raise
        except merr.ResponseError as error:
            raise minio_error(f"Could not list directory {path}", error)

    def size(self, name):
        # type: (str) -> int
        try:
            info = self.client.stat_object(self.bucket_name, name)
            return info.size
        except merr.ResponseError as error:
            raise minio_error(f"Could not access file size for {name}", error)

    def url(self, name, *args, max_age=None):
        # type: (str) -> str
        kwargs = {}
        if max_age is not None:
            kwargs["expires"] = max_age

        # NOTE: Here be dragons, when a external base_url is used the code
        # below is both using "internal" minio clint APIs and somewhat
        # subverting how minio/S3 expects urls to be generated in the first
        # place.

        if self.presign_urls:
            url = self.client.presigned_get_object(self.bucket_name, name, **kwargs)
            if self.base_url is not None:
                parsed_url = urlparse(url)
                path = parsed_url.path.split(self.bucket_name, 1)[1]
                url = "{}{}?{}{}{}".format(
                    self.base_url,
                    path,
                    parsed_url.params,
                    parsed_url.query,
                    parsed_url.fragment,
                )

        else:
            if self.base_url is not None:

                def strip_beg(path):
                    while path.startswith("/"):
                        path = path[1:]
                    return path

                def strip_end(path):
                    while path.endswith("/"):
                        path = path[:-1]
                    return path

                url = "{}/{}".format(
                    strip_end(self.base_url), urllib.parse.quote(strip_beg(name))
                )
            else:
                url = get_target_url(
                    self.client._endpoint_url,
                    bucket_name=self.bucket_name,
                    object_name=name,
                    # bucket_region=region,
                )
        return url

    def accessed_time(self, name):
        # type: (str) -> datetime.datetime
        """
        Not available via the S3 API
        """
        return self.modified_time(name)

    def created_time(self, name):
        # type: (str) -> datetime.datetime
        """
        Not available via the S3 API
        """
        return self.modified_time(name)

    def modified_time(self, name):
        # type: (str) -> datetime.datetime
        try:
            info = self.client.stat_object(self.bucket_name, name)
            return datetime.datetime.fromtimestamp(mktime(info.last_modified))
        except merr.ResponseError as error:
            raise minio_error(
                f"Could not access modification time for file {name}", error
            )


_NoValue = object()


def get_setting(name, default=_NoValue):
    result = getattr(settings, name, default)
    if result is _NoValue:
        print("Attr {} : {}".format(name, getattr(settings, name, default)))
        raise ImproperlyConfigured
    else:
        return result


def create_minio_client_from_settings(*, minio_kwargs=dict()):
    endpoint = get_setting("MINIO_STORAGE_ENDPOINT")
    access_key = get_setting("MINIO_STORAGE_ACCESS_KEY")
    secret_key = get_setting("MINIO_STORAGE_SECRET_KEY")
    secure = get_setting("MINIO_STORAGE_USE_HTTPS", True)
    client = minio.Minio(
        endpoint,
        access_key=access_key,
        secret_key=secret_key,
        secure=secure,
        **minio_kwargs,
    )
    return client


@deconstructible
class MinioMediaStorage(MinioStorage):
    def __init__(self):
        client = create_minio_client_from_settings()
        bucket_name = get_setting("MINIO_STORAGE_MEDIA_BUCKET_NAME")
        base_url = get_setting("MINIO_STORAGE_MEDIA_URL", None)
        auto_create_bucket = get_setting(
            "MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET", False
        )
        auto_create_policy = get_setting(
            "MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY", False
        )
        presign_urls = get_setting("MINIO_STORAGE_MEDIA_USE_PRESIGNED", False)
        backup_format = get_setting("MINIO_STORAGE_MEDIA_BACKUP_FORMAT", False)
        backup_bucket = get_setting("MINIO_STORAGE_MEDIA_BACKUP_BUCKET", False)

        super().__init__(
            client,
            bucket_name,
            auto_create_bucket=auto_create_bucket,
            auto_create_policy=auto_create_policy,
            base_url=base_url,
            presign_urls=presign_urls,
            backup_format=backup_format,
            backup_bucket=backup_bucket,
        )


@deconstructible
class MinioStaticStorage(MinioStorage):
    def __init__(self):
        client = create_minio_client_from_settings()
        base_url = get_setting("MINIO_STORAGE_STATIC_URL", None)
        bucket_name = get_setting("MINIO_STORAGE_STATIC_BUCKET_NAME")
        auto_create_bucket = get_setting(
            "MINIO_STORAGE_AUTO_CREATE_STATIC_BUCKET", False
        )
        auto_create_policy = get_setting(
            "MINIO_STORAGE_AUTO_CREATE_STATIC_POLICY", False
        )

        presign_urls = get_setting("MINIO_STORAGE_STATIC_USE_PRESIGNED", False)

        super().__init__(
            client,
            bucket_name,
            auto_create_bucket=auto_create_bucket,
            auto_create_policy=auto_create_policy,
            base_url=base_url,
            presign_urls=presign_urls,
        )
<commit_msg>fix typo<commit_after>import datetime
import mimetypes
import posixpath
import urllib
from logging import getLogger
from time import mktime
from urllib.parse import urlparse

import minio
import minio.error as merr
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import Storage
from django.utils import timezone
from django.utils.deconstruct import deconstructible
from minio.helpers import get_target_url
from minio_storage import policy
from minio_storage.errors import minio_error
from minio_storage.files import ReadOnlySpooledTemporaryFile

logger = getLogger("minio_storage")


@deconstructible
class MinioStorage(Storage):
    """An implementation of Django's file storage using the minio client.

    The implementation should comply with
    https://docs.djangoproject.com/en/dev/ref/files/storage/.

    """

    file_class = ReadOnlySpooledTemporaryFile

    def __init__(
        self,
        minio_client,
        bucket_name,
        *,
        base_url=None,
        file_class=None,
        auto_create_bucket=False,
        presign_urls=False,
        auto_create_policy=False,
        backup_format=None,
        backup_bucket=None,
        **kwargs,
    ):
        self.client = minio_client
        self.bucket_name = bucket_name
        self.base_url = base_url

        self.backup_format = backup_format
        self.backup_bucket = backup_bucket
        if bool(self.backup_format) != bool(self.backup_bucket):
            raise ImproperlyConfigured(
                "To enable backups, make sure to set both backup format "
                "and backup format"
            )

        if file_class is not None:
            self.file_class = file_class
        self.auto_create_bucket = auto_create_bucket

        self.presign_urls = presign_urls

        if auto_create_bucket and not self.client.bucket_exists(self.bucket_name):

            self.client.make_bucket(self.bucket_name)

            if auto_create_policy:
                self.client.set_bucket_policy(
                    self.bucket_name, self._policy("READ_ONLY")
                )

        elif not self.client.bucket_exists(self.bucket_name):
            raise OSError(f"The bucket {bucket_name} does not exist")

        super().__init__()

    def _sanitize_path(self, name):
        v = posixpath.normpath(name).replace("\\", "/")
        if v == ".":
            v = ""
        if name.endswith("/") and not v.endswith("/"):
            v += "/"
        return v

    def _examine_file(self, name, content):
        """Examines a file and produces information necessary for upload.

        Returns a tuple of the form (content_size, content_type,
        sanitized_name)

        """
        content_size = content.size
        content_type = mimetypes.guess_type(name, strict=False)
        content_type = content_type[0] or "application/octet-stream"
        sane_name = self._sanitize_path(name)
        return (content_size, content_type, sane_name)

    def _open(self, name, mode="rb"):
        try:
            f = self.file_class(self._sanitize_path(name), mode, self)
        except merr.MinioError as e:
            raise minio_error("File {} could not be saved: {}".format(name, str(e)), e)
        return f

    def _policy(self, policy_type: str):
        """
        Dictionary containing basic AWS Policies in JSON format
        Policies: READ_ONLY, WRITE_ONLY, READ_WRITE
        """
        return policy.Policy(policy_type).bucket(self.bucket_name)

    def _save(self, name, content):
        # (str, bytes) -> str
        try:
            if hasattr(content, "seek") and callable(content.seek):
                content.seek(0)
            content_size, content_type, sane_name = self._examine_file(name, content)
            self.client.put_object(
                self.bucket_name, sane_name, content, content_size, content_type
            )
            return sane_name
        except merr.ResponseError as error:
            raise minio_error(f"File {name} could not be saved", error)

    def delete(self, name):
        # type: (str) -> None
        if self.backup_format and self.backup_bucket:
            try:
                obj = self.client.get_object(self.bucket_name, name)
            except merr.ResponseError as error:
                raise minio_error(
                    "Could not obtain file {} " "to make a copy of it".format(name),
                    error,
                )

            try:
                content_length = int(obj.getheader("Content-Length"))
            except ValueError as error:
                raise minio_error(f"Could not backup removed file {name}", error)

            # Creates the backup filename
            target_name = "{}{}".format(
                timezone.now().strftime(self.backup_format), name
            )
            try:
                self.client.put_object(
                    self.backup_bucket, target_name, obj, content_length
                )
            except merr.ResponseError as error:
                raise minio_error(
                    "Could not make a copy of file "
                    "{} before removing it".format(name),
                    error,
                )

        try:
            self.client.remove_object(self.bucket_name, name)
        except merr.ResponseError as error:
            raise minio_error(f"Could not remove file {name}", error)

    def exists(self, name):
        # type: (str) -> bool
        try:
            self.client.stat_object(self.bucket_name, self._sanitize_path(name))
            return True
        except merr.ResponseError as error:
            # TODO - deprecate
            if error.code == "NoSuchKey":
                return False
            else:
                raise minio_error(f"Could not stat file {name}", error)
        except merr.NoSuchKey:
            return False
        except merr.NoSuchBucket:
            raise
        except Exception as error:
            logger.error(error)

    def listdir(self, path):
        #  [None, "", "."] is supported to mean the configured root among various
        #  implementations of Storage implementations so we copy that behaviour even if
        #  maybe None should raise an exception instead.
        #
        #  If the path prefix does not match anything full prefix that does exist this
        #  function will just return empty results, this is different from
        #  FileSystemStorage where an invalid directory would raise an OSError.

        if path in [None, "", "."]:
            path = ""
        else:
            path += "/"

        dirs = []
        files = []
        try:
            objects = self.client.list_objects_v2(self.bucket_name, prefix=path)
            for o in objects:
                p = posixpath.relpath(o.object_name, path)
                if o.is_dir:
                    dirs.append(p)
                else:
                    files.append(p)
            return dirs, files
        except merr.NoSuchBucket:
            raise
        except merr.ResponseError as error:
            raise minio_error(f"Could not list directory {path}", error)

    def size(self, name):
        # type: (str) -> int
        try:
            info = self.client.stat_object(self.bucket_name, name)
            return info.size
        except merr.ResponseError as error:
            raise minio_error(f"Could not access file size for {name}", error)

    def url(self, name, *args, max_age=None):
        # type: (str) -> str
        kwargs = {}
        if max_age is not None:
            kwargs["expires"] = max_age

        # NOTE: Here be dragons, when a external base_url is used the code
        # below is both using "internal" minio clint APIs and somewhat
        # subverting how minio/S3 expects urls to be generated in the first
        # place.

        if self.presign_urls:
            url = self.client.presigned_get_object(self.bucket_name, name, **kwargs)
            if self.base_url is not None:
                parsed_url = urlparse(url)
                path = parsed_url.path.split(self.bucket_name, 1)[1]
                url = "{}{}?{}{}{}".format(
                    self.base_url,
                    path,
                    parsed_url.params,
                    parsed_url.query,
                    parsed_url.fragment,
                )

        else:
            if self.base_url is not None:

                def strip_beg(path):
                    while path.startswith("/"):
                        path = path[1:]
                    return path

                def strip_end(path):
                    while path.endswith("/"):
                        path = path[:-1]
                    return path

                url = "{}/{}".format(
                    strip_end(self.base_url), urllib.parse.quote(strip_beg(name))
                )
            else:
                url = get_target_url(
                    self.client._endpoint_url,
                    bucket_name=self.bucket_name,
                    object_name=name,
                    # bucket_region=region,
                )
        return url

    def accessed_time(self, name):
        # type: (str) -> datetime.datetime
        """
        Not available via the S3 API
        """
        return self.modified_time(name)

    def created_time(self, name):
        # type: (str) -> datetime.datetime
        """
        Not available via the S3 API
        """
        return self.modified_time(name)

    def modified_time(self, name):
        # type: (str) -> datetime.datetime
        try:
            info = self.client.stat_object(self.bucket_name, name)
            return datetime.datetime.fromtimestamp(mktime(info.last_modified))
        except merr.ResponseError as error:
            raise minio_error(
                f"Could not access modification time for file {name}", error
            )


_NoValue = object()


def get_setting(name, default=_NoValue):
    result = getattr(settings, name, default)
    if result is _NoValue:
        print("Attr {} : {}".format(name, getattr(settings, name, default)))
        raise ImproperlyConfigured
    else:
        return result


def create_minio_client_from_settings(*, minio_kwargs=dict()):
    endpoint = get_setting("MINIO_STORAGE_ENDPOINT")
    access_key = get_setting("MINIO_STORAGE_ACCESS_KEY")
    secret_key = get_setting("MINIO_STORAGE_SECRET_KEY")
    secure = get_setting("MINIO_STORAGE_USE_HTTPS", True)
    client = minio.Minio(
        endpoint,
        access_key=access_key,
        secret_key=secret_key,
        secure=secure,
        **minio_kwargs,
    )
    return client


@deconstructible
class MinioMediaStorage(MinioStorage):
    def __init__(self):
        client = create_minio_client_from_settings()
        bucket_name = get_setting("MINIO_STORAGE_MEDIA_BUCKET_NAME")
        base_url = get_setting("MINIO_STORAGE_MEDIA_URL", None)
        auto_create_bucket = get_setting(
            "MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET", False
        )
        auto_create_policy = get_setting(
            "MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY", False
        )
        presign_urls = get_setting("MINIO_STORAGE_MEDIA_USE_PRESIGNED", False)
        backup_format = get_setting("MINIO_STORAGE_MEDIA_BACKUP_FORMAT", False)
        backup_bucket = get_setting("MINIO_STORAGE_MEDIA_BACKUP_BUCKET", False)

        super().__init__(
            client,
            bucket_name,
            auto_create_bucket=auto_create_bucket,
            auto_create_policy=auto_create_policy,
            base_url=base_url,
            presign_urls=presign_urls,
            backup_format=backup_format,
            backup_bucket=backup_bucket,
        )


@deconstructible
class MinioStaticStorage(MinioStorage):
    def __init__(self):
        client = create_minio_client_from_settings()
        base_url = get_setting("MINIO_STORAGE_STATIC_URL", None)
        bucket_name = get_setting("MINIO_STORAGE_STATIC_BUCKET_NAME")
        auto_create_bucket = get_setting(
            "MINIO_STORAGE_AUTO_CREATE_STATIC_BUCKET", False
        )
        auto_create_policy = get_setting(
            "MINIO_STORAGE_AUTO_CREATE_STATIC_POLICY", False
        )

        presign_urls = get_setting("MINIO_STORAGE_STATIC_USE_PRESIGNED", False)

        super().__init__(
            client,
            bucket_name,
            auto_create_bucket=auto_create_bucket,
            auto_create_policy=auto_create_policy,
            base_url=base_url,
            presign_urls=presign_urls,
        )

{'fraction_non_alphanumeric': 0.08222769524112179, 'fraction_numerical': 0.0007944704854214666, 'mean_word_length': 1.9544654383288347, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1517937', 'n_tokens_mistral': 7171, 'n_tokens_neox': 6966, 'n_words': 1666}