<commit_before>"""Resolwe storage model."""
import copy
import logging
import os
from datetime import datetime, timedelta
from pathlib import PurePath
from typing import Iterable, List, Optional, Union

from django.db import models
from django.utils.timezone import now

from resolwe.flow.models import Data
from resolwe.storage.connectors import DEFAULT_CONNECTOR_PRIORITY, connectors
from resolwe.storage.connectors.baseconnector import BaseStorageConnector
from resolwe.storage.connectors.transfer import Transfer
from resolwe.storage.connectors.utils import paralelize
from resolwe.storage.settings import STORAGE_CONNECTORS

logger = logging.getLogger(__name__)


class FileStorage(models.Model):
    """Proxy between Data object and StorageLocation objects."""

    #: creation date and time
    created = models.DateTimeField(auto_now_add=True)

    def has_storage_location(self, connector_name: str) -> bool:
        """Return if StorageLocation object with given connector name exist.

        Only return StorageLocation objects with status set to OK.
        """
        return self.storage_locations.filter(
            connector_name=connector_name, status=StorageLocation.STATUS_DONE
        ).exists()

    @property
    def default_storage_location(self) -> Optional["StorageLocation"]:
        """Return default storage location for this instance.

        This is a storage location using connector with the lowest priority.
        When multiple storage locations have same priority one of them is
        returned.

        Usually only StorageLocation objects with status OK are considered.
        When no such objects exist one of remaining objects is returned (this
        should only happen when data object is still processing). When no
        StorageLocation objects exist None is returned.

        When StorageLocation has ceonnector that is not defined in settings the
        default priority of 100 is applied to it.

        :returns: instance of StorageLocation class or None.
        :rtype: Optional[StorageLocation]
        """
        whens = [
            models.When(
                connector_name=connector_name,
                then=connectors[connector_name].priority,
            )
            for connector_name in connectors
        ]
        q_set_all = self.storage_locations.annotate(
            priority=models.Case(
                *whens,
                default=DEFAULT_CONNECTOR_PRIORITY,
                output_field=models.IntegerField(),
            )
        ).order_by("priority")
        q_set_done = q_set_all.filter(status=StorageLocation.STATUS_DONE)
        return q_set_done.first() or q_set_all.first()

    def get_path(self, prefix: str = None, filename: str = None) -> str:
        """Return path of the default storage location."""
        return self.default_storage_location.get_path(prefix, filename)

    @property
    def subpath(self) -> str:
        """Return URL of the default storage location."""
        return self.default_storage_location.url

    @property
    def files(self) -> models.QuerySet:
        """Get referenced path objects in default storage location."""
        return self.default_storage_location.files

    @property
    def urls(self) -> List[str]:
        """Get a list of URLs of objects stored in default storage location."""
        return self.default_storage_location.urls


class LocationsDoneManager(models.Manager):
    """Return only StorageLocation objects with status set to OK."""

    def get_queryset(self) -> models.QuerySet:
        """Override default queryset."""
        return super().get_queryset().filter(status="OK")

    def _preprocess_delay(self, delay: Optional[int]) -> int:
        """Preprocess delay according to rules.

        1. When delay is None return 0. Entries without explicit delay are
            processed immediately.
        2. When delay is negative return the number of days between the
            current date and 1900. There should be no object created before
            1900 so this means no object will be processed.
        3. Otherwise return the delay.
        """
        if delay is None:
            return 0
        assert isinstance(delay, int), "Delay should be integer or None"
        if delay < 0:
            return (datetime.now() - datetime(1900, 1, 1)).days
        else:
            return delay

    def to_delete(self, connector_name: str) -> models.QuerySet:
        """Get a queryset of locations that must be deleted.

        :returns: a queryset of FileStorage objects whose StorageLocations objects
            belonging to the given connector must be deleted.
        """

        location_settings = copy.deepcopy(STORAGE_CONNECTORS.get(connector_name, {}))
        rules: dict = location_settings.get("config", {}).get("delete", {})

        if not rules:
            return FileStorage.objects.none()

        process_overrides = rules.pop("process_type", {})
        data_slug_overrides = rules.pop("data_slug", {})

        whens = [
            models.When(
                data__slug=data_slug,
                then=timedelta(days=self._preprocess_delay(override_rule.get("delay"))),
            )
            for data_slug, override_rule in data_slug_overrides.items()
        ]
        whens += [
            models.When(
                data__process__type__startswith=process_type,
                then=timedelta(days=self._preprocess_delay(override_rule.get("delay"))),
            )
            for process_type, override_rule in process_overrides.items()
        ]

        connectors_priority_whens = [
            models.When(
                connector_name=connector_name,
                then=connectors[connector_name].priority,
            )
            for connector_name in connectors
        ]

        min_other_copies = rules.get("min_other_copies", 1)
        default_delay = self._preprocess_delay(rules.get("delay"))

        return (
            FileStorage.objects.filter(
                data__status=Data.STATUS_DONE,
                created__lt=now()  # type: ignore
                - models.Case(
                    *whens,
                    default=timedelta(days=default_delay),
                    output_field=models.DurationField(),
                ),
            )
            .annotate(
                all_locations_count=models.Count(
                    "storage_locations",
                    filter=models.Q(
                        storage_locations__status=StorageLocation.STATUS_DONE
                    ),
                    distinct=True,
                )
            )
            .annotate(
                open_logs=models.Exists(
                    AccessLog.objects.filter(
                        storage_location__file_storage=models.OuterRef("id"),
                        storage_location__connector_name=connector_name,
                        finished__isnull=True,
                    )
                ),
                accessed_lately=models.Exists(
                    StorageLocation.objects.filter(
                        file_storage=models.OuterRef("id"),
                        connector_name=connector_name,
                        last_update__gt=now()  # type: ignore
                        - models.Case(
                            *whens,
                            default=timedelta(days=default_delay),
                            output_field=models.DurationField(),
                        ),
                    )
                ),
                default_connector=models.functions.Coalesce(
                    models.Subquery(
                        StorageLocation.objects.filter(<commit_msg>Change model in subquery to match when statement<commit_after>"""Resolwe storage model."""
import copy
import logging
import os
from datetime import datetime, timedelta
from pathlib import PurePath
from typing import Iterable, List, Optional, Union

from django.db import models
from django.utils.timezone import now

from resolwe.flow.models import Data
from resolwe.storage.connectors import DEFAULT_CONNECTOR_PRIORITY, connectors
from resolwe.storage.connectors.baseconnector import BaseStorageConnector
from resolwe.storage.connectors.transfer import Transfer
from resolwe.storage.connectors.utils import paralelize
from resolwe.storage.settings import STORAGE_CONNECTORS

logger = logging.getLogger(__name__)


class FileStorage(models.Model):
    """Proxy between Data object and StorageLocation objects."""

    #: creation date and time
    created = models.DateTimeField(auto_now_add=True)

    def has_storage_location(self, connector_name: str) -> bool:
        """Return if StorageLocation object with given connector name exist.

        Only return StorageLocation objects with status set to OK.
        """
        return self.storage_locations.filter(
            connector_name=connector_name, status=StorageLocation.STATUS_DONE
        ).exists()

    @property
    def default_storage_location(self) -> Optional["StorageLocation"]:
        """Return default storage location for this instance.

        This is a storage location using connector with the lowest priority.
        When multiple storage locations have same priority one of them is
        returned.

        Usually only StorageLocation objects with status OK are considered.
        When no such objects exist one of remaining objects is returned (this
        should only happen when data object is still processing). When no
        StorageLocation objects exist None is returned.

        When StorageLocation has ceonnector that is not defined in settings the
        default priority of 100 is applied to it.

        :returns: instance of StorageLocation class or None.
        :rtype: Optional[StorageLocation]
        """
        whens = [
            models.When(
                connector_name=connector_name,
                then=connectors[connector_name].priority,
            )
            for connector_name in connectors
        ]
        q_set_all = self.storage_locations.annotate(
            priority=models.Case(
                *whens,
                default=DEFAULT_CONNECTOR_PRIORITY,
                output_field=models.IntegerField(),
            )
        ).order_by("priority")
        q_set_done = q_set_all.filter(status=StorageLocation.STATUS_DONE)
        return q_set_done.first() or q_set_all.first()

    def get_path(self, prefix: str = None, filename: str = None) -> str:
        """Return path of the default storage location."""
        return self.default_storage_location.get_path(prefix, filename)

    @property
    def subpath(self) -> str:
        """Return URL of the default storage location."""
        return self.default_storage_location.url

    @property
    def files(self) -> models.QuerySet:
        """Get referenced path objects in default storage location."""
        return self.default_storage_location.files

    @property
    def urls(self) -> List[str]:
        """Get a list of URLs of objects stored in default storage location."""
        return self.default_storage_location.urls


class LocationsDoneManager(models.Manager):
    """Return only StorageLocation objects with status set to OK."""

    def get_queryset(self) -> models.QuerySet:
        """Override default queryset."""
        return super().get_queryset().filter(status="OK")

    def _preprocess_delay(self, delay: Optional[int]) -> int:
        """Preprocess delay according to rules.

        1. When delay is None return 0. Entries without explicit delay are
            processed immediately.
        2. When delay is negative return the number of days between the
            current date and 1900. There should be no object created before
            1900 so this means no object will be processed.
        3. Otherwise return the delay.
        """
        if delay is None:
            return 0
        assert isinstance(delay, int), "Delay should be integer or None"
        if delay < 0:
            return (datetime.now() - datetime(1900, 1, 1)).days
        else:
            return delay

    def to_delete(self, connector_name: str) -> models.QuerySet:
        """Get a queryset of locations that must be deleted.

        :returns: a queryset of FileStorage objects whose StorageLocations objects
            belonging to the given connector must be deleted.
        """

        location_settings = copy.deepcopy(STORAGE_CONNECTORS.get(connector_name, {}))
        rules: dict = location_settings.get("config", {}).get("delete", {})

        if not rules:
            return FileStorage.objects.none()

        process_overrides = rules.pop("process_type", {})
        data_slug_overrides = rules.pop("data_slug", {})

        whens = [
            models.When(
                data__slug=data_slug,
                then=timedelta(days=self._preprocess_delay(override_rule.get("delay"))),
            )
            for data_slug, override_rule in data_slug_overrides.items()
        ]
        whens += [
            models.When(
                data__process__type__startswith=process_type,
                then=timedelta(days=self._preprocess_delay(override_rule.get("delay"))),
            )
            for process_type, override_rule in process_overrides.items()
        ]

        connectors_priority_whens = [
            models.When(
                connector_name=connector_name,
                then=connectors[connector_name].priority,
            )
            for connector_name in connectors
        ]

        min_other_copies = rules.get("min_other_copies", 1)
        default_delay = self._preprocess_delay(rules.get("delay"))

        return (
            FileStorage.objects.filter(
                data__status=Data.STATUS_DONE,
                created__lt=now()  # type: ignore
                - models.Case(
                    *whens,
                    default=timedelta(days=default_delay),
                    output_field=models.DurationField(),
                ),
            )
            .annotate(
                all_locations_count=models.Count(
                    "storage_locations",
                    filter=models.Q(
                        storage_locations__status=StorageLocation.STATUS_DONE
                    ),
                    distinct=True,
                )
            )
            .annotate(
                open_logs=models.Exists(
                    AccessLog.objects.filter(
                        storage_location__file_storage=models.OuterRef("id"),
                        storage_location__connector_name=connector_name,
                        finished__isnull=True,
                    )
                ),
                accessed_lately=models.Exists(
                    FileStorage.objects.filter(
                        pk=models.OuterRef("id"),
                        storage_locations__connector_name=connector_name,
                        storage_locations__status=StorageLocation.STATUS_DONE,
                        storage_locations__last_update__gt=now()  # type: ignore
                        - models.Case(
                            *whens,
                            default=timedelta(days=default_delay),
                            output_field=models.DurationField(),
                        ),
                    )
                ),
                default_connector=models.functions.Coalesce(
                    models.Subquery(
                        StorageLocation.objects.filter(
{'fraction_non_alphanumeric': 0.07518069179143004, 'fraction_numerical': 0.003097573567372225, 'mean_word_length': 2.027940601797577, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5292786', 'n_tokens_mistral': 3895, 'n_tokens_neox': 3693, 'n_words': 1021}