<commit_before>import functools
import logging
import os

import requests
from dateutil.parser import parse as parse_date
from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
from django.core.exceptions import ObjectDoesNotExist
from django.db import models, connection
from django.utils import timezone
from framework.analytics import get_basic_counters
from modularodm.exceptions import NoResultsFound
from osf.models.base import BaseModel, Guid, OptionalGuidMixin, ObjectIDMixin
from osf.models.comment import CommentableMixin
from osf.models.mixins import Taggable
from osf.models.validators import validate_location
from osf.modm_compat import Q
from website.util import api_v2_url
from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField
from psycopg2._psycopg import AsIs
from website import util
from website.files import exceptions
from website.files import utils

__all__ = (
    'File',
    'Folder',
    'FileNode',
    'FileVersion',
    'StoredFileNode',
    'TrashedFileNode',
)

PROVIDER_MAP = {}
logger = logging.getLogger(__name__)


class TrashedFileNode(CommentableMixin, OptionalGuidMixin, ObjectIDMixin, BaseModel):
    """The graveyard for all deleted FileNodes"""
    # TODO DELETE ME POST MIGRATION
    modm_model_path = 'website.files.models.TrashedFileNode'
    modm_query = None
    migration_page_size = 100000
    # /TODO DELETE ME POST MIGRATION

    last_touched = models.DateTimeField(null=True, blank=True)
    history = DateTimeAwareJSONField(default=list, blank=True)
    versions = models.ManyToManyField('FileVersion')

    node = models.ForeignKey('osf.AbstractNode', null=True, blank=True)
    object_id = models.PositiveIntegerField(null=True, blank=True)
    content_type = models.ForeignKey('contenttypes.ContentType', null=True, blank=True)
    parent = GenericForeignKey()

    trashed_children = GenericRelation('self')

    is_file = models.BooleanField(default=True)
    provider = models.CharField(max_length=25, blank=True, null=True)  # max_length in staging was 11

    name = models.CharField(max_length=1000, blank=True, null=True)  # max_length in staging was 858
    path = models.CharField(max_length=200, blank=True, null=True)  # max_length in staging was 140
    # max_length in staging was 265
    _materialized_path = models.CharField(max_length=300, blank=True, null=True)
    checkout = models.ForeignKey('osf.OSFUser', related_name='trashed_files_checked_out', null=True, blank=True)
    deleted_by = models.ForeignKey('osf.OSFUser', related_name='files_deleted_by', null=True, blank=True)
    deleted_on = models.DateTimeField(default=timezone.now)  # auto_now_add=True)
    tags = models.ManyToManyField('osf.Tag')
    suspended = models.BooleanField(default=False)

    copied_from = models.ForeignKey('osf.StoredFileNode', default=None, null=True, blank=True)

    @property
    def materialized_path(self):
        if self.provider == 'osfstorage':
            # TODO Optimize this.
            sql = """
                WITH RECURSIVE
                    materialized_path_cte(id, object_id, provider, GEN_DEPTH, GEN_PATH) AS (
                    SELECT
                      sfn.id,
                      sfn.object_id,
                      sfn.provider,
                      1 :: INT         AS depth,
                      sfn.name :: TEXT AS GEN_PATH
                    FROM "%s" AS sfn
                    WHERE
                      sfn.provider = 'osfstorage' AND
                      sfn.object_id IS NULL
                    UNION ALL
                    SELECT
                      c.id,
                      c.object_id,
                      c.provider,
                      p.GEN_DEPTH + 1                       AS GEN_DEPTH,
                      (p.GEN_PATH || '/' || c.name :: TEXT) AS GEN_PATH
                    FROM materialized_path_cte AS p, "%s" AS c
                    WHERE c.object_id = p.id
                  )
                SELECT gen_path
                FROM materialized_path_cte AS n
                WHERE
                  GEN_DEPTH > 1
                  AND
                  n.id = %s
                LIMIT 1;
            """
            with connection.cursor() as cursor:
                cursor.execute(sql, [AsIs(self._meta.db_table), AsIs(self._meta.db_table), self.pk])
                row = cursor.fetchone()
                if not row:
                    return row
                return row[0]
        else:
            return self._materialized_path

    @materialized_path.setter
    def materialized_path(self, val):
        self._materialized_path = val

    @property
    def deep_url(self):
        """Allows deleted files to resolve to a view
        that will provide a nice error message and http.GONE
        """
        return self.node.web_url_for('addon_deleted_file', trashed_id=self._id)

    # For Comment API compatibility
    @property
    def target_type(self):
        """The object "type" used in the OSF v2 API."""
        return 'files'

    @property
    def root_target_page(self):
        """The comment page type associated with TrashedFileNodes."""
        return 'files'

    @property
    def is_deleted(self):
        return True

    def belongs_to_node(self, node_id):
        """Check whether the file is attached to the specified node."""
        return self.node._id == node_id

    def get_extra_log_params(self, comment):
        return {'file': {'name': self.name, 'url': comment.get_comment_page_url()}}

    def restore(self, recursive=True, parent=None):
        """Recreate a StoredFileNode from the data in this object
        Will re-point all guids and finally remove itself
        :raises KeyExistsException:
        """
        local_field_names = self._meta.get_all_field_names()
        intersecting_field_names = set(
            local_field_names
        ).intersection(set(
            FileNode._meta.get_all_field_names()
        ))

        data = {key: getattr(self, key) for key in intersecting_field_names if
                getattr(self, key, None) is not None}
        if parent:
            data['parent_id'] = parent.pk

        restored = FileNode.resolve_class(self.provider, int(self.is_file))(**data)
        if not restored.parent:
            raise ValueError('No parent to restore to')
        restored.save()

        if recursive:
            for child in self.children:
                child.restore(recursive=recursive, parent=restored)
        TrashedFileNode.remove_one(self)
        return restored


class StoredFileNode(CommentableMixin, OptionalGuidMixin, Taggable, ObjectIDMixin, BaseModel):
    """
        The storage backend for FileNode objects.
        This class should generally not be used or created manually as FileNode
        contains all the helpers required.
        A FileNode wraps a StoredFileNode to provider usable abstraction layer
    """

    # TODO DELETE ME POST MIGRATION
    modm_model_path = 'website.files.models.base.StoredFileNode'
    modm_query = None
    migration_page_size = 10000
    # /TODO DELETE ME POST MIGRATION

    # The last time the touch method was called on this FileNode
    last_touched = models.DateTimeField(null=True, blank=True)
    # A list of dictionaries sorted by the 'modified' key<commit_msg>Add field alias for materialized_path<commit_after>import functools
import logging
import os

import requests
from dateutil.parser import parse as parse_date
from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
from django.core.exceptions import ObjectDoesNotExist
from django.db import models, connection
from django.utils import timezone
from framework.analytics import get_basic_counters
from modularodm.exceptions import NoResultsFound
from osf.models.base import BaseModel, Guid, OptionalGuidMixin, ObjectIDMixin
from osf.models.comment import CommentableMixin
from osf.models.mixins import Taggable
from osf.models.validators import validate_location
from osf.modm_compat import Q
from website.util import api_v2_url
from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField
from psycopg2._psycopg import AsIs
from website import util
from website.files import exceptions
from website.files import utils

__all__ = (
    'File',
    'Folder',
    'FileNode',
    'FileVersion',
    'StoredFileNode',
    'TrashedFileNode',
)

PROVIDER_MAP = {}
logger = logging.getLogger(__name__)


class TrashedFileNode(CommentableMixin, OptionalGuidMixin, ObjectIDMixin, BaseModel):
    """The graveyard for all deleted FileNodes"""
    # TODO DELETE ME POST MIGRATION
    modm_model_path = 'website.files.models.TrashedFileNode'
    modm_query = None
    migration_page_size = 100000
    # /TODO DELETE ME POST MIGRATION

    last_touched = models.DateTimeField(null=True, blank=True)
    history = DateTimeAwareJSONField(default=list, blank=True)
    versions = models.ManyToManyField('FileVersion')

    node = models.ForeignKey('osf.AbstractNode', null=True, blank=True)
    object_id = models.PositiveIntegerField(null=True, blank=True)
    content_type = models.ForeignKey('contenttypes.ContentType', null=True, blank=True)
    parent = GenericForeignKey()

    trashed_children = GenericRelation('self')

    is_file = models.BooleanField(default=True)
    provider = models.CharField(max_length=25, blank=True, null=True)  # max_length in staging was 11

    name = models.CharField(max_length=1000, blank=True, null=True)  # max_length in staging was 858
    path = models.CharField(max_length=200, blank=True, null=True)  # max_length in staging was 140
    # max_length in staging was 265
    _materialized_path = models.CharField(max_length=300, blank=True, null=True)
    checkout = models.ForeignKey('osf.OSFUser', related_name='trashed_files_checked_out', null=True, blank=True)
    deleted_by = models.ForeignKey('osf.OSFUser', related_name='files_deleted_by', null=True, blank=True)
    deleted_on = models.DateTimeField(default=timezone.now)  # auto_now_add=True)
    tags = models.ManyToManyField('osf.Tag')
    suspended = models.BooleanField(default=False)

    copied_from = models.ForeignKey('osf.StoredFileNode', default=None, null=True, blank=True)

    @property
    def materialized_path(self):
        if self.provider == 'osfstorage':
            # TODO Optimize this.
            sql = """
                WITH RECURSIVE
                    materialized_path_cte(id, object_id, provider, GEN_DEPTH, GEN_PATH) AS (
                    SELECT
                      sfn.id,
                      sfn.object_id,
                      sfn.provider,
                      1 :: INT         AS depth,
                      sfn.name :: TEXT AS GEN_PATH
                    FROM "%s" AS sfn
                    WHERE
                      sfn.provider = 'osfstorage' AND
                      sfn.object_id IS NULL
                    UNION ALL
                    SELECT
                      c.id,
                      c.object_id,
                      c.provider,
                      p.GEN_DEPTH + 1                       AS GEN_DEPTH,
                      (p.GEN_PATH || '/' || c.name :: TEXT) AS GEN_PATH
                    FROM materialized_path_cte AS p, "%s" AS c
                    WHERE c.object_id = p.id
                  )
                SELECT gen_path
                FROM materialized_path_cte AS n
                WHERE
                  GEN_DEPTH > 1
                  AND
                  n.id = %s
                LIMIT 1;
            """
            with connection.cursor() as cursor:
                cursor.execute(sql, [AsIs(self._meta.db_table), AsIs(self._meta.db_table), self.pk])
                row = cursor.fetchone()
                if not row:
                    return row
                return row[0]
        else:
            return self._materialized_path

    @materialized_path.setter
    def materialized_path(self, val):
        self._materialized_path = val

    @property
    def deep_url(self):
        """Allows deleted files to resolve to a view
        that will provide a nice error message and http.GONE
        """
        return self.node.web_url_for('addon_deleted_file', trashed_id=self._id)

    # For Comment API compatibility
    @property
    def target_type(self):
        """The object "type" used in the OSF v2 API."""
        return 'files'

    @property
    def root_target_page(self):
        """The comment page type associated with TrashedFileNodes."""
        return 'files'

    @property
    def is_deleted(self):
        return True

    def belongs_to_node(self, node_id):
        """Check whether the file is attached to the specified node."""
        return self.node._id == node_id

    def get_extra_log_params(self, comment):
        return {'file': {'name': self.name, 'url': comment.get_comment_page_url()}}

    def restore(self, recursive=True, parent=None):
        """Recreate a StoredFileNode from the data in this object
        Will re-point all guids and finally remove itself
        :raises KeyExistsException:
        """
        local_field_names = self._meta.get_all_field_names()
        intersecting_field_names = set(
            local_field_names
        ).intersection(set(
            FileNode._meta.get_all_field_names()
        ))

        data = {key: getattr(self, key) for key in intersecting_field_names if
                getattr(self, key, None) is not None}
        if parent:
            data['parent_id'] = parent.pk

        restored = FileNode.resolve_class(self.provider, int(self.is_file))(**data)
        if not restored.parent:
            raise ValueError('No parent to restore to')
        restored.save()

        if recursive:
            for child in self.children:
                child.restore(recursive=recursive, parent=restored)
        TrashedFileNode.remove_one(self)
        return restored


class StoredFileNode(CommentableMixin, OptionalGuidMixin, Taggable, ObjectIDMixin, BaseModel):
    """
        The storage backend for FileNode objects.
        This class should generally not be used or created manually as FileNode
        contains all the helpers required.
        A FileNode wraps a StoredFileNode to provider usable abstraction layer
    """

    # TODO DELETE ME POST MIGRATION
    modm_model_path = 'website.files.models.base.StoredFileNode'
    modm_query = None
    migration_page_size = 10000
    # /TODO DELETE ME POST MIGRATION
    FIELD_ALIASES = {
        '_materialized_path': 'materialized_path'
    }

    # The last time the touch method was called on this FileNode
    last_touched = models.DateTimeField(null=True, blank=True)
    # A list of dictionaries sorted by the 'modified' key
{'fraction_non_alphanumeric': 0.0795701849836779, 'fraction_numerical': 0.005712731229597388, 'mean_word_length': 2.442987590728167, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1414678', 'n_tokens_mistral': 4308, 'n_tokens_neox': 4088, 'n_words': 1100}