<commit_before>from past.builtins import basestring
import functools
import itertools
import logging
import re
from future.moves.urllib.parse import urljoin
import warnings
from rest_framework import status as http_status

import bson
from django.db.models import Q
from dirtyfields import DirtyFieldsMixin
from django.apps import apps
from django_bulk_update.helper import bulk_update
from django.contrib.auth.models import AnonymousUser, Permission
from django.contrib.contenttypes.fields import GenericRelation
from django.core.paginator import Paginator
from django.core.urlresolvers import reverse
from django.db import models, connection
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils import timezone
from django.utils.functional import cached_property
from keen import scoped_keys
from psycopg2._psycopg import AsIs
from typedmodels.models import TypedModel, TypedModelManager
from include import IncludeManager
from guardian.models import (
    GroupObjectPermissionBase,
    UserObjectPermissionBase,
)
from guardian.shortcuts import get_objects_for_user, get_groups_with_perms, get_group_perms

from framework import status
from framework.auth import oauth_scopes
from framework.celery_tasks.handlers import enqueue_task, get_task_from_queue
from framework.exceptions import PermissionsError, HTTPError
from framework.sentry import log_exception
from osf.exceptions import (InvalidTagError, NodeStateError,
                            TagNotFoundError, UserNotAffiliatedError)
from osf.models.contributor import Contributor
from osf.models.collection import CollectionSubmission

from osf.models.identifiers import Identifier, IdentifierMixin
from osf.models.licenses import NodeLicenseRecord
from osf.models.mixins import (AddonModelMixin, CommentableMixin, Loggable, ContributorMixin, GuardianMixin,
                               NodeLinkMixin, Taggable, TaxonomizableMixin, SpamOverrideMixin, RegistrationResponseMixin)
from osf.models.node_relation import NodeRelation
from osf.models.nodelog import NodeLog
from osf.models.sanctions import RegistrationApproval
from osf.models.private_link import PrivateLink
from osf.models.tag import Tag
from osf.models.user import OSFUser
from osf.models.validators import validate_title, validate_doi
from framework.auth.core import Auth
from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField
from osf.utils.fields import NonNaiveDateTimeField
from osf.utils.requests import get_request_and_user_id, string_type_request_headers
from osf.utils import sanitize
from website import language, settings
from website.citations.utils import datetime_to_csl
from website.project.licenses import set_license
from website.project import signals as project_signals
from website.project import tasks as node_tasks
from website.project.model import NodeUpdateError
from website.identifiers.tasks import update_doi_metadata_on_change
from website.identifiers.clients import DataCiteClient
from osf.utils.permissions import (
    ADMIN,
    ADMIN_NODE,
    CONTRIB_PERMISSIONS,
    CREATOR_PERMISSIONS,
    PERMISSIONS,
    READ,
    READ_NODE,
    WRITE
)
from website.util import api_url_for, api_v2_url, web_url_for
from .base import BaseModel, GuidMixin, GuidMixinQuerySet
from api.caching.tasks import update_storage_usage
from api.caching import settings as cache_settings
from api.caching.utils import storage_usage_cache


logger = logging.getLogger(__name__)


class AbstractNodeQuerySet(GuidMixinQuerySet):

    def get_roots(self):
        return self.filter(id__in=self.exclude(type='osf.collection').exclude(type='osf.quickfilesnode').values_list('root_id', flat=True))

    def get_children(self, root, active=False, include_root=False):
        # If `root` is a root node, we can use the 'descendants' related name
        # rather than doing a recursive query
        if root.id == root.root_id:
            query = root.descendants.all() if include_root else root.descendants.exclude(id=root.id)
            if active:
                query = query.filter(is_deleted=False)
            return query
        else:
            sql = """
                WITH RECURSIVE descendants AS (
                SELECT
                    parent_id,
                    child_id,
                    1 AS LEVEL,
                    ARRAY[parent_id] as pids
                FROM %s
                %s
                WHERE is_node_link IS FALSE AND parent_id = %s %s
                UNION ALL
                SELECT
                    d.parent_id,
                    s.child_id,
                    d.level + 1,
                    d.pids || s.parent_id
                FROM descendants AS d
                    JOIN %s AS s
                    ON d.child_id = s.parent_id
                WHERE s.is_node_link IS FALSE AND %s = ANY(pids)
                ) SELECT array_agg(DISTINCT child_id)
                FROM descendants
                WHERE parent_id = %s;
            """
            with connection.cursor() as cursor:
                node_relation_table = AsIs(NodeRelation._meta.db_table)
                cursor.execute(sql, [
                    node_relation_table,
                    AsIs('LEFT JOIN osf_abstractnode ON {}.child_id = osf_abstractnode.id'.format(node_relation_table) if active else ''),
                    root.pk,
                    AsIs('AND osf_abstractnode.is_deleted IS FALSE' if active else ''),
                    node_relation_table,
                    root.pk,
                    root.pk])
                row = cursor.fetchone()[0]
                if not row:
                    return AbstractNode.objects.filter(id=root.pk) if include_root else AbstractNode.objects.none()
                if include_root:
                    row.append(root.pk)
                return AbstractNode.objects.filter(id__in=row)

    def can_view(self, user=None, private_link=None):
        qs = self.filter(is_public=True)

        if private_link is not None:
            if isinstance(private_link, PrivateLink):
                private_link = private_link.key
            if not isinstance(private_link, basestring):
                raise TypeError('"private_link" must be either {} or {}. Got {!r}'.format(str, PrivateLink, private_link))

            qs |= self.filter(private_links__is_deleted=False, private_links__key=private_link)

        if user is not None and not isinstance(user, AnonymousUser):<commit_msg>Fixing set unions in can view<commit_after>from past.builtins import basestring
import functools
import itertools
import logging
import re
from future.moves.urllib.parse import urljoin
import warnings
from rest_framework import status as http_status

import bson
from django.db.models import Q
from dirtyfields import DirtyFieldsMixin
from django.apps import apps
from django_bulk_update.helper import bulk_update
from django.contrib.auth.models import AnonymousUser, Permission
from django.contrib.contenttypes.fields import GenericRelation
from django.core.paginator import Paginator
from django.core.urlresolvers import reverse
from django.db import models, connection
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils import timezone
from django.utils.functional import cached_property
from keen import scoped_keys
from psycopg2._psycopg import AsIs
from typedmodels.models import TypedModel, TypedModelManager
from include import IncludeManager
from guardian.models import (
    GroupObjectPermissionBase,
    UserObjectPermissionBase,
)
from guardian.shortcuts import get_objects_for_user, get_groups_with_perms, get_group_perms

from framework import status
from framework.auth import oauth_scopes
from framework.celery_tasks.handlers import enqueue_task, get_task_from_queue
from framework.exceptions import PermissionsError, HTTPError
from framework.sentry import log_exception
from osf.exceptions import (InvalidTagError, NodeStateError,
                            TagNotFoundError, UserNotAffiliatedError)
from osf.models.contributor import Contributor
from osf.models.collection import CollectionSubmission

from osf.models.identifiers import Identifier, IdentifierMixin
from osf.models.licenses import NodeLicenseRecord
from osf.models.mixins import (AddonModelMixin, CommentableMixin, Loggable, ContributorMixin, GuardianMixin,
                               NodeLinkMixin, Taggable, TaxonomizableMixin, SpamOverrideMixin, RegistrationResponseMixin)
from osf.models.node_relation import NodeRelation
from osf.models.nodelog import NodeLog
from osf.models.sanctions import RegistrationApproval
from osf.models.private_link import PrivateLink
from osf.models.tag import Tag
from osf.models.user import OSFUser
from osf.models.validators import validate_title, validate_doi
from framework.auth.core import Auth
from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField
from osf.utils.fields import NonNaiveDateTimeField
from osf.utils.requests import get_request_and_user_id, string_type_request_headers
from osf.utils import sanitize
from website import language, settings
from website.citations.utils import datetime_to_csl
from website.project.licenses import set_license
from website.project import signals as project_signals
from website.project import tasks as node_tasks
from website.project.model import NodeUpdateError
from website.identifiers.tasks import update_doi_metadata_on_change
from website.identifiers.clients import DataCiteClient
from osf.utils.permissions import (
    ADMIN,
    ADMIN_NODE,
    CONTRIB_PERMISSIONS,
    CREATOR_PERMISSIONS,
    PERMISSIONS,
    READ,
    READ_NODE,
    WRITE
)
from website.util import api_url_for, api_v2_url, web_url_for
from .base import BaseModel, GuidMixin, GuidMixinQuerySet
from api.caching.tasks import update_storage_usage
from api.caching import settings as cache_settings
from api.caching.utils import storage_usage_cache


logger = logging.getLogger(__name__)


class AbstractNodeQuerySet(GuidMixinQuerySet):

    def get_roots(self):
        return self.filter(id__in=self.exclude(type='osf.collection').exclude(type='osf.quickfilesnode').values_list('root_id', flat=True))

    def get_children(self, root, active=False, include_root=False):
        # If `root` is a root node, we can use the 'descendants' related name
        # rather than doing a recursive query
        if root.id == root.root_id:
            query = root.descendants.all() if include_root else root.descendants.exclude(id=root.id)
            if active:
                query = query.filter(is_deleted=False)
            return query
        else:
            sql = """
                WITH RECURSIVE descendants AS (
                SELECT
                    parent_id,
                    child_id,
                    1 AS LEVEL,
                    ARRAY[parent_id] as pids
                FROM %s
                %s
                WHERE is_node_link IS FALSE AND parent_id = %s %s
                UNION ALL
                SELECT
                    d.parent_id,
                    s.child_id,
                    d.level + 1,
                    d.pids || s.parent_id
                FROM descendants AS d
                    JOIN %s AS s
                    ON d.child_id = s.parent_id
                WHERE s.is_node_link IS FALSE AND %s = ANY(pids)
                ) SELECT array_agg(DISTINCT child_id)
                FROM descendants
                WHERE parent_id = %s;
            """
            with connection.cursor() as cursor:
                node_relation_table = AsIs(NodeRelation._meta.db_table)
                cursor.execute(sql, [
                    node_relation_table,
                    AsIs('LEFT JOIN osf_abstractnode ON {}.child_id = osf_abstractnode.id'.format(node_relation_table) if active else ''),
                    root.pk,
                    AsIs('AND osf_abstractnode.is_deleted IS FALSE' if active else ''),
                    node_relation_table,
                    root.pk,
                    root.pk])
                row = cursor.fetchone()[0]
                if not row:
                    return AbstractNode.objects.filter(id=root.pk) if include_root else AbstractNode.objects.none()
                if include_root:
                    row.append(root.pk)
                return AbstractNode.objects.filter(id__in=row)

    def can_view(self, user=None, private_link=None):
        qs = self.filter(is_public=True)

        if private_link is not None:
            if isinstance(private_link, PrivateLink):
                private_link = private_link.key
            if not isinstance(private_link, basestring):
                raise TypeError('"private_link" must be either {} or {}. Got {!r}'.format(str, PrivateLink, private_link))

            #id_list = [item.id for item in qs]
            return self.filter(private_links__is_deleted=False, private_links__key=private_link).filter(is_deleted=False)

        if user is not None and not isinstance(user, AnonymousUser):
{'fraction_non_alphanumeric': 0.05890937019969278, 'fraction_numerical': 0.0007680491551459293, 'mean_word_length': 2.8729922665080307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1412275', 'n_tokens_mistral': 3617, 'n_tokens_neox': 3441, 'n_words': 982}