<commit_before>import re

from collections import OrderedDict

import arrow

from django.apps import apps
from django.conf import settings
from django.contrib import messages as django_messages
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.urls import reverse, reverse_lazy
from django.http import Http404, HttpResponseRedirect
from django.shortcuts import render
from django.views.generic.base import TemplateView, View
from django.views.generic.edit import DeleteView, FormView
from django.db.models import Count, F

import feedparser

from common.activities import (activity_from_data_request_project,
                               personalize_activities,
                               personalize_activities_dict,
                               sort_activities)
from common.mixins import LargePanelMixin, NeverCacheMixin, PrivateMixin
from common.utils import querydict_from_dict
from common.views import BaseOAuth2AuthorizationView
from data_import.models import DataFile, is_public
from public_data.models import PublicDataAccess
from private_sharing.models import (ActivityFeed, DataRequestProject,
                                    FeaturedProject, DataRequestProjectMember,
                                    id_label_to_project)
from private_sharing.utilities import (
    get_source_labels_and_names_including_dynamic, source_to_url_slug)

from .forms import ActivityMessageForm
from .mixins import SourcesContextMixin
from .models import BlogPost, Member, GrantProject

User = get_user_model()
TEN_MINUTES = 60 * 10


class SourceDataFilesDeleteView(PrivateMixin, DeleteView):
    """
    Let the user delete all datafiles for a source. Note that DeleteView was
    written with a single object in mind but will happily delete a QuerySet due
    to duck-typing.
    """

    template_name = 'member/my-member-source-data-files-delete.html'
    success_url = reverse_lazy('my-member-data')

    def get_object(self, queryset=None):
        source = self.kwargs['source']
        self.source = source

        return DataFile.objects.filter(user=self.request.user, source=source)

    def get_context_data(self, **kwargs):
        """
        Add the source to the request context.
        """
        context = super(SourceDataFilesDeleteView, self).get_context_data(
            **kwargs)

        context.update({
            'source': self.kwargs['source'],
        })

        return context

    def get_success_url(self):
        """
        Direct to relevant activity page.
        """
        url_slug = source_to_url_slug(self.source)
        return reverse('activity-management', kwargs={'source': url_slug})


class ExceptionView(View):
    """
    Raises an exception for testing purposes.
    """

    @staticmethod
    def get(request):  # pylint: disable=unused-argument
        raise Exception('A test exception.')


class OAuth2LoginView(LargePanelMixin, TemplateView):
    """
    Give people authorizing with us the ability to easily sign up or log in.
    """

    template_name = 'account/login-oauth2.html'

    def get_context_data(self, **kwargs):
        next_querystring = querydict_from_dict({
            'next': self.request.GET.get('next')
        }).urlencode()

        kwargs.update({
            'next_querystring': next_querystring,
            'connection': self.request.GET.get('connection'),
        })

        return super(OAuth2LoginView, self).get_context_data(**kwargs)


class PublicDataDocumentationView(TemplateView):
    """
    Add activities to the context.
    """

    template_name = 'pages/public-data-api.html'

    def get_context_data(self, **kwargs):
        context = super(PublicDataDocumentationView, self).get_context_data(
            **kwargs)
        activities = OrderedDict(
            get_source_labels_and_names_including_dynamic())

        context.update({
            'activities': activities,
        })

        return context


class AuthorizationView(BaseOAuth2AuthorizationView):
    """
    Add checks for study apps to the OAuth2 authorization view.
    """

    is_study_app = False

    @staticmethod
    def _check_study_app_request(context):
        """
        Return true if this OAuth2 request matches a study app
        """
        # NOTE: This assumes 'scopes' was overwritten by get_context_data.
        scopes = [x[0] for x in context['scopes']]

        try:
            scopes.remove('read')
            scopes.remove('write')
        except ValueError:
            return False

        if len(scopes) != 1:
            return False

        app_label = re.sub('-', '_', scopes[0])
        app = apps.get_app_config(app_label)

        if app and app.verbose_name == context['application'].name:
            return app_label

        return False

    def get_context_data(self, **kwargs):
        context = super(AuthorizationView, self).get_context_data(**kwargs)

        def scope_key(zipped_scope):
            scope, _ = zipped_scope

            # Sort 'write' second to last
            if scope == 'write':
                return 'zzy'

            # Sort 'read' last
            if scope == 'read':
                return 'zzz'

            # Sort all other scopes alphabetically
            return scope

        def scope_class(scope):
            if scope in ['read', 'write']:
                return 'info'

            return 'primary'

        zipped_scopes = zip(context['scopes'], context['scopes_descriptions'])
        zipped_scopes.sort(key=scope_key)

        context['scopes'] = [(scope, description, scope_class(scope))
                             for scope, description in zipped_scopes]

        # For custom display when it's for a study app connection.
        app_label = self._check_study_app_request(context)

        if app_label:
            self.is_study_app = True

            context['app'] = apps.get_app_config(app_label)
            context['app_label'] = app_label
            context['is_study_app'] = True
            context['scopes'] = [x for x in context['scopes']
                                 if x[0] != 'read' and x[0] != 'write']

        return context

    def get_template_names(self):
        if self.is_study_app:
            return ['oauth2_provider/finalize.html']

        return [self.template_name]


class HomeView(NeverCacheMixin, SourcesContextMixin, TemplateView):
    """
    View with recent activity feed, blog posts, and highlighted projects.
    """

    template_name = 'pages/home.html'

    @staticmethod
    def get_recent_blogposts():
        blogposts_cachetag = 'recent-blogposts'
        blogposts = cache.get(blogposts_cachetag)
        if blogposts:
            return blogposts

        blogfeed = feedparser.parse('https://blog.openhumans.org/feed/')
        posts = []
        for item in blogfeed['entries'][0:3]:
            try:
                post = BlogPost.objects.get(rss_id=item['id'])
            except BlogPost.DoesNotExist:
                post = BlogPost.create(rss_feed_entry=item)
            posts.append(post)
        return posts
        cache.set(blogposts_cachetag, posts, timeout=TEN_MINUTES)

    @staticmethod
    def get_recent_activity():
        recent = ActivityFeed.objects.order_by('-timestamp')[0:12]
        recent_1 = recent[:int((len(recent)+1)/2)]
        recent_2 = recent[int((len(recent)+1)/2):]
        return (recent_1, recent_2)

    def get_featured_projects(self):
        """
        Get FeaturedProjects in 'activity' data format

        Override description if one is provided.
        """
        featured_projs = FeaturedProject.objects.order_by('id')[0:3]
        highlighted = []
        activities = personalize_activities_dict(self.request.user)
        try:
            for featured in featured_projs:
                try:
                    activity = activities[featured.project.id_label]
                    if featured.description:
                        activity['commentary'] = featured.description
                    highlighted.append(activity)
                except KeyError:
                    pass
            return highlighted
        except (ValueError, TypeError):
            return []

    def get_context_data(self, *args, **kwargs):
        context = super(HomeView,
                        self).get_context_data(*args, **kwargs)
        recent_activity_1, recent_activity_2 = self.get_recent_activity()

        context.update({
            'recent_activityfeed_1': recent_activity_1,
            'recent_activityfeed_2': recent_activity_2,
            'recent_blogposts': self.get_recent_blogposts(),
            'featured_projects': self.get_featured_projects(),
            'no_description': True,
        })

        return context


class AddDataPageView(NeverCacheMixin, SourcesContextMixin, TemplateView):
    """
    View with data source activities. Never cached.
    """
    template_name = 'pages/add-data.html'

    def get_context_data(self, *args, **kwargs):
        context = super(AddDataPageView,
                        self).get_context_data(*args, **kwargs)
        projects = DataRequestProject.objects.exclude(
            returned_data_description__isnull=True).exclude(
            returned_data_description='').filter(
            approved=True).filter(
            active=True)
        print(projects.count())
        activities = sort_activities(
            {proj.id_label: activity_from_data_request_project(
                project=proj, user=self.request.user) for proj in projects})<commit_msg>hotfix for missing projects on add/explore pages<commit_after>import re

from collections import OrderedDict

import arrow

from django.apps import apps
from django.conf import settings
from django.contrib import messages as django_messages
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.urls import reverse, reverse_lazy
from django.http import Http404, HttpResponseRedirect
from django.shortcuts import render
from django.views.generic.base import TemplateView, View
from django.views.generic.edit import DeleteView, FormView
from django.db.models import Count, F

import feedparser

from common.activities import (activity_from_data_request_project,
                               personalize_activities,
                               personalize_activities_dict,
                               sort_activities)
from common.mixins import LargePanelMixin, NeverCacheMixin, PrivateMixin
from common.utils import querydict_from_dict
from common.views import BaseOAuth2AuthorizationView
from data_import.models import DataFile, is_public
from public_data.models import PublicDataAccess
from private_sharing.models import (ActivityFeed, DataRequestProject,
                                    FeaturedProject, DataRequestProjectMember,
                                    id_label_to_project)
from private_sharing.utilities import (
    get_source_labels_and_names_including_dynamic, source_to_url_slug)

from .forms import ActivityMessageForm
from .mixins import SourcesContextMixin
from .models import BlogPost, Member, GrantProject

User = get_user_model()
TEN_MINUTES = 60 * 10


class SourceDataFilesDeleteView(PrivateMixin, DeleteView):
    """
    Let the user delete all datafiles for a source. Note that DeleteView was
    written with a single object in mind but will happily delete a QuerySet due
    to duck-typing.
    """

    template_name = 'member/my-member-source-data-files-delete.html'
    success_url = reverse_lazy('my-member-data')

    def get_object(self, queryset=None):
        source = self.kwargs['source']
        self.source = source

        return DataFile.objects.filter(user=self.request.user, source=source)

    def get_context_data(self, **kwargs):
        """
        Add the source to the request context.
        """
        context = super(SourceDataFilesDeleteView, self).get_context_data(
            **kwargs)

        context.update({
            'source': self.kwargs['source'],
        })

        return context

    def get_success_url(self):
        """
        Direct to relevant activity page.
        """
        url_slug = source_to_url_slug(self.source)
        return reverse('activity-management', kwargs={'source': url_slug})


class ExceptionView(View):
    """
    Raises an exception for testing purposes.
    """

    @staticmethod
    def get(request):  # pylint: disable=unused-argument
        raise Exception('A test exception.')


class OAuth2LoginView(LargePanelMixin, TemplateView):
    """
    Give people authorizing with us the ability to easily sign up or log in.
    """

    template_name = 'account/login-oauth2.html'

    def get_context_data(self, **kwargs):
        next_querystring = querydict_from_dict({
            'next': self.request.GET.get('next')
        }).urlencode()

        kwargs.update({
            'next_querystring': next_querystring,
            'connection': self.request.GET.get('connection'),
        })

        return super(OAuth2LoginView, self).get_context_data(**kwargs)


class PublicDataDocumentationView(TemplateView):
    """
    Add activities to the context.
    """

    template_name = 'pages/public-data-api.html'

    def get_context_data(self, **kwargs):
        context = super(PublicDataDocumentationView, self).get_context_data(
            **kwargs)
        activities = OrderedDict(
            get_source_labels_and_names_including_dynamic())

        context.update({
            'activities': activities,
        })

        return context


class AuthorizationView(BaseOAuth2AuthorizationView):
    """
    Add checks for study apps to the OAuth2 authorization view.
    """

    is_study_app = False

    @staticmethod
    def _check_study_app_request(context):
        """
        Return true if this OAuth2 request matches a study app
        """
        # NOTE: This assumes 'scopes' was overwritten by get_context_data.
        scopes = [x[0] for x in context['scopes']]

        try:
            scopes.remove('read')
            scopes.remove('write')
        except ValueError:
            return False

        if len(scopes) != 1:
            return False

        app_label = re.sub('-', '_', scopes[0])
        app = apps.get_app_config(app_label)

        if app and app.verbose_name == context['application'].name:
            return app_label

        return False

    def get_context_data(self, **kwargs):
        context = super(AuthorizationView, self).get_context_data(**kwargs)

        def scope_key(zipped_scope):
            scope, _ = zipped_scope

            # Sort 'write' second to last
            if scope == 'write':
                return 'zzy'

            # Sort 'read' last
            if scope == 'read':
                return 'zzz'

            # Sort all other scopes alphabetically
            return scope

        def scope_class(scope):
            if scope in ['read', 'write']:
                return 'info'

            return 'primary'

        zipped_scopes = zip(context['scopes'], context['scopes_descriptions'])
        zipped_scopes.sort(key=scope_key)

        context['scopes'] = [(scope, description, scope_class(scope))
                             for scope, description in zipped_scopes]

        # For custom display when it's for a study app connection.
        app_label = self._check_study_app_request(context)

        if app_label:
            self.is_study_app = True

            context['app'] = apps.get_app_config(app_label)
            context['app_label'] = app_label
            context['is_study_app'] = True
            context['scopes'] = [x for x in context['scopes']
                                 if x[0] != 'read' and x[0] != 'write']

        return context

    def get_template_names(self):
        if self.is_study_app:
            return ['oauth2_provider/finalize.html']

        return [self.template_name]


class HomeView(NeverCacheMixin, SourcesContextMixin, TemplateView):
    """
    View with recent activity feed, blog posts, and highlighted projects.
    """

    template_name = 'pages/home.html'

    @staticmethod
    def get_recent_blogposts():
        blogposts_cachetag = 'recent-blogposts'
        blogposts = cache.get(blogposts_cachetag)
        if blogposts:
            return blogposts

        blogfeed = feedparser.parse('https://blog.openhumans.org/feed/')
        posts = []
        for item in blogfeed['entries'][0:3]:
            try:
                post = BlogPost.objects.get(rss_id=item['id'])
            except BlogPost.DoesNotExist:
                post = BlogPost.create(rss_feed_entry=item)
            posts.append(post)
        return posts
        cache.set(blogposts_cachetag, posts, timeout=TEN_MINUTES)

    @staticmethod
    def get_recent_activity():
        recent = ActivityFeed.objects.order_by('-timestamp')[0:12]
        recent_1 = recent[:int((len(recent)+1)/2)]
        recent_2 = recent[int((len(recent)+1)/2):]
        return (recent_1, recent_2)

    def get_featured_projects(self):
        """
        Get FeaturedProjects in 'activity' data format

        Override description if one is provided.
        """
        featured_projs = FeaturedProject.objects.order_by('id')[0:3]
        highlighted = []
        activities = personalize_activities_dict(self.request.user)
        try:
            for featured in featured_projs:
                try:
                    activity = activities[featured.project.id_label]
                    if featured.description:
                        activity['commentary'] = featured.description
                    highlighted.append(activity)
                except KeyError:
                    pass
            return highlighted
        except (ValueError, TypeError):
            return []

    def get_context_data(self, *args, **kwargs):
        context = super(HomeView,
                        self).get_context_data(*args, **kwargs)
        recent_activity_1, recent_activity_2 = self.get_recent_activity()

        context.update({
            'recent_activityfeed_1': recent_activity_1,
            'recent_activityfeed_2': recent_activity_2,
            'recent_blogposts': self.get_recent_blogposts(),
            'featured_projects': self.get_featured_projects(),
            'no_description': True,
        })

        return context


class AddDataPageView(NeverCacheMixin, SourcesContextMixin, TemplateView):
    """
    View with data source activities. Never cached.
    """
    template_name = 'pages/add-data.html'

    def get_context_data(self, *args, **kwargs):
        context = super(AddDataPageView,
                        self).get_context_data(*args, **kwargs)
        projects = DataRequestProject.objects.filter(
            approved=True).filter(
            active=True)
        print(projects.count())
        activities = sort_activities(
            {proj.id_label: activity_from_data_request_project(
                project=proj, user=self.request.user) for proj in projects})
{'fraction_non_alphanumeric': 0.0893898786063377, 'fraction_numerical': 0.004309212254979242, 'mean_word_length': 2.4067311134980307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3584010', 'n_tokens_mistral': 5255, 'n_tokens_neox': 5060, 'n_words': 1221}