<commit_before># Universal Subtitles, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.
import logging
from datetime import datetime

from django.conf import settings
from django.db.models import Sum, Q
from django.utils import translation
from django.utils.translation import ugettext as _
from icanhaz.models import VideoVisibilityPolicy
from statistic.tasks import st_widget_view_statistic_update
from teams.models import Task, Workflow
from teams.moderation_const import UNMODERATED, WAITING_MODERATION
from teams.permissions import (
    can_create_and_edit_subtitles, can_create_and_edit_translations,
    can_publish_edits_immediately, can_review, can_approve
)
from teams.signals import (
    api_subtitles_edited, api_subtitles_approved, api_subtitles_rejected,
    api_language_new, api_language_edited, api_video_edited
)
from uslogging.models import WidgetDialogLog
from utils import send_templated_email
from utils.forms import flatten_errorlists
from utils.translation import get_user_languages_from_request
from videos import models
from videos.tasks import video_changed_tasks
from widget import video_cache
from widget.base_rpc import BaseRpc
from widget.forms import  FinishReviewForm, FinishApproveForm
from widget.models import SubtitlingSession


yt_logger = logging.getLogger("youtube-ei-error")

ALL_LANGUAGES = settings.ALL_LANGUAGES
LANGUAGES_MAP = dict(ALL_LANGUAGES)

def add_general_settings(request, dict):
    dict.update({
            'writelock_expiration' : models.WRITELOCK_EXPIRATION,
            'embed_version': settings.EMBED_JS_VERSION,
            'languages': ALL_LANGUAGES,
            'metadata_languages': settings.METADATA_LANGUAGES
            })
    if request.user.is_authenticated():
        dict['username'] = request.user.username

class Rpc(BaseRpc):
    def log_session(self, request, draft_pk, log):
        dialog_log = WidgetDialogLog(
            draft_pk=draft_pk,
            browser_id=request.browser_id,
            log=log)
        dialog_log.save()
        send_templated_email(
            settings.WIDGET_LOG_EMAIL,
            'Subtitle save failure',
            'widget/session_log_email.txt',
            { 'log_pk': dialog_log.pk },
            fail_silently=False)
        return { 'response': 'ok' }

    def log_youtube_ei_failure(self, request, page_url):
        user_agent = request.META.get('HTTP_USER_AGENT', '(Unknown)')
        yt_logger.error(
            "Youtube ExternalInterface load failure",
            extra={
                'request': request,
                'data': {
                    'user_agent': user_agent,
                    'page_url': page_url }
                })
        return { 'response': 'ok' }

    def show_widget(self, request, video_url, is_remote, base_state=None, additional_video_urls=None):
        try:
            video_id = video_cache.get_video_id(video_url)
        except Exception as e:
            # for example, private youtube video or private widgets
            return {"error_msg": unicode(e)}

        if video_id is None:
            return None
        visibility_policy = video_cache.get_visibility_policies(video_id)
        if visibility_policy.get('widget', None) != VideoVisibilityPolicy.WIDGET_VISIBILITY_PUBLIC:
            if not VideoVisibilityPolicy.objects.can_show_widget(
                video_id,
                referer=request.META.get('referer'),
                user=request.user):
                return {"error_msg": _("Video embedding disabled by owner")}
        try:
            video_urls = video_cache.get_video_urls(video_id)
        except models.Video.DoesNotExist:
            video_cache.invalidate_video_id(video_url)
            try:
                video_id = video_cache.get_video_id(video_url)
            except Exception as e:
                return {"error_msg": unicode(e)}
            video_urls = video_cache.get_video_urls(video_id)

        return_value = {
            'video_id' : video_id,
            'subtitles': None,
        }
        return_value['video_urls']= video_urls
        return_value['is_moderated'] = video_cache.get_is_moderated(video_id)
        if additional_video_urls is not None:
            for url in additional_video_urls:
                video_cache.associate_extra_url(url, video_id)

        add_general_settings(request, return_value)
        if request.user.is_authenticated():
            return_value['username'] = request.user.username

        return_value['drop_down_contents'] = \
            video_cache.get_video_languages(video_id)

        return_value['my_languages'] = \
            get_user_languages_from_request(request)

        # keeping both forms valid as backwards compatibility layer
        lang_code = base_state and base_state.get("language_code", base_state.get("language", None))
        if base_state is not None and lang_code is not None:
            lang_pk = base_state.get('language_pk', None)
            if lang_pk is  None:
                lang_pk = video_cache.pk_for_default_language(video_id, lang_code)
            subtitles = self._autoplay_subtitles(
                request.user, video_id,
                lang_pk,
                base_state.get('revision', None))
            return_value['subtitles'] = subtitles
        else:
            if is_remote:
                autoplay_language = self._find_remote_autoplay_language(request)
                language_pk = video_cache.pk_for_default_language(video_id, autoplay_language)
                if autoplay_language is not None:
                    subtitles = self._autoplay_subtitles(
                        request.user, video_id, language_pk, None)
                    return_value['subtitles'] = subtitles

        return return_value

    def track_subtitle_play(self, request, video_id):
        st_widget_view_statistic_update.delay(video_id=video_id)
        return { 'response': 'ok' }

    def _find_remote_autoplay_language(self, request):
        language = None
        if not request.user.is_authenticated() or request.user.preferred_language == '':
            language = translation.get_language_from_request(request)
        else:
            language = request.user.preferred_language
        return language if language != '' else None

    def fetch_start_dialog_contents(self, request, video_id):
        my_languages = get_user_languages_from_request(request)
        my_languages.extend([l[:l.find('-')] for l in my_languages if l.find('-') > -1])
        video = models.Video.objects.get(video_id=video_id)
        team_video = video.get_team_video()
        video_languages = [language_summary(l, team_video, request.user) for l
                           in video.subtitlelanguage_set.all()]

        original_language = None
        if video.subtitle_language():
            original_language = video.subtitle_language().language

        tv = video.get_team_video()
        writable_langs = list(tv.team.get_writable_langs()) if tv else []

        return {
            'my_languages': my_languages,
            'video_languages': video_languages,
            'original_language': original_language,
            'limit_languages': writable_langs,
            'is_moderated': video.is_moderated, }

    def fetch_video_id_and_settings(self, request, video_id):
        is_original_language_subtitled = self._subtitle_count(video_id) > 0
        general_settings = {}
        add_general_settings(request, general_settings)
        return {
            'video_id': video_id,
            'is_original_language_subtitled': is_original_language_subtitled,
            'general_settings': general_settings }


    def _check_team_video_locking(self, user, video_id, language_code, is_translation, mode):
        """Check whether the a team prevents the user from editing the subs.

        Returns a dict appropriate for sending back if the user should be
        prevented from editing them, or None if the user can safely edit.

        """
        video = models.Video.objects.get(video_id=video_id)
        team_video = video.get_team_video()

        if not team_video:
            # If there's no team video to worry about, just bail early.
            return None

        # Check that there are no open tasks for this action.
        tasks = team_video.task_set.incomplete().filter(language=language_code)
        if tasks:
            task = tasks[0]
            if not user.is_authenticated() or user != task.assignee:
                return { "can_edit": False, "locked_by": str(task.assignee or task.team) }

        # Check that the team's policies don't prevent the action.
        if mode not in ['review', 'approve']:
            if is_translation:
                can_edit = can_create_and_edit_translations(user, team_video, language_code)
            else:
                can_edit = can_create_and_edit_subtitles(user, team_video, language_code)

            if not can_edit:
                return { "can_edit": False, "locked_by": str(team_video.team) }

    def start_editing(self, request, video_id,
                      language_code,
                      subtitle_language_pk=None,
                      base_language_pk=None,
                      original_language_code=None,
                      mode=None):
        """Called by subtitling widget when subtitling or translation
        is to commence on a video.
        """
        # TODO: remove whenever blank SubtitleLanguages become illegal.
        self._fix_blank_original(video_id)
        if language_code == original_language_code:
            base_language_pk = None
        base_language = None
        if base_language_pk is not None:
            base_language = models.SubtitleLanguage.objects.get(
                pk=base_language_pk)

        language, can_edit = self._get_language_for_editing(
            request, video_id, language_code,
            subtitle_language_pk, base_language)

        # Check for team-related locking.
        locked = self._check_team_video_locking(request.user, video_id,
                                                language_code,
                                                is_translation=bool(base_language_pk),
                                                mode=mode)
        if locked:
            return locked

        if not can_edit:
            return { "can_edit": False,
                     "locked_by" : language.writelock_owner_name }

        session = self._make_subtitling_session(
            request, language, base_language)

        version_for_subs = language.version(public_only=False)

        if not version_for_subs:
            version_for_subs = self._create_version_from_session(session)
            version_no = 0
        else:
            version_no = version_for_subs.version_no + 1
        subtitles = self._subtitles_dict(
            version_for_subs, version_no, base_language_pk is None)
        return_dict = { "can_edit" : True,
                        "session_pk" : session.pk,<commit_msg>Allow edits by non members to team videos.<commit_after># Universal Subtitles, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.
import logging
from datetime import datetime

from django.conf import settings
from django.db.models import Sum, Q
from django.utils import translation
from django.utils.translation import ugettext as _
from icanhaz.models import VideoVisibilityPolicy
from statistic.tasks import st_widget_view_statistic_update
from teams.models import Task, Workflow
from teams.moderation_const import UNMODERATED, WAITING_MODERATION
from teams.permissions import (
    can_create_and_edit_subtitles, can_create_and_edit_translations,
    can_publish_edits_immediately, can_review, can_approve
)
from teams.signals import (
    api_subtitles_edited, api_subtitles_approved, api_subtitles_rejected,
    api_language_new, api_language_edited, api_video_edited
)
from uslogging.models import WidgetDialogLog
from utils import send_templated_email
from utils.forms import flatten_errorlists
from utils.translation import get_user_languages_from_request
from videos import models
from videos.tasks import video_changed_tasks
from widget import video_cache
from widget.base_rpc import BaseRpc
from widget.forms import  FinishReviewForm, FinishApproveForm
from widget.models import SubtitlingSession


yt_logger = logging.getLogger("youtube-ei-error")

ALL_LANGUAGES = settings.ALL_LANGUAGES
LANGUAGES_MAP = dict(ALL_LANGUAGES)

def add_general_settings(request, dict):
    dict.update({
            'writelock_expiration' : models.WRITELOCK_EXPIRATION,
            'embed_version': settings.EMBED_JS_VERSION,
            'languages': ALL_LANGUAGES,
            'metadata_languages': settings.METADATA_LANGUAGES
            })
    if request.user.is_authenticated():
        dict['username'] = request.user.username

class Rpc(BaseRpc):
    def log_session(self, request, draft_pk, log):
        dialog_log = WidgetDialogLog(
            draft_pk=draft_pk,
            browser_id=request.browser_id,
            log=log)
        dialog_log.save()
        send_templated_email(
            settings.WIDGET_LOG_EMAIL,
            'Subtitle save failure',
            'widget/session_log_email.txt',
            { 'log_pk': dialog_log.pk },
            fail_silently=False)
        return { 'response': 'ok' }

    def log_youtube_ei_failure(self, request, page_url):
        user_agent = request.META.get('HTTP_USER_AGENT', '(Unknown)')
        yt_logger.error(
            "Youtube ExternalInterface load failure",
            extra={
                'request': request,
                'data': {
                    'user_agent': user_agent,
                    'page_url': page_url }
                })
        return { 'response': 'ok' }

    def show_widget(self, request, video_url, is_remote, base_state=None, additional_video_urls=None):
        try:
            video_id = video_cache.get_video_id(video_url)
        except Exception as e:
            # for example, private youtube video or private widgets
            return {"error_msg": unicode(e)}

        if video_id is None:
            return None
        visibility_policy = video_cache.get_visibility_policies(video_id)
        if visibility_policy.get('widget', None) != VideoVisibilityPolicy.WIDGET_VISIBILITY_PUBLIC:
            if not VideoVisibilityPolicy.objects.can_show_widget(
                video_id,
                referer=request.META.get('referer'),
                user=request.user):
                return {"error_msg": _("Video embedding disabled by owner")}
        try:
            video_urls = video_cache.get_video_urls(video_id)
        except models.Video.DoesNotExist:
            video_cache.invalidate_video_id(video_url)
            try:
                video_id = video_cache.get_video_id(video_url)
            except Exception as e:
                return {"error_msg": unicode(e)}
            video_urls = video_cache.get_video_urls(video_id)

        return_value = {
            'video_id' : video_id,
            'subtitles': None,
        }
        return_value['video_urls']= video_urls
        return_value['is_moderated'] = video_cache.get_is_moderated(video_id)
        if additional_video_urls is not None:
            for url in additional_video_urls:
                video_cache.associate_extra_url(url, video_id)

        add_general_settings(request, return_value)
        if request.user.is_authenticated():
            return_value['username'] = request.user.username

        return_value['drop_down_contents'] = \
            video_cache.get_video_languages(video_id)

        return_value['my_languages'] = \
            get_user_languages_from_request(request)

        # keeping both forms valid as backwards compatibility layer
        lang_code = base_state and base_state.get("language_code", base_state.get("language", None))
        if base_state is not None and lang_code is not None:
            lang_pk = base_state.get('language_pk', None)
            if lang_pk is  None:
                lang_pk = video_cache.pk_for_default_language(video_id, lang_code)
            subtitles = self._autoplay_subtitles(
                request.user, video_id,
                lang_pk,
                base_state.get('revision', None))
            return_value['subtitles'] = subtitles
        else:
            if is_remote:
                autoplay_language = self._find_remote_autoplay_language(request)
                language_pk = video_cache.pk_for_default_language(video_id, autoplay_language)
                if autoplay_language is not None:
                    subtitles = self._autoplay_subtitles(
                        request.user, video_id, language_pk, None)
                    return_value['subtitles'] = subtitles

        return return_value

    def track_subtitle_play(self, request, video_id):
        st_widget_view_statistic_update.delay(video_id=video_id)
        return { 'response': 'ok' }

    def _find_remote_autoplay_language(self, request):
        language = None
        if not request.user.is_authenticated() or request.user.preferred_language == '':
            language = translation.get_language_from_request(request)
        else:
            language = request.user.preferred_language
        return language if language != '' else None

    def fetch_start_dialog_contents(self, request, video_id):
        my_languages = get_user_languages_from_request(request)
        my_languages.extend([l[:l.find('-')] for l in my_languages if l.find('-') > -1])
        video = models.Video.objects.get(video_id=video_id)
        team_video = video.get_team_video()
        video_languages = [language_summary(l, team_video, request.user) for l
                           in video.subtitlelanguage_set.all()]

        original_language = None
        if video.subtitle_language():
            original_language = video.subtitle_language().language

        tv = video.get_team_video()
        writable_langs = list(tv.team.get_writable_langs()) if tv else []

        return {
            'my_languages': my_languages,
            'video_languages': video_languages,
            'original_language': original_language,
            'limit_languages': writable_langs,
            'is_moderated': video.is_moderated, }

    def fetch_video_id_and_settings(self, request, video_id):
        is_original_language_subtitled = self._subtitle_count(video_id) > 0
        general_settings = {}
        add_general_settings(request, general_settings)
        return {
            'video_id': video_id,
            'is_original_language_subtitled': is_original_language_subtitled,
            'general_settings': general_settings }


    def _check_team_video_locking(self, user, video_id, language_code, is_translation, mode, is_edit):
        """Check whether the a team prevents the user from editing the subs.

        Returns a dict appropriate for sending back if the user should be
        prevented from editing them, or None if the user can safely edit.

        """
        video = models.Video.objects.get(video_id=video_id)
        team_video = video.get_team_video()

        if not team_video:
            # If there's no team video to worry about, just bail early.
            return None

        # Check that there are no open tasks for this action.
        tasks = team_video.task_set.incomplete().filter(language=language_code)
        if tasks:
            task = tasks[0]
            if not user.is_authenticated() or user != task.assignee:
                return { "can_edit": False, "locked_by": str(task.assignee or task.team) }

        # Check that the team's policies don't prevent the action.
        if not is_edit and mode not in ['review', 'approve']:
            if is_translation:
                can_edit = can_create_and_edit_translations(user, team_video, language_code)
            else:
                can_edit = can_create_and_edit_subtitles(user, team_video, language_code)

            if not can_edit:
                return { "can_edit": False, "locked_by": str(team_video.team) }

    def start_editing(self, request, video_id,
                      language_code,
                      subtitle_language_pk=None,
                      base_language_pk=None,
                      original_language_code=None,
                      mode=None):
        """Called by subtitling widget when subtitling or translation
        is to commence on a video.
        """
        # TODO: remove whenever blank SubtitleLanguages become illegal.
        self._fix_blank_original(video_id)
        if language_code == original_language_code:
            base_language_pk = None
        base_language = None
        if base_language_pk is not None:
            base_language = models.SubtitleLanguage.objects.get(
                pk=base_language_pk)

        language, can_edit = self._get_language_for_editing(
            request, video_id, language_code,
            subtitle_language_pk, base_language)

        if not can_edit:
            return { "can_edit": False,
                     "locked_by" : language.writelock_owner_name }

        session = self._make_subtitling_session(
            request, language, base_language)

        version_for_subs = language.version(public_only=False)

        if not version_for_subs:
            version_for_subs = self._create_version_from_session(session)
            version_no = 0
            is_edit = False
        else:
            version_no = version_for_subs.version_no + 1
            is_edit = True

        locked = self._check_team_video_locking(request.user, video_id,
                        language_code, bool(base_language_pk), mode, is_edit)
        if locked:
            return locked

        subtitles = self._subtitles_dict(
            version_for_subs, version_no, base_language_pk is None)
        return_dict = { "can_edit" : True,
                        "session_pk" : session.pk,
{'fraction_non_alphanumeric': 0.07302335547460896, 'fraction_numerical': 0.001028497964431112, 'mean_word_length': 2.5186972255729794, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3781387', 'n_tokens_mistral': 6638, 'n_tokens_neox': 6452, 'n_words': 1559}