<commit_before># Universal Subtitles, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

import logging
logger = logging.getLogger("videos-models")

import string
import random
from datetime import datetime, date, timedelta
import time

from django.utils.safestring import mark_safe
from django.core.cache import cache
from django.db import models
from django.db.models.signals import post_save
from django.db.models import Q
from django.db import IntegrityError
from django.utils.dateformat import format as date_format
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.template.defaultfilters import slugify
from django.utils.http import urlquote_plus
from django.utils import simplejson as json
from django.core.urlresolvers import reverse

from auth.models import CustomUser as User, Awards
from videos import EffectiveSubtitle, is_synced, is_synced_value
from videos.types import video_type_registrar
from videos.feed_parser import FeedParser
from comments.models import Comment
from statistic import st_widget_view_statistic
from statistic.tasks import st_sub_fetch_handler_update, st_video_view_handler_update
from widget import video_cache
from utils.redis_utils import RedisSimpleField
from utils.amazon import S3EnabledImageField
from utils.panslugify import pan_slugify

from apps.teams.moderation_const import (
    WAITING_MODERATION, APPROVED, MODERATION_STATUSES, UNMODERATED, REJECTED
)


NO_SUBTITLES, SUBTITLES_FINISHED = range(2)
VIDEO_TYPE_HTML5 = 'H'
VIDEO_TYPE_YOUTUBE = 'Y'
VIDEO_TYPE_BLIPTV = 'B'
VIDEO_TYPE_GOOGLE = 'G'
VIDEO_TYPE_FORA = 'F'
VIDEO_TYPE_USTREAM = 'U'
VIDEO_TYPE_VIMEO = 'V'
VIDEO_TYPE_DAILYMOTION = 'D'
VIDEO_TYPE_FLV = 'L'
VIDEO_TYPE_BRIGHTCOVE = 'C'
VIDEO_TYPE_MP3 = 'M'
VIDEO_TYPE = (
    (VIDEO_TYPE_HTML5, 'HTML5'),
    (VIDEO_TYPE_YOUTUBE, 'Youtube'),
    (VIDEO_TYPE_BLIPTV, 'Blip.tv'),
    (VIDEO_TYPE_GOOGLE, 'video.google.com'),
    (VIDEO_TYPE_FORA, 'Fora.tv'),
    (VIDEO_TYPE_USTREAM, 'Ustream.tv'),
    (VIDEO_TYPE_VIMEO, 'Vimeo.com'),
    (VIDEO_TYPE_DAILYMOTION, 'dailymotion.com'),
    (VIDEO_TYPE_FLV, 'FLV'),
    (VIDEO_TYPE_BRIGHTCOVE, 'brightcove.com'),
    (VIDEO_TYPE_MP3, 'MP3'),
)
VIDEO_META_CHOICES = (
    (1, 'Author',),
    (2, 'Creation Date',),
)
VIDEO_META_TYPE_NAMES = {}
VIDEO_META_TYPE_VARS = {}
VIDEO_META_TYPE_IDS = {}


def update_metadata_choices():
    """Refresh the VIDEO_META_TYPE_* set of constants.

    When VIDEO_META_CHOICES is updated through VideoMetadata.add_metadata_type()
    the set of extra lookup variables needs to be updated as well.  This
    function does that.

    You should never need to call this directly --
    VideoMetadata.add_metadata_type() will take care of it.

    """
    global VIDEO_META_TYPE_NAMES, VIDEO_META_TYPE_VARS , VIDEO_META_TYPE_IDS
    VIDEO_META_TYPE_NAMES = dict(VIDEO_META_CHOICES)
    VIDEO_META_TYPE_VARS = dict((k, name.lower().replace(' ', '_'))
                                for k, name in VIDEO_META_CHOICES)
    VIDEO_META_TYPE_IDS = dict([choice[::-1] for choice in VIDEO_META_CHOICES])
update_metadata_choices()

WRITELOCK_EXPIRATION = 30 # 30 seconds

ALL_LANGUAGES = [(val, _(name))for val, name in settings.ALL_LANGUAGES]


class AlreadyEditingException(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __unicode__(self):
        return self.msg


# Video
class PublicVideoManager(models.Manager):
    def get_query_set(self):
        return super(PublicVideoManager, self).get_query_set().filter(is_public=True)

class Video(models.Model):
    """Central object in the system"""

    video_id = models.CharField(max_length=255, unique=True)
    title = models.CharField(max_length=2048, blank=True)
    description = models.TextField(blank=True)
    duration = models.PositiveIntegerField(null=True, blank=True, help_text=_(u'in seconds'))
    allow_community_edits = models.BooleanField()
    allow_video_urls_edit = models.BooleanField(default=True)
    writelock_time = models.DateTimeField(null=True, editable=False)
    writelock_session_key = models.CharField(max_length=255, editable=False)
    writelock_owner = models.ForeignKey(User, null=True, editable=False,
                                        related_name="writelock_owners")
    is_subtitled = models.BooleanField(default=False)
    was_subtitled = models.BooleanField(default=False, db_index=True)
    thumbnail = models.CharField(max_length=500, blank=True)
    small_thumbnail = models.CharField(max_length=500, blank=True)
    s3_thumbnail = S3EnabledImageField(
        blank=True,
        upload_to='video/thumbnail/',
        thumb_sizes=(
            (290,165),
            (120,90),))
    edited = models.DateTimeField(null=True, editable=False)
    created = models.DateTimeField(auto_now_add=True)
    user = models.ForeignKey(User, null=True, blank=True)
    followers = models.ManyToManyField(User, blank=True, related_name='followed_videos', editable=False)
    complete_date = models.DateTimeField(null=True, blank=True, editable=False)
    featured = models.DateTimeField(null=True, blank=True)

    subtitles_fetched_count = models.IntegerField(_(u'Sub.fetched'), default=0, db_index=True, editable=False)
    # counter for evertime the widget plays accounted for both on and off site
    widget_views_count = models.IntegerField(_(u'Widget views'), default=0, db_index=True, editable=False)
    # counter for the # of times the video page is shown in the unisubs website
    view_count = models.PositiveIntegerField(_(u'Views'), default=0, db_index=True, editable=False)

    # Denormalizing the subtitles(had_version) count, in order to get faster joins
    # updated from update_languages_count()
    languages_count = models.PositiveIntegerField(default=0, db_index=True, editable=False)
    moderated_by = models.ForeignKey("teams.Team", blank=True, null=True, related_name="moderating")

    # denormalized convenience from VideoVisibility, should not be set
    # directely
    is_public = models.BooleanField(default=True)


    objects = models.Manager()
    public  = PublicVideoManager()

    def __unicode__(self):
        title = self.title_display()
        if len(title) > 60:
            title = title[:60]+'...'
        return title

    def update_search_index(self):
        """Queue a Celery task that will update this video's Solr entry."""
        from utils.celery_search_index import update_search_index
        update_search_index.delay(self.__class__, self.pk)

    @property
    def views(self):
        """Return a dict of the number of views recorded for this video.

        The map will look like:

            {'month': 100, 'week': 5, 'year': 10223, 'total': 20333}

        Caches this map in memcache for two hours.

        """
        if not hasattr(self, '_video_views_statistic'):
            cache_key = 'video_views_statistic_%s' % self.pk
            views_st = cache.get(cache_key)

            if not views_st:
                views_st = st_widget_view_statistic.get_views(video=self)
                views_st['total'] = self.widget_views_count
                cache.set(cache_key, views_st, 60*60*2)

            self._video_views_statistic = views_st

        return self._video_views_statistic

    def title_display(self, truncate=True):
        v = self.latest_version()

        if v and v.title and v.title.strip():
            title = v.title
        elif self.title and self.title.strip():
            title = self.title
        else:
            try:
                url = self.videourl_set.all()[:1].get().url
                if not url:
                    return 'No title'
            except models.ObjectDoesNotExist:
                return 'No title'

            url = url.strip('/')

            if url.startswith('http://'):
                url = url[7:]

            parts = url.split('/')
            if len(parts) > 1:
                title = '%s/.../%s' % (parts[0], parts[-1])
            else:
                title = url

        if truncate and len(title) > 35:
            title = title[:35] + '...'

        return title

    def update_view_counter(self):
        """Queue a Celery task that will increment the number of views for this video."""
        try:
            st_video_view_handler_update.delay(video_id=self.video_id)
        except:
            from sentry.client.models import client
            client.create_from_exception()

    def update_subtitles_fetched(self, lang=None):
        """Queue a Celery task that will increment the number of times this video's subtitles were fetched."""
        try:
            sl_pk = lang.pk if lang else None
            st_sub_fetch_handler_update.delay(video_id=self.video_id, sl_pk=sl_pk)
            if lang:
                from videos.tasks import update_subtitles_fetched_counter_for_sl

                update_subtitles_fetched_counter_for_sl.delay(sl_pk=lang.pk)
        except:
            from sentry.client.models import client
            client.create_from_exception()

    def get_thumbnail(self, fallback=True):
        """Return a URL to this video's thumbnail.

        This may be an absolute or relative URL, depending on whether the
        thumbnail is stored in our media folder or on S3.

        If fallback is True, it will fallback to the default thumbnail

        """
        if self.s3_thumbnail:
            return self.s3_thumbnail.url

        if self.thumbnail:
            return self.thumbnail
        if fallback:
            return "%simages/video-no-thumbnail-medium.png" % settings.STATIC_URL_BASE

    def get_small_thumbnail(self):
        """Return a URL to a small version of this video's thumbnail, or '' if there isn't one.

        This may be an absolute or relative URL, depending on whether the
        thumbnail is stored in our media folder or on S3.

        """
        if self.s3_thumbnail:
            return self.s3_thumbnail.thumb_url(120, 90)

        if self.small_thumbnail:
            return self.small_thumbnail
        return "%simages/video-no-thumbnail-small.png" % settings.STATIC_URL_BASE

    def get_medium_thumbnail(self):
        """Return a URL to a medium version of this video's thumbnail, or '' if there isn't one.

        This may be an absolute or relative URL, depending on whether the
        thumbnail is stored in our media folder or on S3.

        """
        if self.s3_thumbnail:
            return self.s3_thumbnail.thumb_url(290, 165)

        if self.thumbnail:
            return self.thumbnail

        return "%simages/video-no-thumbnail-medium.png" % settings.STATIC_URL


    def get_team_video(self):
        """Return the TeamVideo object for this video, or None if there isn't one."""
        from teams.models import TeamVideo

        try:
            return self.teamvideo
        except TeamVideo.DoesNotExist:
            return None


    def thumbnail_link(self):
        """Return a URL to this video's thumbnail, or '' if there isn't one.

        Unlike get_thumbnail, this URL will always be absolute.

        """
        if not self.thumbnail:
            return ''

        if self.thumbnail.startswith('http://'):
            return self.thumbnail

        return settings.STATIC_URL+self.thumbnail

    def is_html5(self):
        """Return whether if the original URL for this video is an HTML5 one."""
        try:
            return self.videourl_set.filter(original=True)[:1].get().is_html5()
        except models.ObjectDoesNotExist:
            return False

    def search_page_url(self):
        return self.get_absolute_url()

    def title_for_url(self):
        """Return this video's title with non-URL-friendly characters replaced.

        NOTE: this method is used in videos.search_indexes.VideoSearchResult to
        prevent duplication of code in search result and in DB-query result.

        """
        return pan_slugify(self.title)

    def _get_absolute_url(self, video_id=None):
        """
        NOTE: this method is used in videos.search_indexes.VideoSearchResult
        to prevent duplication of code in search result and in DB-query result

        This is a little hack, because Django uses get_absolute_url in own way,
        so it was impossible just copy to VideoSearchResult
        """
        kwargs = {'video_id': video_id or self.video_id}
        title = self.title_for_url()
        if title:
            kwargs['title'] = title
            return reverse('videos:video_with_title',
                           kwargs=kwargs)
        return reverse('videos:video',  kwargs=kwargs)

    get_absolute_url = _get_absolute_url

    def get_primary_videourl_obj(self):
        """Return the primary video URL for this video if one exists, otherwise None.

        This will return a VideoUrl object.

        """
        try:
            return self.videourl_set.filter(primary=True).all()[:1].get()
        except models.ObjectDoesNotExist:
            return None
            
    def get_video_url(self):
        """Return the primary video URL for this video if one exists, otherwise None.

        This will return a string of an actual URL, not a VideoUrl.

        """
        vurl = self.get_primary_videourl_obj()
        return vurl.effective_url if vurl else None

    @classmethod
    def get_or_create_for_url(cls, video_url=None, vt=None, user=None, timestamp=None):
        assert video_url or vt, 'should be video URL or VideoType'
        vt = vt or video_type_registrar.video_type_for_url(video_url)
        if not vt:
            return None, False

        try:
            video_url_obj = VideoUrl.objects.get(
                url=vt.convert_to_video_url())
            video, created = video_url_obj.video, False
        except models.ObjectDoesNotExist:
            video, created = None, False

        if not video:
            try:
                video_url_obj = VideoUrl.objects.get(
                    type=vt.abbreviation, **vt.create_kwars())
                if user:
                    Action.create_video_handler(video_url_obj.video, user)
                return video_url_obj.video, False
            except VideoUrl.DoesNotExist:
                obj = Video()
                obj = vt.set_values(obj)
                if obj.title:
                    obj.slug = slugify(obj.title)
                obj.user = user
                obj.save()

                from videos.tasks import save_thumbnail_in_s3
                save_thumbnail_in_s3.delay(obj.pk)

                Action.create_video_handler(obj, user)

                #Save video url
                defaults = {
                    'type': vt.abbreviation,
                    'original': True,
                    'primary': True,
                    'added_by': user,
                    'video': obj
                }
                if vt.video_id:
                    defaults['videoid'] = vt.video_id
                video_url_obj, created = VideoUrl.objects.get_or_create(url=vt.convert_to_video_url(),
                                                                        defaults=defaults)
                assert video_url_obj.video == obj
                obj.update_search_index()
                video, created = obj, True

        if timestamp and video_url_obj.created != timestamp:
           video_url_obj.created = timestamp
           video_url_obj.save(updates_timestamp=False)
        user and user.notify_by_message and video.followers.add(user)
        if not video_url_obj.owner_username:
            if hasattr(vt, 'username'):
                video_url_obj.owner_username = vt.username
                video_url_obj.save()
        return video, created

    @property
    def language(self):
        """Return the language code of this video's original language as a string.

        Will return None if unknown.

        """
        ol = self._original_subtitle_language()

        if ol and ol.language:
            return ol.language

    @property
    def filename(self):
        """Return a filename-safe version of this video's string representation.

        Could be useful when providing a user with a file related to this video
        to download, etc.

        """
        from django.utils.text import get_valid_filename

        return get_valid_filename(self.__unicode__())

    def lang_filename(self, language):
        """Return a filename-safe version of this video's string representation with a language code.

        Could be useful when providing a user with a file of subs to download,
        etc.

        """
        name = self.filename
        lang = language.language or u'original'
        return u'%s.%s' % (name, lang)

    @property
    def subtitle_state(self):
        """Return the subtitling state for this video.

        The value returned will be one of the NO_SUBTITLES or SUBTITLES_FINISHED
        constants.

        """
        return NO_SUBTITLES if self.latest_version() is None else SUBTITLES_FINISHED

    def _original_subtitle_language(self):
        """Return the SubtitleLanguage in the original language of this video, or None.

        Caches the result in the object.

        """
        if not hasattr(self, '_original_subtitle'):
            try:
                original = self.subtitlelanguage_set.filter(is_original=True)[:1].get()
            except models.ObjectDoesNotExist:
                original = None

            setattr(self, '_original_subtitle', original)

        return getattr(self, '_original_subtitle')

    def has_original_language(self):
        """Return whether this video has a SubtitleLanguage for its original language.

        NOTE: this uses another method which caches the result in the object, so
        this will effectively be cached in-object as well.

        """
        original_language = self._original_subtitle_language()
        if original_language:
            return original_language.language != ''

    def subtitle_language(self, language_code=None):
        """Return the SubtitleLanguage for this video with the given language code, or None.

        If None is passed as a language_code, the original language
        SubtitleLanguage will be returned.  In this case the value will be
        cached in-object.

        This method can produce surprising results if the video has more
        than one subtitle language with the same code. This is an artifact
        of when we did not allow this. In this case, we return the
        language with the most subtitles.

        """
        try:
            if language_code is None:
                return self._original_subtitle_language()
            else:
                return self.subtitlelanguage_set.filter(
                    language=language_code).order_by('-subtitle_count')[:1].get()
        except models.ObjectDoesNotExist:
            return None

    def subtitle_languages(self, language_code):
        """Return all SubtitleLanguages for this video with the given language code."""
        return self.subtitlelanguage_set.filter(language=language_code)

    def version(self, version_no=None, language=None, public_only=True):
        """Return the SubtitleVersion for this video matching the given criteria.

        If language is given (it must be a SubtitleLanguage, NOT a string
        language code) the version will be looked up for that, otherwise the
        original language will be used.

        If version_no is given, the version with that number will be returned.

        If public_only is True (the default) only versions visible to the public
        (i.e.: not moderated) will be considered.  If it is false all versions
        are eligable.

        If no version fitting all the criteria is found, None is returned.

        """
        if language is None:
            language = self.subtitle_language()
        return None if language is None else language.version(version_no, public_only=public_only)

    def latest_version(self, language_code=None, public_only=True):
        """Return the latest SubtitleVersion for this video matching the given criteria.

        If language is given (it must be a SubtitleLanguage, NOT a string
        language code) the version will be looked up for that, otherwise the
        original language will be used.

        If public_only is True (the default) only versions visible to the public
        (i.e.: not moderated) will be considered.  If it is false all versions
        are eligable.

        If no version fitting all the criteria is found, None is returned.

        """
        language = self.subtitle_language(language_code)
        return None if language is None else language.latest_version(public_only=public_only)

    def subtitles(self, version_no=None, language_code=None, language_pk=None):
        if language_pk is None:
            language = self.subtitle_language(language_code)
        else:
            try:
                language = self.subtitlelanguage_set.get(pk=language_pk)
            except models.ObjectDoesNotExist:
                language = None
        version = self.version(version_no, language)
        if version:
            return version.subtitles()
        else:
            return Subtitle.objects.none()

    def latest_subtitles(self, language_code=None, public_only=True):
        version = self.latest_version(language_code, public_only=public_only)
        return [] if version is None else version.subtitles(public_only=public_only)

    def translation_language_codes(self):
        """All iso language codes with finished translations."""
        return set([sl.language for sl
                    in self.subtitlelanguage_set.filter(
                    is_complete=True).filter(is_original=False)])

    @property
    def writelock_owner_name(self):
        """The user who currently has a subtitling writelock on this video."""
        if self.writelock_owner == None:
            return "anonymous"
        else:
            return self.writelock_owner.__unicode__()

    @property
    def is_writelocked(self):
        """Is this video writelocked for subtitling?"""
        if self.writelock_time == None:
            return False
        delta = datetime.now() - self.writelock_time
        seconds = delta.days * 24 * 60 * 60 + delta.seconds
        return seconds < WRITELOCK_EXPIRATION

    def can_writelock(self, request):
        """Can I place a writelock on this video for subtitling?"""
        return self.writelock_session_key == \
            request.browser_id or \
            not self.is_writelocked

    def writelock(self, request):
        """Writelock this video for subtitling."""
        self._make_writelock(request.user, request.browser_id)

    def _make_writelock(self, user, key):
        if user.is_authenticated():
            self.writelock_owner = user
        else:
            self.writelock_owner = None
        self.writelock_session_key = key
        self.writelock_time = datetime.now()

    def release_writelock(self):
        """Writelock this video for subtitling."""
        self.writelock_owner = None
        self.writelock_session_key = ''
        self.writelock_time = None

    def notification_list(self, exclude=None):
        qs = self.followers.filter(notify_by_email=True, is_active=True)
        if exclude:
            if not isinstance(exclude, (list, tuple)):
                exclude = [exclude]
            qs = qs.exclude(pk__in=[u.pk for u in exclude if u and u.is_authenticated()])
        return qs

    def notification_list_all(self, exclude=None):
        users = []
        for language in self.subtitlelanguage_set.all():
            for u in language.notification_list(exclude):
                if not u in users:
                    users.append(u)
        for user in self.notification_list(exclude):
            if not user in users:
                users.append(user)
        return users

    def subtitle_language_dict(self):
        langs = {}
        for sl in self.subtitlelanguage_set.all():
            if not sl.language:
                continue
            if sl.language in langs:
                langs[sl.language].append(sl)
            else:
                langs[sl.language] = [sl]
        return langs

    @property
    def is_complete(self):
        """
        We consider complete a video which has one or more
        subtitle languages either marked as complete, or
        having 100%  as the percent_done.
        """
        for sl in self.subtitlelanguage_set.all():
            if sl.is_complete_and_synced():
                return True
        return False

    def completed_subtitle_languages(self, public_only=True):
        return [sl for sl in self.subtitlelanguage_set.all()
                if sl.is_complete_and_synced(public_only=public_only)]

    @property
    def policy(self):

        if not hasattr(self, "_cached_policy"):
            from icanhaz.models import VideoVisibilityPolicy
            try:
                self._cached_policy =  VideoVisibilityPolicy.objects.get(video=self)
            except VideoVisibilityPolicy.DoesNotExist:
                self._cached_policy =  None
        return self._cached_policy


    def get_title_display(self):
        """Return a suitable title to display to a user for this video.

        This will use the most specific title if it's present, but if it's blank
        it will fall back to the less-specific-but-at-least-it-exists video
        title instead.

        """
        l = self.subtitle_language()
        return l.get_title_display() if l else self.title

    def get_description_display(self):
        """Return a suitable description to display to a user for this video.

        This will use the most specific description if it's present, but if it's
        blank it will fall back to the less-specific-but-at-least-it-exists
        video description instead.

        """
        l = self.subtitle_language()
        return l.get_description_display() if l else self.description


    @property
    def is_moderated(self):
        return bool(self.moderated_by_id)

    def metadata(self):
        '''Return a dict of metadata for this video.

        Example:

        { 'author': 'Sample author',
          'creation_date': datetime(...), }

        '''
        meta = dict([(VIDEO_META_TYPE_VARS[md.key], md.data)
                     for md in self.videometadata_set.all()])

        meta['creation_date'] = VideoMetadata.string_to_date(meta.get('creation_date'))

        return meta


    class Meta(object):
        permissions = (
            ("can_moderate_version"   , "Can moderate version" ,),
        )


def create_video_id(sender, instance, **kwargs):
    """Generate (and set) a random video_id for this video before saving.

    Also fills in the edited timestamp.
    TODO: Split the 'edited' update out into a new function.

    """
    instance.edited = datetime.now()
    if not instance or instance.video_id:
        return
    alphanum = string.letters+string.digits
    instance.video_id = ''.join([random.choice(alphanum) for i in xrange(12)])

def video_delete_handler(sender, instance, **kwargs):
    video_cache.invalidate_cache(instance.video_id)

models.signals.pre_save.connect(create_video_id, sender=Video)
models.signals.pre_delete.connect(video_delete_handler, sender=Video)
models.signals.m2m_changed.connect(User.video_followers_change_handler, sender=Video.followers.through)


# VideoMetadata
class VideoMetadata(models.Model):
    video = models.ForeignKey(Video)
    key = models.PositiveIntegerField(choices=VIDEO_META_CHOICES)
    data = models.CharField(max_length=255)

    created = models.DateTimeField(editable=False, auto_now_add=True)
    modified = models.DateTimeField(editable=False, auto_now=True)

    @classmethod
    def add_metadata_type(cls, num, readable_name):
        """Add a new key choice.

        These can't be added at class creation time because some of those types
        live on the integration repo and therefore can't be referenced from
        here.

        This makes sure that if code is trying to do this dynamically we'll
        never allow it to overwrite a key with a different name.

        """
        field = VideoMetadata._meta.get_field_by_name('key')[0]

        choices = field.choices
        for x in choices:
            if x[0] == num and x[1] != readable_name:
                raise ValueError(
                    "Cannot add a metadata value twice, tried %s -> %s which clashes with %s -> %s" %
                    (num, readable_name, x[0], x[1]))
            elif x[0] == num and x[1] == readable_name:
                return
        choices = choices + ((num, readable_name,),)

        # public attr is read only
        global VIDEO_META_CHOICES
        VIDEO_META_CHOICES = field._choices = choices
        update_metadata_choices()


    class Meta:
        ordering = ('created',)
        verbose_name_plural = 'video metadata'

    def __unicode__(self):
        data = self.data
        if len(data) > 30:
            data = data[:30] + '...'
        return u'%s - %s: %s' % (self.video,
                                 self.get_key_display(),
                                 data)

    @classmethod
    def date_to_string(cls, d):
        return d.strftime('%Y-%m-%d') if d else ''

    @classmethod
    def string_to_date(cls, s):
        return datetime.strptime(s, '%Y-%m-%d').date() if s else None


# SubtitleLanguage
class SubtitleLanguage(models.Model):
    video = models.ForeignKey(Video)
    is_original = models.BooleanField()
    language = models.CharField(max_length=16, choices=ALL_LANGUAGES, blank=True)
    writelock_time = models.DateTimeField(null=True, editable=False)
    writelock_session_key = models.CharField(max_length=255, blank=True, editable=False)
    writelock_owner = models.ForeignKey(User, null=True, blank=True, editable=False)
    is_complete = models.BooleanField(default=False)
    subtitle_count = models.IntegerField(default=0, editable=False)

    # has_version: Is there more than one version, and does the latest version
    # have more than 0 subtitles?
    has_version = models.BooleanField(default=False, editable=False)

    # had_version: Is there more than one version, and did some previous version
    # have more than 0 subtitles?
    had_version = models.BooleanField(default=False, editable=False)

    is_forked = models.BooleanField(default=False, editable=False)
    created = models.DateTimeField()
    subtitles_fetched_count = models.IntegerField(default=0, editable=False)
    followers = models.ManyToManyField(User, blank=True, related_name='followed_languages', editable=False)
    percent_done = models.IntegerField(default=0, editable=False)
    standard_language = models.ForeignKey('self', null=True, blank=True, editable=False)

    subtitles_fetched_counter = RedisSimpleField()

    class Meta:
        unique_together = (('video', 'language', 'standard_language'),)

    def __unicode__(self):
        return self.language_display()


    def nonblank_subtitle_count(self, public_only=False):
        return len([s for s in self.latest_subtitles(public_only=public_only)
                    if s.text.strip()])


    def get_title(self, public_only=True):
        """Return the title for this language.

        Tries to use the following (in order):

        * The latest version's title.
        * The video's title.

        """
        v = self.latest_version(public_only=public_only)
        return v.title if v else self.video.title

    def get_title_display(self):
        """Return a suitable title to display to a user for this language.

        This will use the most specific title if it's present, but if it's blank
        it will fall back to the less-specific-but-at-least-it-exists video
        title instead.

        """
        return self.get_title() or self.video.title

    def get_description(self, public_only=True):
        """Return the description for this language.

        Tries to use the following (in order):

        * The latest version's description.
        * The video's description.

        """
        v = self.latest_version(public_only=public_only)
        return v.description if v else self.video.description

    def get_description_display(self):
        """Return a suitable description to display to a user for this language.

        This will use the most specific description if it's present, but if it's
        blank it will fall back to the less-specific-but-at-least-it-exists
        video description instead.

        """
        return self.get_description() or self.video.description


    def is_dependent(self):
        """
        AKA is this language a translation? Stand alone languages must
        either be an original one, or a forked one.
        """
        return not self.is_original and not self.is_forked

    def is_complete_and_synced(self, public_only=True):
        """
        For transcripts, this means the user marked it as completed.
        For translations, the original language must be marked as completed.

        We consider a set of subs where the very last has no end time
        to be synced, as that is a convention for 'until end of time'.
        """
        if not self.is_dependent() and not self.is_complete:
            return False
        if self.is_dependent():
            if self.percent_done != 100:
                return False
            standard_lang = self.standard_language
            if not standard_lang or not standard_lang.is_complete:
                return False
        subtitles = self.latest_subtitles(public_only=public_only)
        if len(subtitles) == 0:

            return False
        if len([s for s in subtitles[:-1] if not s.has_complete_timing()]) > 0:
            return False
        if not is_synced_value(subtitles[-1].start_time):
            return False
        return True

    def get_widget_url(self, mode=None, task_id=None):
        # duplicates unisubs.widget.SubtitleDialogOpener.prototype.openDialogOrRedirect_
        video = self.video
        video_url = video.get_video_url()
        config = {
            "videoID": video.video_id,
            "videoURL": video_url,
            "effectiveVideoURL": video_url,
            "languageCode": self.language,
            "subLanguagePK": self.pk,
            "originalLanguageCode": video.language,
            "mode": mode,
            "task": task_id, }
        if self.is_dependent():
            config['baseLanguagePK'] = self.standard_language and self.standard_language.pk
        return reverse('onsite_widget')+'?config='+urlquote_plus(json.dumps(config))

    @models.permalink
    def get_absolute_url(self):
        return ('videos:translation_history',
                [self.video.video_id, self.language or 'unknown', self.pk])

    def language_display(self):
        if self.is_original and not self.language:
            return 'Original'
        return self.get_language_display()

    @property
    def writelock_owner_name(self):
        if self.writelock_owner == None:
            return "anonymous"
        else:
            return self.writelock_owner.__unicode__()

    @property
    def is_writelocked(self):
        if self.writelock_time == None:
            return False
        delta = datetime.now() - self.writelock_time
        seconds = delta.days * 24 * 60 * 60 + delta.seconds
        return seconds < WRITELOCK_EXPIRATION

    def is_rtl(self):
        from utils.translation import is_rtl
        return is_rtl(self.language)

    def can_writelock(self, request):
        return self.writelock_session_key == \
            request.browser_id or \
            not self.is_writelocked

    def writelock(self, request):
        if request.user.is_authenticated():
            self.writelock_owner = request.user
        else:
            self.writelock_owner = None
        self.writelock_session_key = request.browser_id
        self.writelock_time = datetime.now()

    def release_writelock(self):
        self.writelock_owner = None
        self.writelock_session_key = ''
        self.writelock_time = None

    def _filter_public(self, versions, public_only):
        if public_only:
            versions = versions.filter(moderation_status__in=[APPROVED, UNMODERATED])
        return versions

    def version(self, version_no=None, public_only=True):
        if version_no is None:
            return self.latest_version(public_only)
        try:
            return self._filter_public(self.subtitleversion_set.filter(version_no=version_no), public_only)[0]
        except (models.ObjectDoesNotExist, IndexError):
            pass

    @property
    def last_version(self):
        return self.latest_version(public_only=True)

    def latest_version(self, public_only=True):
        try:
            return self._filter_public(self.subtitleversion_set.all(), public_only)[0]
        except (SubtitleVersion.DoesNotExist, IndexError):
            return None

    def latest_subtitles(self, public_only=True):
        version = self.latest_version(public_only=public_only)
        if version:
            return version.subtitles(public_only=public_only)
        return []

    def notification_list(self, exclude=None):
        qs = self.followers.filter(notify_by_email=True, is_active=True)

        if exclude:
            if not isinstance(exclude, (list, tuple)):
                exclude = [exclude]
            qs = qs.exclude(pk__in=[u.pk for u in exclude if u])
        return qs

    def translations(self):
        return SubtitleLanguage.objects.filter(video=self.video, is_original=False, is_forked=False)

    def fork(self, from_version=None, user=None, result_of_rollback=False, attach_to_language=None):
        """
        If this a dependent and non write locked language,
        then will fork it, making all it's subs timing not
        depend on anything else.
        If locked, will throw an AlreadyEditingException .
        If attach_to_language is passed, we will copy those
        subs to a new language, else self will be used
        """
        to_language = attach_to_language or self
        if from_version:
            original_subs = from_version.subtitle_set.all()
        else:
            if self.standard_language is None:
                return
            original_subs = self.standard_language.latest_version().subtitle_set.all()

        if self.is_writelocked:
            raise AlreadyEditingException(_("Sorry, you cannot upload subtitles right now because someone is editing the language you are uploading or a translation of it"))
        try:
            old_version = self.subtitleversion_set.all()[:1].get()
            version_no = old_version.version_no + 1
        except SubtitleVersion.DoesNotExist:
            old_version = None
            version_no = 0

        kwargs = dict(
            language=to_language, version_no=version_no,
            datetime_started=datetime.now(),
            note=u'Uploaded', is_forked=True, time_change=1,
            text_change=1, result_of_rollback=result_of_rollback,
            forked_from=from_version)
        if user:
            kwargs['user'] = user
        version = SubtitleVersion(**kwargs)
        version.save()

        if old_version:<commit_msg>Allow forking to bypass write lock. This might create an error when a user is editing subs and forks happen while he is editing them, but the end result will be cleaner than having the language missing since the link to the source language has been lost<commit_after># Universal Subtitles, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

import logging
logger = logging.getLogger("videos-models")

import string
import random
from datetime import datetime, date, timedelta
import time

from django.utils.safestring import mark_safe
from django.core.cache import cache
from django.db import models
from django.db.models.signals import post_save
from django.db.models import Q
from django.db import IntegrityError
from django.utils.dateformat import format as date_format
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.template.defaultfilters import slugify
from django.utils.http import urlquote_plus
from django.utils import simplejson as json
from django.core.urlresolvers import reverse

from auth.models import CustomUser as User, Awards
from videos import EffectiveSubtitle, is_synced, is_synced_value
from videos.types import video_type_registrar
from videos.feed_parser import FeedParser
from comments.models import Comment
from statistic import st_widget_view_statistic
from statistic.tasks import st_sub_fetch_handler_update, st_video_view_handler_update
from widget import video_cache
from utils.redis_utils import RedisSimpleField
from utils.amazon import S3EnabledImageField
from utils.panslugify import pan_slugify

from apps.teams.moderation_const import (
    WAITING_MODERATION, APPROVED, MODERATION_STATUSES, UNMODERATED, REJECTED
)


NO_SUBTITLES, SUBTITLES_FINISHED = range(2)
VIDEO_TYPE_HTML5 = 'H'
VIDEO_TYPE_YOUTUBE = 'Y'
VIDEO_TYPE_BLIPTV = 'B'
VIDEO_TYPE_GOOGLE = 'G'
VIDEO_TYPE_FORA = 'F'
VIDEO_TYPE_USTREAM = 'U'
VIDEO_TYPE_VIMEO = 'V'
VIDEO_TYPE_DAILYMOTION = 'D'
VIDEO_TYPE_FLV = 'L'
VIDEO_TYPE_BRIGHTCOVE = 'C'
VIDEO_TYPE_MP3 = 'M'
VIDEO_TYPE = (
    (VIDEO_TYPE_HTML5, 'HTML5'),
    (VIDEO_TYPE_YOUTUBE, 'Youtube'),
    (VIDEO_TYPE_BLIPTV, 'Blip.tv'),
    (VIDEO_TYPE_GOOGLE, 'video.google.com'),
    (VIDEO_TYPE_FORA, 'Fora.tv'),
    (VIDEO_TYPE_USTREAM, 'Ustream.tv'),
    (VIDEO_TYPE_VIMEO, 'Vimeo.com'),
    (VIDEO_TYPE_DAILYMOTION, 'dailymotion.com'),
    (VIDEO_TYPE_FLV, 'FLV'),
    (VIDEO_TYPE_BRIGHTCOVE, 'brightcove.com'),
    (VIDEO_TYPE_MP3, 'MP3'),
)
VIDEO_META_CHOICES = (
    (1, 'Author',),
    (2, 'Creation Date',),
)
VIDEO_META_TYPE_NAMES = {}
VIDEO_META_TYPE_VARS = {}
VIDEO_META_TYPE_IDS = {}


def update_metadata_choices():
    """Refresh the VIDEO_META_TYPE_* set of constants.

    When VIDEO_META_CHOICES is updated through VideoMetadata.add_metadata_type()
    the set of extra lookup variables needs to be updated as well.  This
    function does that.

    You should never need to call this directly --
    VideoMetadata.add_metadata_type() will take care of it.

    """
    global VIDEO_META_TYPE_NAMES, VIDEO_META_TYPE_VARS , VIDEO_META_TYPE_IDS
    VIDEO_META_TYPE_NAMES = dict(VIDEO_META_CHOICES)
    VIDEO_META_TYPE_VARS = dict((k, name.lower().replace(' ', '_'))
                                for k, name in VIDEO_META_CHOICES)
    VIDEO_META_TYPE_IDS = dict([choice[::-1] for choice in VIDEO_META_CHOICES])
update_metadata_choices()

WRITELOCK_EXPIRATION = 30 # 30 seconds

ALL_LANGUAGES = [(val, _(name))for val, name in settings.ALL_LANGUAGES]


class AlreadyEditingException(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __unicode__(self):
        return self.msg


# Video
class PublicVideoManager(models.Manager):
    def get_query_set(self):
        return super(PublicVideoManager, self).get_query_set().filter(is_public=True)

class Video(models.Model):
    """Central object in the system"""

    video_id = models.CharField(max_length=255, unique=True)
    title = models.CharField(max_length=2048, blank=True)
    description = models.TextField(blank=True)
    duration = models.PositiveIntegerField(null=True, blank=True, help_text=_(u'in seconds'))
    allow_community_edits = models.BooleanField()
    allow_video_urls_edit = models.BooleanField(default=True)
    writelock_time = models.DateTimeField(null=True, editable=False)
    writelock_session_key = models.CharField(max_length=255, editable=False)
    writelock_owner = models.ForeignKey(User, null=True, editable=False,
                                        related_name="writelock_owners")
    is_subtitled = models.BooleanField(default=False)
    was_subtitled = models.BooleanField(default=False, db_index=True)
    thumbnail = models.CharField(max_length=500, blank=True)
    small_thumbnail = models.CharField(max_length=500, blank=True)
    s3_thumbnail = S3EnabledImageField(
        blank=True,
        upload_to='video/thumbnail/',
        thumb_sizes=(
            (290,165),
            (120,90),))
    edited = models.DateTimeField(null=True, editable=False)
    created = models.DateTimeField(auto_now_add=True)
    user = models.ForeignKey(User, null=True, blank=True)
    followers = models.ManyToManyField(User, blank=True, related_name='followed_videos', editable=False)
    complete_date = models.DateTimeField(null=True, blank=True, editable=False)
    featured = models.DateTimeField(null=True, blank=True)

    subtitles_fetched_count = models.IntegerField(_(u'Sub.fetched'), default=0, db_index=True, editable=False)
    # counter for evertime the widget plays accounted for both on and off site
    widget_views_count = models.IntegerField(_(u'Widget views'), default=0, db_index=True, editable=False)
    # counter for the # of times the video page is shown in the unisubs website
    view_count = models.PositiveIntegerField(_(u'Views'), default=0, db_index=True, editable=False)

    # Denormalizing the subtitles(had_version) count, in order to get faster joins
    # updated from update_languages_count()
    languages_count = models.PositiveIntegerField(default=0, db_index=True, editable=False)
    moderated_by = models.ForeignKey("teams.Team", blank=True, null=True, related_name="moderating")

    # denormalized convenience from VideoVisibility, should not be set
    # directely
    is_public = models.BooleanField(default=True)


    objects = models.Manager()
    public  = PublicVideoManager()

    def __unicode__(self):
        title = self.title_display()
        if len(title) > 60:
            title = title[:60]+'...'
        return title

    def update_search_index(self):
        """Queue a Celery task that will update this video's Solr entry."""
        from utils.celery_search_index import update_search_index
        update_search_index.delay(self.__class__, self.pk)

    @property
    def views(self):
        """Return a dict of the number of views recorded for this video.

        The map will look like:

            {'month': 100, 'week': 5, 'year': 10223, 'total': 20333}

        Caches this map in memcache for two hours.

        """
        if not hasattr(self, '_video_views_statistic'):
            cache_key = 'video_views_statistic_%s' % self.pk
            views_st = cache.get(cache_key)

            if not views_st:
                views_st = st_widget_view_statistic.get_views(video=self)
                views_st['total'] = self.widget_views_count
                cache.set(cache_key, views_st, 60*60*2)

            self._video_views_statistic = views_st

        return self._video_views_statistic

    def title_display(self, truncate=True):
        v = self.latest_version()

        if v and v.title and v.title.strip():
            title = v.title
        elif self.title and self.title.strip():
            title = self.title
        else:
            try:
                url = self.videourl_set.all()[:1].get().url
                if not url:
                    return 'No title'
            except models.ObjectDoesNotExist:
                return 'No title'

            url = url.strip('/')

            if url.startswith('http://'):
                url = url[7:]

            parts = url.split('/')
            if len(parts) > 1:
                title = '%s/.../%s' % (parts[0], parts[-1])
            else:
                title = url

        if truncate and len(title) > 35:
            title = title[:35] + '...'

        return title

    def update_view_counter(self):
        """Queue a Celery task that will increment the number of views for this video."""
        try:
            st_video_view_handler_update.delay(video_id=self.video_id)
        except:
            from sentry.client.models import client
            client.create_from_exception()

    def update_subtitles_fetched(self, lang=None):
        """Queue a Celery task that will increment the number of times this video's subtitles were fetched."""
        try:
            sl_pk = lang.pk if lang else None
            st_sub_fetch_handler_update.delay(video_id=self.video_id, sl_pk=sl_pk)
            if lang:
                from videos.tasks import update_subtitles_fetched_counter_for_sl

                update_subtitles_fetched_counter_for_sl.delay(sl_pk=lang.pk)
        except:
            from sentry.client.models import client
            client.create_from_exception()

    def get_thumbnail(self, fallback=True):
        """Return a URL to this video's thumbnail.

        This may be an absolute or relative URL, depending on whether the
        thumbnail is stored in our media folder or on S3.

        If fallback is True, it will fallback to the default thumbnail

        """
        if self.s3_thumbnail:
            return self.s3_thumbnail.url

        if self.thumbnail:
            return self.thumbnail
        if fallback:
            return "%simages/video-no-thumbnail-medium.png" % settings.STATIC_URL_BASE

    def get_small_thumbnail(self):
        """Return a URL to a small version of this video's thumbnail, or '' if there isn't one.

        This may be an absolute or relative URL, depending on whether the
        thumbnail is stored in our media folder or on S3.

        """
        if self.s3_thumbnail:
            return self.s3_thumbnail.thumb_url(120, 90)

        if self.small_thumbnail:
            return self.small_thumbnail
        return "%simages/video-no-thumbnail-small.png" % settings.STATIC_URL_BASE

    def get_medium_thumbnail(self):
        """Return a URL to a medium version of this video's thumbnail, or '' if there isn't one.

        This may be an absolute or relative URL, depending on whether the
        thumbnail is stored in our media folder or on S3.

        """
        if self.s3_thumbnail:
            return self.s3_thumbnail.thumb_url(290, 165)

        if self.thumbnail:
            return self.thumbnail

        return "%simages/video-no-thumbnail-medium.png" % settings.STATIC_URL


    def get_team_video(self):
        """Return the TeamVideo object for this video, or None if there isn't one."""
        from teams.models import TeamVideo

        try:
            return self.teamvideo
        except TeamVideo.DoesNotExist:
            return None


    def thumbnail_link(self):
        """Return a URL to this video's thumbnail, or '' if there isn't one.

        Unlike get_thumbnail, this URL will always be absolute.

        """
        if not self.thumbnail:
            return ''

        if self.thumbnail.startswith('http://'):
            return self.thumbnail

        return settings.STATIC_URL+self.thumbnail

    def is_html5(self):
        """Return whether if the original URL for this video is an HTML5 one."""
        try:
            return self.videourl_set.filter(original=True)[:1].get().is_html5()
        except models.ObjectDoesNotExist:
            return False

    def search_page_url(self):
        return self.get_absolute_url()

    def title_for_url(self):
        """Return this video's title with non-URL-friendly characters replaced.

        NOTE: this method is used in videos.search_indexes.VideoSearchResult to
        prevent duplication of code in search result and in DB-query result.

        """
        return pan_slugify(self.title)

    def _get_absolute_url(self, video_id=None):
        """
        NOTE: this method is used in videos.search_indexes.VideoSearchResult
        to prevent duplication of code in search result and in DB-query result

        This is a little hack, because Django uses get_absolute_url in own way,
        so it was impossible just copy to VideoSearchResult
        """
        kwargs = {'video_id': video_id or self.video_id}
        title = self.title_for_url()
        if title:
            kwargs['title'] = title
            return reverse('videos:video_with_title',
                           kwargs=kwargs)
        return reverse('videos:video',  kwargs=kwargs)

    get_absolute_url = _get_absolute_url

    def get_primary_videourl_obj(self):
        """Return the primary video URL for this video if one exists, otherwise None.

        This will return a VideoUrl object.

        """
        try:
            return self.videourl_set.filter(primary=True).all()[:1].get()
        except models.ObjectDoesNotExist:
            return None
            
    def get_video_url(self):
        """Return the primary video URL for this video if one exists, otherwise None.

        This will return a string of an actual URL, not a VideoUrl.

        """
        vurl = self.get_primary_videourl_obj()
        return vurl.effective_url if vurl else None

    @classmethod
    def get_or_create_for_url(cls, video_url=None, vt=None, user=None, timestamp=None):
        assert video_url or vt, 'should be video URL or VideoType'
        vt = vt or video_type_registrar.video_type_for_url(video_url)
        if not vt:
            return None, False

        try:
            video_url_obj = VideoUrl.objects.get(
                url=vt.convert_to_video_url())
            video, created = video_url_obj.video, False
        except models.ObjectDoesNotExist:
            video, created = None, False

        if not video:
            try:
                video_url_obj = VideoUrl.objects.get(
                    type=vt.abbreviation, **vt.create_kwars())
                if user:
                    Action.create_video_handler(video_url_obj.video, user)
                return video_url_obj.video, False
            except VideoUrl.DoesNotExist:
                obj = Video()
                obj = vt.set_values(obj)
                if obj.title:
                    obj.slug = slugify(obj.title)
                obj.user = user
                obj.save()

                from videos.tasks import save_thumbnail_in_s3
                save_thumbnail_in_s3.delay(obj.pk)

                Action.create_video_handler(obj, user)

                #Save video url
                defaults = {
                    'type': vt.abbreviation,
                    'original': True,
                    'primary': True,
                    'added_by': user,
                    'video': obj
                }
                if vt.video_id:
                    defaults['videoid'] = vt.video_id
                video_url_obj, created = VideoUrl.objects.get_or_create(url=vt.convert_to_video_url(),
                                                                        defaults=defaults)
                assert video_url_obj.video == obj
                obj.update_search_index()
                video, created = obj, True

        if timestamp and video_url_obj.created != timestamp:
           video_url_obj.created = timestamp
           video_url_obj.save(updates_timestamp=False)
        user and user.notify_by_message and video.followers.add(user)
        if not video_url_obj.owner_username:
            if hasattr(vt, 'username'):
                video_url_obj.owner_username = vt.username
                video_url_obj.save()
        return video, created

    @property
    def language(self):
        """Return the language code of this video's original language as a string.

        Will return None if unknown.

        """
        ol = self._original_subtitle_language()

        if ol and ol.language:
            return ol.language

    @property
    def filename(self):
        """Return a filename-safe version of this video's string representation.

        Could be useful when providing a user with a file related to this video
        to download, etc.

        """
        from django.utils.text import get_valid_filename

        return get_valid_filename(self.__unicode__())

    def lang_filename(self, language):
        """Return a filename-safe version of this video's string representation with a language code.

        Could be useful when providing a user with a file of subs to download,
        etc.

        """
        name = self.filename
        lang = language.language or u'original'
        return u'%s.%s' % (name, lang)

    @property
    def subtitle_state(self):
        """Return the subtitling state for this video.

        The value returned will be one of the NO_SUBTITLES or SUBTITLES_FINISHED
        constants.

        """
        return NO_SUBTITLES if self.latest_version() is None else SUBTITLES_FINISHED

    def _original_subtitle_language(self):
        """Return the SubtitleLanguage in the original language of this video, or None.

        Caches the result in the object.

        """
        if not hasattr(self, '_original_subtitle'):
            try:
                original = self.subtitlelanguage_set.filter(is_original=True)[:1].get()
            except models.ObjectDoesNotExist:
                original = None

            setattr(self, '_original_subtitle', original)

        return getattr(self, '_original_subtitle')

    def has_original_language(self):
        """Return whether this video has a SubtitleLanguage for its original language.

        NOTE: this uses another method which caches the result in the object, so
        this will effectively be cached in-object as well.

        """
        original_language = self._original_subtitle_language()
        if original_language:
            return original_language.language != ''

    def subtitle_language(self, language_code=None):
        """Return the SubtitleLanguage for this video with the given language code, or None.

        If None is passed as a language_code, the original language
        SubtitleLanguage will be returned.  In this case the value will be
        cached in-object.

        This method can produce surprising results if the video has more
        than one subtitle language with the same code. This is an artifact
        of when we did not allow this. In this case, we return the
        language with the most subtitles.

        """
        try:
            if language_code is None:
                return self._original_subtitle_language()
            else:
                return self.subtitlelanguage_set.filter(
                    language=language_code).order_by('-subtitle_count')[:1].get()
        except models.ObjectDoesNotExist:
            return None

    def subtitle_languages(self, language_code):
        """Return all SubtitleLanguages for this video with the given language code."""
        return self.subtitlelanguage_set.filter(language=language_code)

    def version(self, version_no=None, language=None, public_only=True):
        """Return the SubtitleVersion for this video matching the given criteria.

        If language is given (it must be a SubtitleLanguage, NOT a string
        language code) the version will be looked up for that, otherwise the
        original language will be used.

        If version_no is given, the version with that number will be returned.

        If public_only is True (the default) only versions visible to the public
        (i.e.: not moderated) will be considered.  If it is false all versions
        are eligable.

        If no version fitting all the criteria is found, None is returned.

        """
        if language is None:
            language = self.subtitle_language()
        return None if language is None else language.version(version_no, public_only=public_only)

    def latest_version(self, language_code=None, public_only=True):
        """Return the latest SubtitleVersion for this video matching the given criteria.

        If language is given (it must be a SubtitleLanguage, NOT a string
        language code) the version will be looked up for that, otherwise the
        original language will be used.

        If public_only is True (the default) only versions visible to the public
        (i.e.: not moderated) will be considered.  If it is false all versions
        are eligable.

        If no version fitting all the criteria is found, None is returned.

        """
        language = self.subtitle_language(language_code)
        return None if language is None else language.latest_version(public_only=public_only)

    def subtitles(self, version_no=None, language_code=None, language_pk=None):
        if language_pk is None:
            language = self.subtitle_language(language_code)
        else:
            try:
                language = self.subtitlelanguage_set.get(pk=language_pk)
            except models.ObjectDoesNotExist:
                language = None
        version = self.version(version_no, language)
        if version:
            return version.subtitles()
        else:
            return Subtitle.objects.none()

    def latest_subtitles(self, language_code=None, public_only=True):
        version = self.latest_version(language_code, public_only=public_only)
        return [] if version is None else version.subtitles(public_only=public_only)

    def translation_language_codes(self):
        """All iso language codes with finished translations."""
        return set([sl.language for sl
                    in self.subtitlelanguage_set.filter(
                    is_complete=True).filter(is_original=False)])

    @property
    def writelock_owner_name(self):
        """The user who currently has a subtitling writelock on this video."""
        if self.writelock_owner == None:
            return "anonymous"
        else:
            return self.writelock_owner.__unicode__()

    @property
    def is_writelocked(self):
        """Is this video writelocked for subtitling?"""
        if self.writelock_time == None:
            return False
        delta = datetime.now() - self.writelock_time
        seconds = delta.days * 24 * 60 * 60 + delta.seconds
        return seconds < WRITELOCK_EXPIRATION

    def can_writelock(self, request):
        """Can I place a writelock on this video for subtitling?"""
        return self.writelock_session_key == \
            request.browser_id or \
            not self.is_writelocked

    def writelock(self, request):
        """Writelock this video for subtitling."""
        self._make_writelock(request.user, request.browser_id)

    def _make_writelock(self, user, key):
        if user.is_authenticated():
            self.writelock_owner = user
        else:
            self.writelock_owner = None
        self.writelock_session_key = key
        self.writelock_time = datetime.now()

    def release_writelock(self):
        """Writelock this video for subtitling."""
        self.writelock_owner = None
        self.writelock_session_key = ''
        self.writelock_time = None

    def notification_list(self, exclude=None):
        qs = self.followers.filter(notify_by_email=True, is_active=True)
        if exclude:
            if not isinstance(exclude, (list, tuple)):
                exclude = [exclude]
            qs = qs.exclude(pk__in=[u.pk for u in exclude if u and u.is_authenticated()])
        return qs

    def notification_list_all(self, exclude=None):
        users = []
        for language in self.subtitlelanguage_set.all():
            for u in language.notification_list(exclude):
                if not u in users:
                    users.append(u)
        for user in self.notification_list(exclude):
            if not user in users:
                users.append(user)
        return users

    def subtitle_language_dict(self):
        langs = {}
        for sl in self.subtitlelanguage_set.all():
            if not sl.language:
                continue
            if sl.language in langs:
                langs[sl.language].append(sl)
            else:
                langs[sl.language] = [sl]
        return langs

    @property
    def is_complete(self):
        """
        We consider complete a video which has one or more
        subtitle languages either marked as complete, or
        having 100%  as the percent_done.
        """
        for sl in self.subtitlelanguage_set.all():
            if sl.is_complete_and_synced():
                return True
        return False

    def completed_subtitle_languages(self, public_only=True):
        return [sl for sl in self.subtitlelanguage_set.all()
                if sl.is_complete_and_synced(public_only=public_only)]

    @property
    def policy(self):

        if not hasattr(self, "_cached_policy"):
            from icanhaz.models import VideoVisibilityPolicy
            try:
                self._cached_policy =  VideoVisibilityPolicy.objects.get(video=self)
            except VideoVisibilityPolicy.DoesNotExist:
                self._cached_policy =  None
        return self._cached_policy


    def get_title_display(self):
        """Return a suitable title to display to a user for this video.

        This will use the most specific title if it's present, but if it's blank
        it will fall back to the less-specific-but-at-least-it-exists video
        title instead.

        """
        l = self.subtitle_language()
        return l.get_title_display() if l else self.title

    def get_description_display(self):
        """Return a suitable description to display to a user for this video.

        This will use the most specific description if it's present, but if it's
        blank it will fall back to the less-specific-but-at-least-it-exists
        video description instead.

        """
        l = self.subtitle_language()
        return l.get_description_display() if l else self.description


    @property
    def is_moderated(self):
        return bool(self.moderated_by_id)

    def metadata(self):
        '''Return a dict of metadata for this video.

        Example:

        { 'author': 'Sample author',
          'creation_date': datetime(...), }

        '''
        meta = dict([(VIDEO_META_TYPE_VARS[md.key], md.data)
                     for md in self.videometadata_set.all()])

        meta['creation_date'] = VideoMetadata.string_to_date(meta.get('creation_date'))

        return meta


    class Meta(object):
        permissions = (
            ("can_moderate_version"   , "Can moderate version" ,),
        )


def create_video_id(sender, instance, **kwargs):
    """Generate (and set) a random video_id for this video before saving.

    Also fills in the edited timestamp.
    TODO: Split the 'edited' update out into a new function.

    """
    instance.edited = datetime.now()
    if not instance or instance.video_id:
        return
    alphanum = string.letters+string.digits
    instance.video_id = ''.join([random.choice(alphanum) for i in xrange(12)])

def video_delete_handler(sender, instance, **kwargs):
    video_cache.invalidate_cache(instance.video_id)

models.signals.pre_save.connect(create_video_id, sender=Video)
models.signals.pre_delete.connect(video_delete_handler, sender=Video)
models.signals.m2m_changed.connect(User.video_followers_change_handler, sender=Video.followers.through)


# VideoMetadata
class VideoMetadata(models.Model):
    video = models.ForeignKey(Video)
    key = models.PositiveIntegerField(choices=VIDEO_META_CHOICES)
    data = models.CharField(max_length=255)

    created = models.DateTimeField(editable=False, auto_now_add=True)
    modified = models.DateTimeField(editable=False, auto_now=True)

    @classmethod
    def add_metadata_type(cls, num, readable_name):
        """Add a new key choice.

        These can't be added at class creation time because some of those types
        live on the integration repo and therefore can't be referenced from
        here.

        This makes sure that if code is trying to do this dynamically we'll
        never allow it to overwrite a key with a different name.

        """
        field = VideoMetadata._meta.get_field_by_name('key')[0]

        choices = field.choices
        for x in choices:
            if x[0] == num and x[1] != readable_name:
                raise ValueError(
                    "Cannot add a metadata value twice, tried %s -> %s which clashes with %s -> %s" %
                    (num, readable_name, x[0], x[1]))
            elif x[0] == num and x[1] == readable_name:
                return
        choices = choices + ((num, readable_name,),)

        # public attr is read only
        global VIDEO_META_CHOICES
        VIDEO_META_CHOICES = field._choices = choices
        update_metadata_choices()


    class Meta:
        ordering = ('created',)
        verbose_name_plural = 'video metadata'

    def __unicode__(self):
        data = self.data
        if len(data) > 30:
            data = data[:30] + '...'
        return u'%s - %s: %s' % (self.video,
                                 self.get_key_display(),
                                 data)

    @classmethod
    def date_to_string(cls, d):
        return d.strftime('%Y-%m-%d') if d else ''

    @classmethod
    def string_to_date(cls, s):
        return datetime.strptime(s, '%Y-%m-%d').date() if s else None


# SubtitleLanguage
class SubtitleLanguage(models.Model):
    video = models.ForeignKey(Video)
    is_original = models.BooleanField()
    language = models.CharField(max_length=16, choices=ALL_LANGUAGES, blank=True)
    writelock_time = models.DateTimeField(null=True, editable=False)
    writelock_session_key = models.CharField(max_length=255, blank=True, editable=False)
    writelock_owner = models.ForeignKey(User, null=True, blank=True, editable=False)
    is_complete = models.BooleanField(default=False)
    subtitle_count = models.IntegerField(default=0, editable=False)

    # has_version: Is there more than one version, and does the latest version
    # have more than 0 subtitles?
    has_version = models.BooleanField(default=False, editable=False)

    # had_version: Is there more than one version, and did some previous version
    # have more than 0 subtitles?
    had_version = models.BooleanField(default=False, editable=False)

    is_forked = models.BooleanField(default=False, editable=False)
    created = models.DateTimeField()
    subtitles_fetched_count = models.IntegerField(default=0, editable=False)
    followers = models.ManyToManyField(User, blank=True, related_name='followed_languages', editable=False)
    percent_done = models.IntegerField(default=0, editable=False)
    standard_language = models.ForeignKey('self', null=True, blank=True, editable=False)

    subtitles_fetched_counter = RedisSimpleField()

    class Meta:
        unique_together = (('video', 'language', 'standard_language'),)

    def __unicode__(self):
        return self.language_display()


    def nonblank_subtitle_count(self, public_only=False):
        return len([s for s in self.latest_subtitles(public_only=public_only)
                    if s.text.strip()])


    def get_title(self, public_only=True):
        """Return the title for this language.

        Tries to use the following (in order):

        * The latest version's title.
        * The video's title.

        """
        v = self.latest_version(public_only=public_only)
        return v.title if v else self.video.title

    def get_title_display(self):
        """Return a suitable title to display to a user for this language.

        This will use the most specific title if it's present, but if it's blank
        it will fall back to the less-specific-but-at-least-it-exists video
        title instead.

        """
        return self.get_title() or self.video.title

    def get_description(self, public_only=True):
        """Return the description for this language.

        Tries to use the following (in order):

        * The latest version's description.
        * The video's description.

        """
        v = self.latest_version(public_only=public_only)
        return v.description if v else self.video.description

    def get_description_display(self):
        """Return a suitable description to display to a user for this language.

        This will use the most specific description if it's present, but if it's
        blank it will fall back to the less-specific-but-at-least-it-exists
        video description instead.

        """
        return self.get_description() or self.video.description


    def is_dependent(self):
        """
        AKA is this language a translation? Stand alone languages must
        either be an original one, or a forked one.
        """
        return not self.is_original and not self.is_forked

    def is_complete_and_synced(self, public_only=True):
        """
        For transcripts, this means the user marked it as completed.
        For translations, the original language must be marked as completed.

        We consider a set of subs where the very last has no end time
        to be synced, as that is a convention for 'until end of time'.
        """
        if not self.is_dependent() and not self.is_complete:
            return False
        if self.is_dependent():
            if self.percent_done != 100:
                return False
            standard_lang = self.standard_language
            if not standard_lang or not standard_lang.is_complete:
                return False
        subtitles = self.latest_subtitles(public_only=public_only)
        if len(subtitles) == 0:

            return False
        if len([s for s in subtitles[:-1] if not s.has_complete_timing()]) > 0:
            return False
        if not is_synced_value(subtitles[-1].start_time):
            return False
        return True

    def get_widget_url(self, mode=None, task_id=None):
        # duplicates unisubs.widget.SubtitleDialogOpener.prototype.openDialogOrRedirect_
        video = self.video
        video_url = video.get_video_url()
        config = {
            "videoID": video.video_id,
            "videoURL": video_url,
            "effectiveVideoURL": video_url,
            "languageCode": self.language,
            "subLanguagePK": self.pk,
            "originalLanguageCode": video.language,
            "mode": mode,
            "task": task_id, }
        if self.is_dependent():
            config['baseLanguagePK'] = self.standard_language and self.standard_language.pk
        return reverse('onsite_widget')+'?config='+urlquote_plus(json.dumps(config))

    @models.permalink
    def get_absolute_url(self):
        return ('videos:translation_history',
                [self.video.video_id, self.language or 'unknown', self.pk])

    def language_display(self):
        if self.is_original and not self.language:
            return 'Original'
        return self.get_language_display()

    @property
    def writelock_owner_name(self):
        if self.writelock_owner == None:
            return "anonymous"
        else:
            return self.writelock_owner.__unicode__()

    @property
    def is_writelocked(self):
        if self.writelock_time == None:
            return False
        delta = datetime.now() - self.writelock_time
        seconds = delta.days * 24 * 60 * 60 + delta.seconds
        return seconds < WRITELOCK_EXPIRATION

    def is_rtl(self):
        from utils.translation import is_rtl
        return is_rtl(self.language)

    def can_writelock(self, request):
        return self.writelock_session_key == \
            request.browser_id or \
            not self.is_writelocked

    def writelock(self, request):
        if request.user.is_authenticated():
            self.writelock_owner = request.user
        else:
            self.writelock_owner = None
        self.writelock_session_key = request.browser_id
        self.writelock_time = datetime.now()

    def release_writelock(self):
        self.writelock_owner = None
        self.writelock_session_key = ''
        self.writelock_time = None

    def _filter_public(self, versions, public_only):
        if public_only:
            versions = versions.filter(moderation_status__in=[APPROVED, UNMODERATED])
        return versions

    def version(self, version_no=None, public_only=True):
        if version_no is None:
            return self.latest_version(public_only)
        try:
            return self._filter_public(self.subtitleversion_set.filter(version_no=version_no), public_only)[0]
        except (models.ObjectDoesNotExist, IndexError):
            pass

    @property
    def last_version(self):
        return self.latest_version(public_only=True)

    def latest_version(self, public_only=True):
        try:
            return self._filter_public(self.subtitleversion_set.all(), public_only)[0]
        except (SubtitleVersion.DoesNotExist, IndexError):
            return None

    def latest_subtitles(self, public_only=True):
        version = self.latest_version(public_only=public_only)
        if version:
            return version.subtitles(public_only=public_only)
        return []

    def notification_list(self, exclude=None):
        qs = self.followers.filter(notify_by_email=True, is_active=True)

        if exclude:
            if not isinstance(exclude, (list, tuple)):
                exclude = [exclude]
            qs = qs.exclude(pk__in=[u.pk for u in exclude if u])
        return qs

    def translations(self):
        return SubtitleLanguage.objects.filter(video=self.video, is_original=False, is_forked=False)

    def fork(self, from_version=None, user=None, result_of_rollback=False,
             attach_to_language=None, bypass_writelock=False):
        """
        If this a dependent language, fork it, making all it's subs
        timing not depend on the original source.
        If locked, will throw an AlreadyEditingException
        unless you pass bypass_writelock.
        If attach_to_language is passed, we will copy those
        subs to a new language, else self will be used
        """
        to_language = attach_to_language or self
        if from_version:
            original_subs = from_version.subtitle_set.all()
        else:
            if self.standard_language is None:
                return
            original_subs = self.standard_language.latest_version().subtitle_set.all()

        if self.is_writelocked and not bypass_writelock:
            raise AlreadyEditingException(_("Sorry, you cannot upload subtitles right now because someone is editing the language you are uploading or a translation of it"))
        try:
            old_version = self.subtitleversion_set.all()[:1].get()
            version_no = old_version.version_no + 1
        except SubtitleVersion.DoesNotExist:
            old_version = None
            version_no = 0

        kwargs = dict(
            language=to_language, version_no=version_no,
            datetime_started=datetime.now(),
            note=u'Uploaded', is_forked=True, time_change=1,
            text_change=1, result_of_rollback=result_of_rollback,
            forked_from=from_version)
        if user:
            kwargs['user'] = user
        version = SubtitleVersion(**kwargs)
        version.save()

        if old_version:
{'fraction_non_alphanumeric': 0.08110631200288776, 'fraction_numerical': 0.004281855636739317, 'mean_word_length': 2.5152045504266023, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '729617', 'n_tokens_mistral': 23216, 'n_tokens_neox': 22209, 'n_words': 6470}