<commit_before># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

from teams.models import TeamVideo
from datetime import datetime
from utils.metrics import Meter, Timer

def update_metadata(video_pk):
    from videos.models import Video
    with Timer('metadata-update-time'):
        video = Video.objects.get(pk=video_pk)
        video.edited = datetime.now()
        video.save()
        _update_is_public(video)
        _update_subtitle_counts(video)
        _update_percent_done(video)
        _update_has_had_version(video)
        _update_is_was_subtitled(video)
        _update_languages_count(video)
        _update_complete_date(video)
        _invalidate_cache(video)

def _update_changes_on_version(version, last_version):
    new_subtitles = version.subtitles()
    subs_length = len(new_subtitles)

    if not last_version:
        return 1, 1
    elif subs_length == 0:
        old_subs_length = last_version.subtitle_set.count()
        time_change = 0 if old_subs_length == 0 else 1
        text_change = version.time_change
        return time_change, text_change
    else:
        return _update_changes_on_nonzero_version(version, last_version)

def _update_changes_on_nonzero_version(version, last_version):
    subtitles = version.subtitles()
    last_subtitles = dict([(item.subtitle_id, item)
                           for item in last_version.subtitles()])
    time_count_changed, text_count_changed = 0, 0
    new_subtitles_ids = set()

    for subtitle in subtitles:
        new_subtitles_ids.add(subtitle.subtitle_id)
        if subtitle.subtitle_id in last_subtitles:
            last_subtitle = last_subtitles[subtitle.subtitle_id]
            if not last_subtitle.text == subtitle.text:
                text_count_changed += 1
            if not subtitle.has_same_timing(last_subtitle):
                time_count_changed += 1
        else:
            time_count_changed += 1
            text_count_changed += 1

    for subtitle_id in last_subtitles.keys():
        if subtitle_id not in new_subtitles_ids:
            text_count_changed += 1
            time_count_changed += 1

    subs_length = len(subtitles)
    time_change = min(time_count_changed / 1. / subs_length, 1)
    text_change = min(text_count_changed / 1. / subs_length, 1)

    return time_change, text_change

def _update_subtitle_counts(video):
    from videos.models import Subtitle
    for sl in video.subtitlelanguage_set.all():
        original_value = sl.subtitle_count
        new_value = sl.nonblank_subtitle_count() or 0

        if original_value != new_value:
            sl.subtitle_count = new_value
            sl.save()

def _update_percent_done(video):
    for sl in video.subtitlelanguage_set.all():
        percent_done = sl.calculate_percent_done()

        if percent_done is None:
            continue

        if percent_done != sl.percent_done:
            sl.percent_done = percent_done
            sl.is_complete = percent_done == 100
            sl.save()

def _update_has_had_version(video):
    for sl in video.subtitlelanguage_set.all():
        version = sl.latest_version()
        if not version:
            sl.had_version = False
            sl.has_version = False
            sl.save()
        elif len(version.subtitles()) == 0:
            if sl.has_version:
                sl.has_version = False
                sl.save()
        else:
            if not sl.has_version or not sl.had_version:
                sl.had_version = True<commit_msg>We don't have subtitle_count on SubtitleLangauge anymore<commit_after># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

from teams.models import TeamVideo
from datetime import datetime
from utils.metrics import Meter, Timer

def update_metadata(video_pk):
    from videos.models import Video
    with Timer('metadata-update-time'):
        video = Video.objects.get(pk=video_pk)
        video.edited = datetime.now()
        video.save()
        _update_is_public(video)
        _update_percent_done(video)
        _update_has_had_version(video)
        _update_is_was_subtitled(video)
        _update_languages_count(video)
        _update_complete_date(video)
        _invalidate_cache(video)

def _update_changes_on_version(version, last_version):
    new_subtitles = version.subtitles()
    subs_length = len(new_subtitles)

    if not last_version:
        return 1, 1
    elif subs_length == 0:
        old_subs_length = last_version.subtitle_set.count()
        time_change = 0 if old_subs_length == 0 else 1
        text_change = version.time_change
        return time_change, text_change
    else:
        return _update_changes_on_nonzero_version(version, last_version)

def _update_changes_on_nonzero_version(version, last_version):
    subtitles = version.subtitles()
    last_subtitles = dict([(item.subtitle_id, item)
                           for item in last_version.subtitles()])
    time_count_changed, text_count_changed = 0, 0
    new_subtitles_ids = set()

    for subtitle in subtitles:
        new_subtitles_ids.add(subtitle.subtitle_id)
        if subtitle.subtitle_id in last_subtitles:
            last_subtitle = last_subtitles[subtitle.subtitle_id]
            if not last_subtitle.text == subtitle.text:
                text_count_changed += 1
            if not subtitle.has_same_timing(last_subtitle):
                time_count_changed += 1
        else:
            time_count_changed += 1
            text_count_changed += 1

    for subtitle_id in last_subtitles.keys():
        if subtitle_id not in new_subtitles_ids:
            text_count_changed += 1
            time_count_changed += 1

    subs_length = len(subtitles)
    time_change = min(time_count_changed / 1. / subs_length, 1)
    text_change = min(text_count_changed / 1. / subs_length, 1)

    return time_change, text_change

def _update_percent_done(video):
    for sl in video.subtitlelanguage_set.all():
        percent_done = sl.calculate_percent_done()

        if percent_done is None:
            continue

        if percent_done != sl.percent_done:
            sl.percent_done = percent_done
            sl.is_complete = percent_done == 100
            sl.save()

def _update_has_had_version(video):
    for sl in video.subtitlelanguage_set.all():
        version = sl.latest_version()
        if not version:
            sl.had_version = False
            sl.has_version = False
            sl.save()
        elif len(version.subtitles()) == 0:
            if sl.has_version:
                sl.has_version = False
                sl.save()
        else:
            if not sl.has_version or not sl.had_version:
                sl.had_version = True
{'fraction_non_alphanumeric': 0.07018849206349206, 'fraction_numerical': 0.007316468253968254, 'mean_word_length': 2.7114588127013346, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1532059', 'n_tokens_mistral': 2437, 'n_tokens_neox': 2357, 'n_words': 640}