<commit_before># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.
import random
import re
from datetime import datetime
from uuid import uuid4
from xml.dom.minidom import parseString
from xml.parsers.expat import ExpatError
import bleach

from django.db.models import ObjectDoesNotExist
from django.utils import simplejson as json
from lxml import etree

from utils.html import unescape as unescape_html


# see video.models.Subtitle..start_time
MAX_SUB_TIME = (60 * 60 * 100) - 1
DEFAULT_ALLOWED_TAGS = ['i', 'b', 'u']
def is_version_same(version, parser):
    if not version:
        return False

    subtitles = list(parser)

    if version.subtitle_set.count() != len(subtitles):
        return False

    for item in zip(subtitles, list(version.subtitle_set.all())):
        if item[0]['subtitle_text'] != item[1].subtitle_text or \
            item[0]['start_time'] != item[1].start_time or \
            item[0]['end_time'] != item[1].end_time:
                return False

    return True

def strip_tags(text, tags=None):
    """
    Returns text with the tags stripped.
    By default we allow the standard formatting tags
    to pass (i,b,u).
    Any other tag's content will be present, but with tags removed.
    """
    if tags is None:
        tags = DEFAULT_ALLOWED_TAGS
    return bleach.clean(text, tags=tags, strip=True)
    
def save_subtitle(video, language, parser, user=None, update_video=True,
                  forks=True, as_forked=True, translated_from=None):
    from videos.models import SubtitleVersion, Subtitle, SubtitleMetadata
    from videos.tasks import video_changed_tasks
    from utils.unisubsmarkup import strip_tags

    key = str(uuid4()).replace('-', '')

    if language.is_original:
        as_forked = True

    video._make_writelock(user, key)
    video.save()

    try:
        old_version = language.subtitleversion_set.all()[:1].get()
        version_no = old_version.version_no + 1
    except ObjectDoesNotExist:
        old_version = None
        version_no = 0

    version = None

    if not is_version_same(old_version, parser):
        forked_from = as_forked and translated_from and translated_from.version()

        title = old_version.title if old_version and old_version.title else video.get_title_display()
        description = old_version.description if old_version and old_version.description else video.get_description_display()
<commit_msg>Fixed missing import<commit_after># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.
import random
import re
from datetime import datetime
from uuid import uuid4
from xml.dom.minidom import parseString
from xml.parsers.expat import ExpatError
import bleach

from django.db.models import ObjectDoesNotExist
from django.utils import simplejson as json
from lxml import etree

from utils.html import unescape as unescape_html


# see video.models.Subtitle..start_time
MAX_SUB_TIME = (60 * 60 * 100) - 1
DEFAULT_ALLOWED_TAGS = ['i', 'b', 'u']
def is_version_same(version, parser):
    if not version:
        return False

    subtitles = list(parser)

    if version.subtitle_set.count() != len(subtitles):
        return False

    for item in zip(subtitles, list(version.subtitle_set.all())):
        if item[0]['subtitle_text'] != item[1].subtitle_text or \
            item[0]['start_time'] != item[1].start_time or \
            item[0]['end_time'] != item[1].end_time:
                return False

    return True

def strip_tags(text, tags=None):
    """
    Returns text with the tags stripped.
    By default we allow the standard formatting tags
    to pass (i,b,u).
    Any other tag's content will be present, but with tags removed.
    """
    if tags is None:
        tags = DEFAULT_ALLOWED_TAGS
    return bleach.clean(text, tags=tags, strip=True)
    
def save_subtitle(video, language, parser, user=None, update_video=True,
                  forks=True, as_forked=True, translated_from=None):
    from videos.models import SubtitleVersion, Subtitle, SubtitleMetadata
    from videos.tasks import video_changed_tasks

    key = str(uuid4()).replace('-', '')

    if language.is_original:
        as_forked = True

    video._make_writelock(user, key)
    video.save()

    try:
        old_version = language.subtitleversion_set.all()[:1].get()
        version_no = old_version.version_no + 1
    except ObjectDoesNotExist:
        old_version = None
        version_no = 0

    version = None

    if not is_version_same(old_version, parser):
        forked_from = as_forked and translated_from and translated_from.version()

        title = old_version.title if old_version and old_version.title else video.get_title_display()
        description = old_version.description if old_version and old_version.description else video.get_description_display()

{'fraction_non_alphanumeric': 0.08155684754521964, 'fraction_numerical': 0.008397932816537468, 'mean_word_length': 3.5204379562043795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5788151', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1778, 'n_words': 618}