<commit_before>from os import path
import urlparse

from django import forms
from lxml.html.clean import clean_html

from localtv import models
from localtv import util


# common cleaning methods
def clean_tags(self):
    tags = []
    for tag_text in self.cleaned_data['tags'].strip().split(','):
        # make sure there's only one space in each tag
        cleaned_tag = ' '.join(tag_text.strip().split())
        if len(cleaned_tag) > 25:
            raise forms.ValidationError(
                'Tags cannot be greater than 25 characters in length')
        tags.append(cleaned_tag)

    return tags

def clean_description(self):
    return clean_html(self.cleaned_data['description'])


class SubmitVideoForm(forms.Form):
    url = forms.URLField()
    tags = forms.CharField(required=False)

    clean_tags = clean_tags

class ScrapedSubmitVideoForm(forms.Form):
    website_url = forms.URLField()
    tags = forms.CharField()
    file_url = forms.URLField(widget=forms.HiddenInput, required=False)
    embed = forms.CharField(widget=forms.HiddenInput, required=False)
    name = forms.CharField(max_length=250)
    description = forms.CharField(widget=forms.widgets.Textarea)
    #categories = forms.ModelMultipleChoiceField(models.Category.objects.all())

    clean_tags = clean_tags
    clean_description = clean_description
<commit_msg>Clearly tags shouldn't be required here as they aren't for the first step<commit_after>from os import path
import urlparse

from django import forms
from lxml.html.clean import clean_html

from localtv import models
from localtv import util


# common cleaning methods
def clean_tags(self):
    tags = []
    for tag_text in self.cleaned_data['tags'].strip().split(','):
        # make sure there's only one space in each tag
        cleaned_tag = ' '.join(tag_text.strip().split())
        if len(cleaned_tag) > 25:
            raise forms.ValidationError(
                'Tags cannot be greater than 25 characters in length')
        tags.append(cleaned_tag)

    return tags

def clean_description(self):
    return clean_html(self.cleaned_data['description'])


class SubmitVideoForm(forms.Form):
    url = forms.URLField()
    tags = forms.CharField(required=False)

    clean_tags = clean_tags

class ScrapedSubmitVideoForm(forms.Form):
    website_url = forms.URLField()
    tags = forms.CharField(required=False)
    file_url = forms.URLField(widget=forms.HiddenInput, required=False)
    embed = forms.CharField(widget=forms.HiddenInput, required=False)
    name = forms.CharField(max_length=250)
    description = forms.CharField(widget=forms.widgets.Textarea)
    #categories = forms.ModelMultipleChoiceField(models.Category.objects.all())

    clean_tags = clean_tags
    clean_description = clean_description

{'fraction_non_alphanumeric': 0.09415584415584416, 'fraction_numerical': 0.005050505050505051, 'mean_word_length': 3.890652557319224, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '655588', 'n_tokens_mistral': 819, 'n_tokens_neox': 788, 'n_words': 202}