<commit_before>from appfd.forms import BasemapForm, FileForm, LinkForm, TextForm, TimezoneForm, TweetForm
from django.core.files.uploadedfile import SimpleUploadedFile

def clean_tweet(POST, debug=True):
    try:
        if debug: print "starting clean_tweet with", POST
        if "text" in POST:
            form = TweetForm({"text": POST["text"]})
            if form.is_valid():
                if debug: print "form is valid"
                result = {}
                result['text'] = form.cleaned_data['text']
                return result

    except Exception as e:
        print "CAUGHT EXCEPTION in clean_tweet:", e

def clean(POST, FILES, debug=False):
    try:
        print "starting clean"

        style = {}
        if "basemap" in POST:
            basemapForm = BasemapForm({"basemap": POST['basemap']})
            if basemapForm.is_valid():
                style['basemap'] = basemapForm.cleaned_data['basemap']
        if debug: print "style after cleaning:", style

        extra_context = {}
        if "end_user_timezone" in POST:
            if debug: print "end_user_timezone before cleaning:", POST['end_user_timezone']
            timezoneForm = TimezoneForm({"timezone": POST['end_user_timezone']})
            timezoneForm.full_clean()
            if debug: print "cleaned_data after clean:", timezoneForm.cleaned_data
            end_user_timezone = timezoneForm.cleaned_data.get("timezone", None)
            if end_user_timezone:
                if debug: print "zone:", end_user_timezone.zone
                extra_context["end_user_timezone"] = end_user_timezone.zone

        if "case_insensitive" in POST:
            case_insensitive_value = POST["case_insensitive"]
            if case_insensitive_value in [True, "True", "true"]:
                extra_context['case_insensitive'] = True

        map_format = POST['map_format'] if POST.get("map_format", None) in ["geojson", "gif", "jpg", "png", "xy"] else "all"

        sources = []
        for n in range(1000):
            key = "source_" + str(n) + "_type"
            #print 'key:', key
            #print "post:", POST 
            if key in POST:
                source_type = POST[key]
                print "source_type:", source_type
                data_key = "source_" + str(n) + "_data"
                print data_key 
                if source_type == "file":
                    if data_key in FILES:
                        print "FILES:", FILES
                        data = FILES[data_key]
                        # post data goes 
                        form = FileForm(files={"data": data})
                        if form.is_valid():
                            cleaned_data = form.cleaned_data['data']<commit_msg>added open_source to cleaning<commit_after>from appfd.forms import BasemapForm, FileForm, LinkForm, TextForm, TimezoneForm, TweetForm
from django.core.files.uploadedfile import SimpleUploadedFile


trues = [True, "True", "true", "t", "T", 1, "1"]

def clean_tweet(POST, debug=True):
    try:
        if debug: print "starting clean_tweet with", POST
        if "text" in POST:
            form = TweetForm({"text": POST["text"]})
            if form.is_valid():
                if debug: print "form is valid"
                result = {}
                result['text'] = form.cleaned_data['text']
                return result

    except Exception as e:
        print "CAUGHT EXCEPTION in clean_tweet:", e

def clean(POST, FILES, debug=False):
    try:
        print "starting clean"

        style = {}
        if "basemap" in POST:
            basemapForm = BasemapForm({"basemap": POST['basemap']})
            if basemapForm.is_valid():
                style['basemap'] = basemapForm.cleaned_data['basemap']
        if debug: print "style after cleaning:", style

        extra_context = {}
        if "end_user_timezone" in POST:
            if debug: print "end_user_timezone before cleaning:", POST['end_user_timezone']
            timezoneForm = TimezoneForm({"timezone": POST['end_user_timezone']})
            timezoneForm.full_clean()
            if debug: print "cleaned_data after clean:", timezoneForm.cleaned_data
            end_user_timezone = timezoneForm.cleaned_data.get("timezone", None)
            if end_user_timezone:
                if debug: print "zone:", end_user_timezone.zone
                extra_context["end_user_timezone"] = end_user_timezone.zone

        for text in ["case_insensitive", "open_source"]:
            if POST.get(text, False) in trues:
                extra_context[text] = True


        map_format = POST['map_format'] if POST.get("map_format", None) in ["geojson", "gif", "jpg", "png", "xy"] else "all"

        sources = []
        for n in range(1000):
            key = "source_" + str(n) + "_type"
            #print 'key:', key
            #print "post:", POST 
            if key in POST:
                source_type = POST[key]
                print "source_type:", source_type
                data_key = "source_" + str(n) + "_data"
                print data_key 
                if source_type == "file":
                    if data_key in FILES:
                        print "FILES:", FILES
                        data = FILES[data_key]
                        # post data goes 
                        form = FileForm(files={"data": data})
                        if form.is_valid():
                            cleaned_data = form.cleaned_data['data']
{'fraction_non_alphanumeric': 0.10913426265590609, 'fraction_numerical': 0.001834189288334556, 'mean_word_length': 1.9098185699039487, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2992652', 'n_tokens_mistral': 1624, 'n_tokens_neox': 1522, 'n_words': 398}