<commit_before>import csv

from django import forms
from django.contrib.contenttypes.models import ContentType
from django.core.files.base import ContentFile
from django.conf import settings

from csvimporter.models import CSV

class CSVForm(forms.ModelForm):
    class Meta:
        model = CSV
    exclude_types = getattr(settings, 'CSVIMPORTER_EXCLUDE', [])
    # TODO: this could be so much nicer.
    content_types = ContentType.objects.all()
    for t in exclude_types:
        if '.' in t:
            content_types = content_types.exclude(app_label__iexact=t.split('.')[0], model__iexact=t.split('.')[1].lower())
        else:
            content_types = content_types.exclude(app_label__iexact=t)
    content_type = forms.ModelChoiceField(queryset=content_types)
    
key_to_field_map = getattr(settings, 'CSVIMPORTER_KEY_TO_FIELD_MAP', lambda k: k.replace(' ','_').lower())
class CSVAssociateForm(forms.Form):
    def __init__(self, instance, *args, **kwargs):
        self.instance = instance
        self.reader = csv.DictReader(instance.csv_file)<commit_msg>fixed a bug with trailing spaces in fieldnames<commit_after>import csv

from django import forms
from django.contrib.contenttypes.models import ContentType
from django.core.files.base import ContentFile
from django.conf import settings

from csvimporter.models import CSV

class CSVForm(forms.ModelForm):
    class Meta:
        model = CSV
    exclude_types = getattr(settings, 'CSVIMPORTER_EXCLUDE', [])
    # TODO: this could be so much nicer.
    content_types = ContentType.objects.all()
    for t in exclude_types:
        if '.' in t:
            content_types = content_types.exclude(app_label__iexact=t.split('.')[0], model__iexact=t.split('.')[1].lower())
        else:
            content_types = content_types.exclude(app_label__iexact=t)
    content_type = forms.ModelChoiceField(queryset=content_types)
    
key_to_field_map = getattr(settings, 'CSVIMPORTER_KEY_TO_FIELD_MAP', lambda k: k.replace(' ','_').lower())
class CSVAssociateForm(forms.Form):
    def __init__(self, instance, *args, **kwargs):
        self.instance = instance
        # BUGFIX: this removes trailing spaces in fieldnames.
        instance.csv_file.file[0] = re.sub('\s+,', ',', instance.csv_file.file[0])
        self.reader = csv.DictReader(instance.csv_file)
{'fraction_non_alphanumeric': 0.11077586206896552, 'fraction_numerical': 0.002586206896551724, 'mean_word_length': 3.9594017094017095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1507471', 'n_tokens_mistral': 732, 'n_tokens_neox': 711, 'n_words': 162}