<commit_before>from django import forms
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _

from .models import DataSource


class DataSourceCreateForm(forms.ModelForm):
    class Meta:
        model = DataSource
        fields = [
            'owner', 'file_path', 'checksum',
        ]
        checksum_field = DataSource._meta.get_field('checksum')

    def clean_checksum(self):
        checksum = self.cleaned_data['checksum']
        return checksum.lower()

    def clean(self):
        """Validate the data source existed and has correct checksum.

        The file path has the pattern

        DATA_SOURCES_DIR / owner.pk / file_path
        """
        if self.errors:
            return  # no owner given, skip rest of the check
        owner = self.cleaned_data['owner']
        file_path = self.cleaned_data['file_path']
        checksum = self.cleaned_data['checksum']

        full_file_path = settings.BIOCLOUD_DATA_SOURCES_DIR.joinpath(
            str(owner.pk), file_path
        )
        # First assert the file path exists
        if not full_file_path.exists():
            raise ValidationError(
                _('Path %(full_file_path)s does not exist'),
                params={'full_file_path': full_file_path},
                code='file_path_not_exist',
            )

        if checksum:
            # A checksum is provided. Check if it matches with the checksum
            # computed from the file content.
            checksum_from_file = self.Meta.checksum_field.checksum_for_file(
                full_file_path
            )
            if checksum != checksum_from_file:
                raise ValidationError(
                    _(
                        'Checksum from input (%(model_checksum)s) mismatches '
                        'with the checksum computed from the file content '
                        '(%(file_checksum)s).'
                    ),
                    params={
                        'model_checksum': checksum,
                        'file_checksum': checksum_from_file,
                    },
                    code='checksum_mismatch',
                )

<commit_msg>Fix DataSource does not call clean properly<commit_after>from django import forms
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _

from .models import DataSource


class DataSourceCreateForm(forms.ModelForm):
    class Meta:
        model = DataSource
        fields = [
            'owner', 'file_path', 'checksum',
        ]
        checksum_field = DataSource._meta.get_field('checksum')

    def clean_checksum(self):
        checksum = self.cleaned_data['checksum']
        return checksum.lower()

    def clean(self):
        """Validate the data source existed and has correct checksum.

        The file path has the pattern

        DATA_SOURCES_DIR / owner.pk / file_path
        """
        super().clean()
        if self.errors:
            return  # no owner given, skip rest of the check
        owner = self.cleaned_data['owner']
        file_path = self.cleaned_data['file_path']
        checksum = self.cleaned_data['checksum']

        full_file_path = settings.BIOCLOUD_DATA_SOURCES_DIR.joinpath(
            str(owner.pk), file_path
        )
        # First assert the file path exists
        if not full_file_path.exists():
            raise ValidationError(
                _('Path %(full_file_path)s does not exist'),
                params={'full_file_path': full_file_path},
                code='file_path_not_exist',
            )

        if checksum:
            # A checksum is provided. Check if it matches with the checksum
            # computed from the file content.
            checksum_from_file = self.Meta.checksum_field.checksum_for_file(
                full_file_path
            )
            if checksum != checksum_from_file:
                raise ValidationError(
                    _(
                        'Checksum from input (%(model_checksum)s) mismatches '
                        'with the checksum computed from the file content '
                        '(%(file_checksum)s).'
                    ),
                    params={
                        'model_checksum': checksum,
                        'file_checksum': checksum_from_file,
                    },
                    code='checksum_mismatch',
                )


{'fraction_non_alphanumeric': 0.07554275587062473, 'fraction_numerical': 0.0, 'mean_word_length': 1.7446808510638299, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2819121', 'n_tokens_mistral': 1210, 'n_tokens_neox': 1150, 'n_words': 297}