<commit_before>import hashlib
import logging
from django import forms
from django.core.validators import (
    RegexValidator, MinLengthValidator, MaxLengthValidator
)
from django.db import models
from django.utils.translation import ugettext_lazy as _

logger = logging.getLogger(__name__)


class SHA256ChecksumField(models.CharField):
    default_validators = [
        MinLengthValidator(64),
        MaxLengthValidator(64),
        RegexValidator(
            regex=r'^[A-Fa-f0-9]+$',
            message=_(
                'Invalid SHA256 checksum format, expected only '
                'hexadecimal digits'
            ),
            code='sha256sum_non_hexadecimal',
        ),
    ]
    description = _("SHA256 checksum")

    def __init__(self, *args, blank=False, **kwargs):
        # SHA-256 checksum in hexadecimal representation takes up 64 chars,
        # e.g. 4bit * 64 = 256bit (1 char represents only 4 bit here)
        kwargs['max_length'] = 64
        super().__init__(*args, blank=blank, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super().deconstruct()
        del kwargs['max_length']<commit_msg>Use custom ExactLengthValidator for checksum<commit_after>import hashlib
import logging
from django import forms
from django.core.validators import RegexValidator, BaseValidator
from django.db import models
from django.utils.deconstruct import deconstructible
from django.utils.translation import ugettext_lazy as _, ungettext_lazy

logger = logging.getLogger(__name__)


@deconstructible
class ExactLengthValidator(BaseValidator):
    compare = lambda self, a, b: a != b
    clean = lambda self, x: len(x)
    message = ungettext_lazy(
        'Ensure this value has exact %(limit_value)d character '
        '(it has %(show_value)d).',
        'Ensure this value has exact %(limit_value)d characters '
        '(it has %(show_value)d).',
        'limit_value'
    )
    code = 'limit_value'


class SHA256ChecksumField(models.CharField):
    default_validators = [
        ExactLengthValidator(64),
        RegexValidator(
            regex=r'^[A-Fa-f0-9]+$',
            message=_(
                'Invalid SHA256 checksum format, expected only '
                'hexadecimal digits'
            ),
            code='sha256sum_non_hexadecimal',
        ),
    ]
    description = _("SHA256 checksum")

    def __init__(self, *args, blank=False, **kwargs):
        # SHA-256 checksum in hexadecimal representation takes up 64 chars,
        # e.g. 4bit * 64 = 256bit (1 char represents only 4 bit here)
        kwargs['max_length'] = 64
        super().__init__(*args, blank=blank, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super().deconstruct()
        del kwargs['max_length']
{'fraction_non_alphanumeric': 0.0999275887038378, 'fraction_numerical': 0.023171614771904415, 'mean_word_length': 2.8589385474860336, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7629199', 'n_tokens_mistral': 855, 'n_tokens_neox': 793, 'n_words': 214}