<commit_before>import logging

from django.db import transaction


logger = logging.getLogger(__name__)


class Normalizer:

    def __init__(self, app_config):
        self.config = app_config

    def normalize(self, raw_data):
        from share.models import Normalization
        from share.models import NormalizationQueue
        with transaction.atomic():
            try:
                NormalizationQueue(data=raw_data).delete()
            except NormalizationQueue.NotFound:
                pass
            Normalization(data=raw_data).save()

    def blocks(self, size=50):
        from share.models import NormalizationQueue
        ids = NormalizationQueue.objects.values_list('data_id', flat=True).filter(data__source=self.config.as_source(), )
        for i in range(0, len(ids), size):
            yield ids[i:i+50]<commit_msg>Normalizers updates<commit_after>import logging

from django.db import transaction


logger = logging.getLogger(__name__)


class Normalizer:

    def __init__(self, app_config):
        self.config = app_config

    def normalize(self, raw_data):
        # from share.models import Normalization
        from share.models import NormalizationQueue
        with transaction.atomic():
            try:
                models = self.do_normalize(raw_data)
                # NormalizationQueue(data=raw_data).delete()
            except NormalizationQueue.DoesNotExist:
                pass

        print(raw_data)
        while models:
            x = models.pop(0)
            x.source = self.config.as_source()
            for field in x._meta.fields:
                if 'version' not in field.name:
                    setattr(x, field.name, getattr(x, field.name))
            try:
                x.save()
            except Exception as e:
                print(e)
                models.append(x)
            # Normalization(data=raw_data).save()

    def blocks(self, size=50):
        from share.models import NormalizationQueue
        ids = NormalizationQueue.objects.values_list('data_id', flat=True).filter(data__source=self.config.as_source(), )
        for i in range(0, len(ids), size):
            yield ids[i:i+50]
{'fraction_non_alphanumeric': 0.08999081726354453, 'fraction_numerical': 0.005050505050505051, 'mean_word_length': 2.2091310751104567, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1970169', 'n_tokens_mistral': 601, 'n_tokens_neox': 573, 'n_words': 134}