<commit_before>from django.contrib.contenttypes.generic import GenericForeignKey
from django.db import models
from django.db.models import signals
from django.contrib.contenttypes.models import ContentType
from django.utils.translation import ugettext_lazy as _
from django.utils.importlib import import_module

from djangomarkup.processors import ProcessorConfigurationError, ProcessorError

#class RichTextField(models.TextField):
#    def __init__(self, syntax, *args, **kwargs):
#        self.syntax = syntax
#        super(RichTextField, self).__init__(*args, **kwargs)
#
#    def formfield(self, **kwargs):
#        from djangomarkup.fields import RichTextField as RichTextFormField
#        kwargs.update({
#            'syntax' : self.syntax,
#            'field_name' : self.name,
#
#        })
#        return RichTextFormField(**kwargs)

class TextProcessor(models.Model):
    """
    Text processors are used to convert user-inputted source text in given processor format
    (i.e. Markdown, Czechtile, ...) and convert them into target format
    (now XHTML is assumed)
    """
    function = models.CharField(max_length=96, unique=True)
    name = models.CharField(max_length=96, blank=True)
    processor_options = models.CharField(max_length=255, blank=True)

    def __unicode__(self):
        return self.name

    def get_function(self):
        """
        Return function object for my function.
        raise ProcessorConfigurationError when function could not be resolved.
        """
        if not hasattr(self, '_function'):
            try:
                modname, funcname = self.function.rsplit('.', 1)
                mod = import_module(modname)
                self._function = getattr(mod, funcname)
            except (ImportError, AttributeError, ValueError), err:
                raise ProcessorConfigurationError(err)

        return self._function

    def convert(self, src_txt):
        function = self.get_function()
        try:
            return function(src_txt)
        except Exception, err:
            raise ProcessorError(err)

    class Meta:
        verbose_name = (_('Text processor'))
        verbose_name_plural = (_('Text processors'))


class SourceTextManager(models.Manager):
    def delete_for_object(self, instance):
        try:
            pk = int(instance.pk)
        except ValueError:
            return
        
        self.filter(
            content_type=ContentType.objects.get_for_model(instance),
            object_id=pk,
        ).delete()

    def extract_from_instance(self, instance, processor, fields, content_type=None):
        if not content_type:
            content_type = ContentType.objects.get_for_model(instance)

        dirty = False
        for f in fields:
            st, created = self.get_or_create(
                        content_type=content_type,
                        object_id=instance.pk,
                        field=f,
                        defaults={
                            'processor': processor,
                            'content': getattr(instance, f),
                        }
                    )
            if created:
                setattr(instance, f, st.render())
                dirty = True
        if dirty:
            instance.save(force_update=True)

    def extract_from_model(self, model, processor, fields):
        if not fields:
            return

        ct = ContentType.objects.get_for_model(model)

        for m in model._default_manager.all():
            self.extract_from_instance(m, processor, fields, ct)


class SourceText(models.Model):
    """<commit_msg>added force_save parameter<commit_after>from django.contrib.contenttypes.generic import GenericForeignKey
from django.db import models
from django.db.models import signals
from django.contrib.contenttypes.models import ContentType
from django.utils.translation import ugettext_lazy as _
from django.utils.importlib import import_module

from djangomarkup.processors import ProcessorConfigurationError, ProcessorError

#class RichTextField(models.TextField):
#    def __init__(self, syntax, *args, **kwargs):
#        self.syntax = syntax
#        super(RichTextField, self).__init__(*args, **kwargs)
#
#    def formfield(self, **kwargs):
#        from djangomarkup.fields import RichTextField as RichTextFormField
#        kwargs.update({
#            'syntax' : self.syntax,
#            'field_name' : self.name,
#
#        })
#        return RichTextFormField(**kwargs)

class TextProcessor(models.Model):
    """
    Text processors are used to convert user-inputted source text in given processor format
    (i.e. Markdown, Czechtile, ...) and convert them into target format
    (now XHTML is assumed)
    """
    function = models.CharField(max_length=96, unique=True)
    name = models.CharField(max_length=96, blank=True)
    processor_options = models.CharField(max_length=255, blank=True)

    def __unicode__(self):
        return self.name

    def get_function(self):
        """
        Return function object for my function.
        raise ProcessorConfigurationError when function could not be resolved.
        """
        if not hasattr(self, '_function'):
            try:
                modname, funcname = self.function.rsplit('.', 1)
                mod = import_module(modname)
                self._function = getattr(mod, funcname)
            except (ImportError, AttributeError, ValueError), err:
                raise ProcessorConfigurationError(err)

        return self._function

    def convert(self, src_txt):
        function = self.get_function()
        try:
            return function(src_txt)
        except Exception, err:
            raise ProcessorError(err)

    class Meta:
        verbose_name = (_('Text processor'))
        verbose_name_plural = (_('Text processors'))


class SourceTextManager(models.Manager):
    def delete_for_object(self, instance):
        try:
            pk = int(instance.pk)
        except ValueError:
            return
        
        self.filter(
            content_type=ContentType.objects.get_for_model(instance),
            object_id=pk,
        ).delete()

    def extract_from_instance(self, instance, processor, fields, content_type=None, force_save=False):
        if not content_type:
            content_type = ContentType.objects.get_for_model(instance)

        dirty = False
        for f in fields:
            st, created = self.get_or_create(
                        content_type=content_type,
                        object_id=instance.pk,
                        field=f,
                        defaults={
                            'processor': processor,
                            'content': getattr(instance, f),
                        }
                    )
            if created:
                setattr(instance, f, st.render())
                dirty = True
        if dirty or force_save:
            instance.save(force_update=True)

    def extract_from_model(self, model, processor, fields):
        if not fields:
            return

        ct = ContentType.objects.get_for_model(model)

        for m in model._default_manager.all():
            self.extract_from_instance(m, processor, fields, ct)


class SourceText(models.Model):
    """
{'fraction_non_alphanumeric': 0.08772171043585865, 'fraction_numerical': 0.002199917503093634, 'mean_word_length': 2.195957820738137, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '199206', 'n_tokens_mistral': 1953, 'n_tokens_neox': 1874, 'n_words': 473}