<commit_before># -*- encoding: utf-8 -*-
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django_dsl.fields import DjangoDSLField

from .behaviors import Labelled


class Datasource(Labelled):
    """Datasource gets data from the database.

    It uses self.base_model.objects.all(), unless there's something in
    dsl_query field.

    If there is, it compiles that DSL and uses
    self.base_model.objects.filter().
    """

    base_model = models.ForeignKey(
        ContentType,
        verbose_name=_("Base model"),
    )

    dsl_query = DjangoDSLField(
        verbose_name=_("DSL query"))

    class Meta:
        verbose_name = _("Datasource")
        verbose_name_plural = _("Datasources")<commit_msg>Use base_model, <commit_after># -*- encoding: utf-8 -*-
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django_dsl import compiler
from django_dsl.exceptions import CompileException
from django_dsl.fields import DjangoDSLField

from .behaviors import Labelled, WithBaseModel


class Datasource(Labelled, WithBaseModel):
    """Datasource gets data from the database.

    It uses self.base_model.objects.all(), unless there's something in
    dsl_query field.

    If there is, it compiles that DSL and uses
    self.base_model.objects.filter().
    """

    dsl_query = DjangoDSLField(
        verbose_name=_("DSL query"))

    class Meta:
        verbose_name = _("Datasource")
        verbose_name_plural = _("Datasources")

    def get_model(self):
        return self.base_model.model_class()

    def get_shortcuts(self):
        return getattr(self.get_model(), 'django_dsl_shortcuts', {})

    def get_filter(self):
        return compiler.compile(
            self.dsl_query,
            shortcuts=self.get_shortcuts())
{'fraction_non_alphanumeric': 0.08931259407927747, 'fraction_numerical': 0.0010035122930255895, 'mean_word_length': 3.51131221719457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4316394', 'n_tokens_mistral': 615, 'n_tokens_neox': 582, 'n_words': 157}