<commit_before>from django.conf import settings
from django.core.exceptions import ValidationError
from django.db.models.signals import pre_delete
from django.db import models
from django.template.defaultfilters import slugify

from elasticsearch import Elasticsearch
from json_field import JSONField

from bulbs.campaigns.models import Campaign
from bulbs.content.custom_search import custom_search_model
from bulbs.content.models import Content
from bulbs.content.mixins import DetailImageMixin
from bulbs.utils.methods import today_as_utc_datetime, is_valid_digit


es = Elasticsearch(settings.ES_URLS)


class SpecialCoverage(DetailImageMixin, models.Model):
    name = models.CharField(max_length=255, unique=True)
    slug = models.SlugField(max_length=255, blank=True, editable=True, unique=True)
    description = models.TextField(default="", blank=True)
    query = JSONField(default={}, blank=True)
    videos = JSONField(default=[], blank=True)
    active = models.BooleanField(default=False)
    promoted = models.BooleanField(default=False)
    start_date = models.DateTimeField(blank=True, null=True)
    end_date = models.DateTimeField(blank=True, null=True)
    campaign = models.ForeignKey(
        Campaign, null=True, default=None, blank=True, on_delete=models.SET_NULL)
    # Property-specific custom configuration
    config = JSONField(default={}, blank=True)

    def __unicode__(self):
        return self.name

    def clean(self):
        super(SpecialCoverage, self).clean()
        self.clean_publish_dates()
        self.clean_query()
        self.clean_videos()

    def clean_publish_dates(self):
        """
        If an end_date value is provided, the start_date must be less.
        """
        if self.end_date:
            if not self.start_date:
                raise ValidationError("""The End Date requires a Start Date value.""")
            elif self.end_date < self.start_date:
                raise ValidationError("""The End Date must not precede the Start Date.""")
        if self.start_date and not self.end_date:
            raise ValidationError("""The Start Date requires an End Date.""")

    def clean_query(self):
        """
        Removes any `None` value from an elasticsearch query.
        """
        if self.query and self.query != {}:
            for key, value in self.query.items():
                if isinstance(value, list) and None in value:
                    self.query[key] = [v for v in value if v is not None]

    def clean_videos(self):
        """
        Validates that all values in the video list are integer ids and removes all None values.
        """
        if self.videos:
            self.videos = [int(v) for v in self.videos if v is not None and is_valid_digit(v)]

    def save(self, *args, **kwargs):
        """Saving ensures that the slug, if not set, is set to the slugified name."""
        self.clean()

        if not self.slug:
            self.slug = slugify(self.name)

        super(SpecialCoverage, self).save(*args, **kwargs)

        if self.query and self.query != {}:
            if self.is_active:
                self._save_percolator()
            else:
                self._delete_percolator()

    def _save_percolator(self):
        """
        S`aves the query field as an elasticsearch percolator
        """
        index = Content.search_objects.mapping.index
        query_filter = self.get_content(published=False).to_dict()

        q = {}

        if "query" in query_filter:
            q = {"query": query_filter.get("query", {})}
        else:
            # We don't know how to save this
            return

        # We'll need this data, to decide which special coverage section to use
        if self.campaign:
            q["sponsored"] = True
            q["start_date"] = self.campaign.start_date
            q["end_date"] = self.campaign.end_date

        es.index(
            index=index,
            doc_type=".percolator",<commit_msg><NAME> is an idiot<commit_after>from django.conf import settings
from django.core.exceptions import ValidationError
from django.db.models.signals import pre_delete
from django.db import models
from django.template.defaultfilters import slugify

from elasticsearch import Elasticsearch
from json_field import JSONField

from bulbs.campaigns.models import Campaign
from bulbs.content.custom_search import custom_search_model
from bulbs.content.models import Content
from bulbs.content.mixins import DetailImageMixin
from bulbs.utils.methods import today_as_utc_datetime, is_valid_digit


es = Elasticsearch(settings.ES_URLS)


class SpecialCoverage(DetailImageMixin, models.Model):
    name = models.CharField(max_length=255, unique=True)
    slug = models.SlugField(max_length=255, blank=True, editable=True, unique=True)
    description = models.TextField(default="", blank=True)
    query = JSONField(default={}, blank=True)
    videos = JSONField(default=[], blank=True)
    active = models.BooleanField(default=False)
    promoted = models.BooleanField(default=False)
    start_date = models.DateTimeField(blank=True, null=True)
    end_date = models.DateTimeField(blank=True, null=True)
    campaign = models.ForeignKey(
        Campaign, null=True, default=None, blank=True, on_delete=models.SET_NULL)
    # Property-specific custom configuration
    config = JSONField(default={}, blank=True)

    def __unicode__(self):
        return self.name

    def clean(self):
        super(SpecialCoverage, self).clean()
        self.clean_publish_dates()
        self.clean_query()
        self.clean_videos()

    def clean_publish_dates(self):
        """
        If an end_date value is provided, the start_date must be less.
        """
        if self.end_date:
            if not self.start_date:
                raise ValidationError("""The End Date requires a Start Date value.""")
            elif self.end_date < self.start_date:
                raise ValidationError("""The End Date must not precede the Start Date.""")
        if self.start_date and not self.end_date:
            raise ValidationError("""The Start Date requires an End Date.""")

    def clean_query(self):
        """
        Removes any `None` value from an elasticsearch query.
        """
        if self.query and self.query != {}:
            for key, value in self.query.items():
                if isinstance(value, list) and None in value:
                    self.query[key] = [v for v in value if v is not None]

    def clean_videos(self):
        """
        Validates that all values in the video list are integer ids and removes all None values.
        """
        if self.videos:
            self.videos = [int(v) for v in self.videos if v is not None and is_valid_digit(v)]

    def save(self, *args, **kwargs):
        """Saving ensures that the slug, if not set, is set to the slugified name."""
        self.clean()

        if not self.slug:
            self.slug = slugify(self.name)

        super(SpecialCoverage, self).save(*args, **kwargs)

        if self.query and self.query != {}:
            if self.is_active:
                self._save_percolator()
            else:
                self._delete_percolator()

    def _save_percolator(self):
        """
        Saves the query field as an elasticsearch percolator
        """
        index = Content.search_objects.mapping.index
        query_filter = self.get_content(published=False).to_dict()

        q = {}

        if "query" in query_filter:
            q = {"query": query_filter.get("query", {})}
        else:
            # We don't know how to save this
            return

        # We'll need this data, to decide which special coverage section to use
        if self.campaign:
            q["sponsored"] = True
            q["start_date"] = self.campaign.start_date
            q["end_date"] = self.campaign.end_date

        es.index(
            index=index,
            doc_type=".percolator",
{'fraction_non_alphanumeric': 0.09579556898288016, 'fraction_numerical': 0.0015105740181268882, 'mean_word_length': 2.588527551942186, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2247223', 'n_tokens_mistral': 2292, 'n_tokens_neox': 2177, 'n_words': 638}