<commit_before>from datetime import date, datetime, timedelta

from django.db import models

import bleach
from constance import config
from django_mysql.models import QuerySet

from .constants import (ALLOWED_TAGS, ALLOWED_ATTRIBUTES, ALLOWED_PROTOCOLS,
                        ALLOWED_STYLES)
from .content import parse as parse_content


class BaseDocumentManager(models.Manager):
    """Manager for Documents, assists for queries"""
    def get_queryset(self):
        return QuerySet(self.model)

    def clean_content(self, content_in):
        tags = ALLOWED_TAGS
        attributes = ALLOWED_ATTRIBUTES
        styles = ALLOWED_STYLES
        protocols = ALLOWED_PROTOCOLS
        allowed_hosts = config.KUMA_WIKI_IFRAME_ALLOWED_HOSTS

        bleached_content = bleach.clean(content_in, attributes=attributes,
                                        tags=tags, styles=styles,
                                        protocols=protocols)
        filtered_content = (parse_content(bleached_content)
                            .filterIframeHosts(allowed_hosts)
                            .serialize())
        return filtered_content

    def get_by_natural_key(self, locale, slug):
        return self.get(locale=locale, slug=slug)

    def get_by_stale_rendering(self):
        """Find documents whose renderings have gone stale"""
        return (self.exclude(render_expires__isnull=True)
                    .filter(render_expires__lte=datetime.now()))

    def filter_for_list(self, locale=None, tag=None, tag_name=None,
                        errors=None, noparent=None, toplevel=None):
        docs = (self.filter(is_redirect=False)
                    .exclude(slug__startswith='User:')
                    .exclude(slug__startswith='Talk:')
                    .exclude(slug__startswith='User_talk:')
                    .exclude(slug__startswith='Template_talk:')
                    .exclude(slug__startswith='Project_talk:')
                    .order_by('slug'))
        if locale:
            docs = docs.filter(locale=locale)
        if tag:
            docs = docs.filter(tags__in=[tag])
        if tag_name:
            docs = docs.filter(tags__name=tag_name)
        if errors:
            docs = (docs.exclude(rendered_errors__isnull=True)
                        .exclude(rendered_errors__exact='[]'))
        if noparent:
            # List translated pages without English source associated
            docs = docs.filter(parent__isnull=True)
        if toplevel:
            docs = docs.filter(parent_topic__isnull=True)

        # Leave out the html, since that leads to huge cache objects and we
        # never use the content in lists.
        docs = docs.defer('html')
        return docs

    def filter_for_review(self, locale=None, tag=None, tag_name=None):
        """Filter for documents with current revision flagged for review"""
        query = 'current_revision__review_tags__%s'<commit_msg>bug 1392457: Only return data needed for display<commit_after>from datetime import date, datetime, timedelta

from django.db import models

import bleach
from constance import config
from django_mysql.models import QuerySet

from .constants import (ALLOWED_TAGS, ALLOWED_ATTRIBUTES, ALLOWED_PROTOCOLS,
                        ALLOWED_STYLES)
from .content import parse as parse_content


class BaseDocumentManager(models.Manager):
    """Manager for Documents, assists for queries"""
    def get_queryset(self):
        return QuerySet(self.model)

    def clean_content(self, content_in):
        tags = ALLOWED_TAGS
        attributes = ALLOWED_ATTRIBUTES
        styles = ALLOWED_STYLES
        protocols = ALLOWED_PROTOCOLS
        allowed_hosts = config.KUMA_WIKI_IFRAME_ALLOWED_HOSTS

        bleached_content = bleach.clean(content_in, attributes=attributes,
                                        tags=tags, styles=styles,
                                        protocols=protocols)
        filtered_content = (parse_content(bleached_content)
                            .filterIframeHosts(allowed_hosts)
                            .serialize())
        return filtered_content

    def get_by_natural_key(self, locale, slug):
        return self.get(locale=locale, slug=slug)

    def get_by_stale_rendering(self):
        """Find documents whose renderings have gone stale"""
        return (self.exclude(render_expires__isnull=True)
                    .filter(render_expires__lte=datetime.now()))

    def filter_for_list(self, locale=None, tag=None, tag_name=None,
                        errors=None, noparent=None, toplevel=None):
        """
        Returns a filtered queryset for a list of names and urls.
        """

        docs = (self.filter(is_redirect=False)
                    .exclude(slug__startswith='User:')
                    .exclude(slug__startswith='Talk:')
                    .exclude(slug__startswith='User_talk:')
                    .exclude(slug__startswith='Template_talk:')
                    .exclude(slug__startswith='Project_talk:')
                    .order_by('slug'))
        if locale:
            docs = docs.filter(locale=locale)
        if tag:
            docs = docs.filter(tags__in=[tag])
        if tag_name:
            docs = docs.filter(tags__name=tag_name)
        if errors:
            docs = (docs.exclude(rendered_errors__isnull=True)
                        .exclude(rendered_errors__exact='[]'))
        if noparent:
            # List translated pages without English source associated
            docs = docs.filter(parent__isnull=True)
        if toplevel:
            docs = docs.filter(parent_topic__isnull=True)

        # Only include fields needed for a list of links to docs
        docs = docs.only('id', 'locale', 'slug', 'deleted', 'summary_text')
        return docs

    def filter_for_review(self, locale=None, tag=None, tag_name=None):
        """Filter for documents with current revision flagged for review"""
        query = 'current_revision__review_tags__%s'
{'fraction_non_alphanumeric': 0.08198363666722325, 'fraction_numerical': 0.0011688094840540991, 'mean_word_length': 2.282191780821918, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2446749', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1621, 'n_words': 365}