<commit_before>"""ElasticSearch filters specifially for content"""

import datetime
import dateutil.parser
import dateutil.tz
from django.utils import timezone

from elasticsearch_dsl.filter import Exists, MatchAll, Nested, Prefix, Range, Term, Terms
from elasticsearch_dsl.query import FunctionScore

from six import string_types, text_type, binary_type


def Evergreen(evergreen=True):
    return Term(evergreen=evergreen)


def parse_datetime(value):
    """Returns a datetime object for a given argument

    This helps to convert strings, dates and datetimes to proper tz-enabled
    datetime objects."""

    if isinstance(value, (string_types, text_type, binary_type)):
        value = dateutil.parser.parse(value)
        value.replace(tzinfo=dateutil.tz.tzutc())
        return value
    elif isinstance(value, datetime.datetime):
        value.replace(tzinfo=dateutil.tz.tzutc())
        return value
    elif isinstance(value, datetime.date):
        value = datetime.datetime(value.year, value.month, value.day)
        value.replace(tzinfo=dateutil.tz.tzutc())
        return value
    else:
        raise ValueError('Value must be parsable to datetime object. Got `{}`'.format(type(value)))


def Published(before=None, after=None):  # noqa
    published_params = {}
    if after is not None:
        published_params["gte"] = parse_datetime(after)

    if before is not None:
        published_params["lte"] = parse_datetime(before)

    if before is None and after is None:
        published_params["lte"] = timezone.now()

    return Range(published=published_params)


def Status(status):  # noqa
    if status:
        return Term(status=status)
    else:
        return MatchAll()


def AllSponsored():  # noqa
    """Filter for any content that has a campaign."""
    return Nested(path="campaign", filter=Exists(field="campaign.id"))


def Authors(usernames):  # noqa
    included = []
    excluded = []
    for username in usernames:
        if username.startswith("-"):
            excluded.append(username[1:])
        else:
            included.append(username)
    f = MatchAll()
    if included:
        f &= Terms(**{"authors.username": included})
    if excluded:
        f &= Terms(**{"authors.username": excluded})
    return f


def Tags(slugs):  # noqa
    included = []
    excluded = []
    for slug in slugs:
        if slug.startswith("-"):
            excluded.append(slug[1:])
        else:
            included.append(slug)

    f = MatchAll()
    if included:
        f &= Nested(path="tags", filter=Terms(**{"tags.slug": included}))
    if excluded:
        f &= ~Nested(path="tags", filter=Terms(**{"tags.slug": excluded}))

    return f


def FeatureTypes(slugs):  # noqa
    included = []
    excluded = []
    for slug in slugs:
        if slug.startswith("-"):
            excluded.append(slug[1:])
        else:
            included.append(slug)

    f = MatchAll()<commit_msg>Create `NegateQueryFilter`<commit_after>"""ElasticSearch filters specifially for content"""

import datetime
import dateutil.parser
import dateutil.tz
from django.utils import timezone

from elasticsearch_dsl.filter import Exists, MatchAll, Nested, Not, Range, Term, Terms
from elasticsearch_dsl.query import FunctionScore

from six import string_types, text_type, binary_type


def Evergreen(evergreen=True):
    return Term(evergreen=evergreen)


def parse_datetime(value):
    """Returns a datetime object for a given argument

    This helps to convert strings, dates and datetimes to proper tz-enabled
    datetime objects."""

    if isinstance(value, (string_types, text_type, binary_type)):
        value = dateutil.parser.parse(value)
        value.replace(tzinfo=dateutil.tz.tzutc())
        return value
    elif isinstance(value, datetime.datetime):
        value.replace(tzinfo=dateutil.tz.tzutc())
        return value
    elif isinstance(value, datetime.date):
        value = datetime.datetime(value.year, value.month, value.day)
        value.replace(tzinfo=dateutil.tz.tzutc())
        return value
    else:
        raise ValueError('Value must be parsable to datetime object. Got `{}`'.format(type(value)))


def Published(before=None, after=None):  # noqa
    published_params = {}
    if after is not None:
        published_params["gte"] = parse_datetime(after)

    if before is not None:
        published_params["lte"] = parse_datetime(before)

    if before is None and after is None:
        published_params["lte"] = timezone.now()

    return Range(published=published_params)


def Status(status):  # noqa
    if status:
        return Term(status=status)
    else:
        return MatchAll()


def AllSponsored():  # noqa
    """Filter for any content that has a campaign."""
    return Nested(path="campaign", filter=Exists(field="campaign.id"))


def Authors(usernames):  # noqa
    included = []
    excluded = []
    for username in usernames:
        if username.startswith("-"):
            excluded.append(username[1:])
        else:
            included.append(username)
    f = MatchAll()
    if included:
        f &= Terms(**{"authors.username": included})
    if excluded:
        f &= Terms(**{"authors.username": excluded})
    return f


def NegateQueryFilter(es_query):  # noqa
    """
    Return a filter removing the contents of the provided query.
    """
    query = es_query.to_dict().get("query", {})
    filtered = query.get("filtered", {})
    negated_filter = filtered.get("filter", {})
    return Not(**negated_filter)


def Tags(slugs):  # noqa
    included = []
    excluded = []
    for slug in slugs:
        if slug.startswith("-"):
            excluded.append(slug[1:])
        else:
            included.append(slug)

    f = MatchAll()
    if included:
        f &= Nested(path="tags", filter=Terms(**{"tags.slug": included}))
    if excluded:
        f &= ~Nested(path="tags", filter=Terms(**{"tags.slug": excluded}))

    return f


def FeatureTypes(slugs):  # noqa
    included = []
    excluded = []
    for slug in slugs:
        if slug.startswith("-"):
            excluded.append(slug[1:])
        else:
            included.append(slug)

    f = MatchAll()
{'fraction_non_alphanumeric': 0.11972059779077322, 'fraction_numerical': 0.0009746588693957114, 'mean_word_length': 2.972258064516129, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '204450', 'n_tokens_mistral': 1824, 'n_tokens_neox': 1799, 'n_words': 477}