<commit_before>import datetime
from dateutil.relativedelta import relativedelta
from django.db.models import Q
from django.utils.translation import ugettext as _

from corehq.apps.accounting.models import Subscription, SoftwarePlanEdition
from corehq.apps.domain.models import Domain
from corehq.apps.es.cases import CaseES
from corehq.apps.es.domains import DomainES
from corehq.apps.es.forms import FormES
from corehq.apps.es.sms import SMSES
from corehq.apps.es.users import UserES
from corehq.apps.groups.models import Group
from corehq.apps.hqadmin.reporting.exceptions import (
    HistoTypeNotFoundException,
    IntervalNotFoundException,
)
from corehq.apps.sms.mixin import SMSBackend
from corehq.elastic import (
    ADD_TO_ES_FILTER,
    DATE_FIELDS,
    es_histogram,
    ES_MAX_CLAUSE_COUNT,
    es_query,
    ES_URLS,
)

from casexml.apps.stock.models import StockReport, StockTransaction
from corehq.util.dates import get_timestamp_millis

LARGE_ES_NUMBER = 10 ** 6


def add_params_to_query(query, params):
    if params:
        for k in params:
            query = query.filter({"terms": {k: params[k]}})
    return query


def get_data_point(count, date):
    return {
        "count": count,
        "time": get_timestamp_millis(date),
    }


def format_return_data(shown_data, initial_value, datespan):
    return {
        'histo_data': {
            "All Domains": add_blank_data(
                shown_data, datespan.startdate, datespan.enddate),
        },
        'initial_values': {"All Domains": initial_value},
        'startdate': datespan.startdate_key_utc,
        'enddate': datespan.enddate_key_utc,
    }


def add_blank_data(histo_data, start, end):
    if not histo_data:
        return [get_data_point(0, date) for date in [start, end]]
    return histo_data


def get_timestep(interval):
    if interval == 'day':
        return relativedelta(days=1)
    elif interval == 'week':
        return relativedelta(weeks=1)
    elif interval == 'month':
        return relativedelta(months=1)
    elif interval == 'year':
        return relativedelta(years=1)
    raise IntervalNotFoundException(unicode(interval))


def daterange(interval, start_date, end_date):
    """
    Generator that yields dates from start_date to end_date in the interval
    specified
    """
    cur_date = start_date
    step = get_timestep(interval)

    while cur_date <= end_date:
        yield cur_date
        cur_date += step


def intervals(interval, start_date, end_date):
    for starting_date in daterange(interval, start_date, end_date):
        closing_date = (
            starting_date + get_timestep(interval) - relativedelta(days=1)
        )
        ending_date = closing_date if closing_date < end_date else end_date
        yield (starting_date, ending_date)


def get_real_project_spaces(facets=None):
    """
    Returns a set of names of real domains
    """
    real_domain_query = (
        DomainES()
        .terms_facet('name', 'name')
        .size(0)
    )
    if facets:
        real_domain_query = add_params_to_query(real_domain_query, facets)
    real_domain_query_results = real_domain_query.run().facets.name.result
    return [_['term'] for _ in real_domain_query_results]


def get_sms_query(begin, end, facet_name, facet_terms, domains,
        size=LARGE_ES_NUMBER):
    """
    Returns query in domains from the begin to end
    """
    return (SMSES()
            .domain(domains)
            .received(gte=begin, lte=end)
            .terms_facet(facet_name, facet_terms, size)
            .size(0))


def get_active_countries_stats_data(domains, datespan, interval,
        datefield='received_on'):
    """
    Returns list of timestamps and how many countries were active in the 30
    days before the timestamp
    """
    histo_data = []
    for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
        t = timestamp
        f = timestamp - relativedelta(days=30)
        form_query = (FormES()
            .domain(domains)
            .submitted(gte=f, lte=t)
            .terms_facet('domains', 'domain', size=LARGE_ES_NUMBER)
            .size(0))

        domains = form_query.run().facet('domains', "terms")
        domains = [x['term'] for x in domains]
        countries = (DomainES()
                .in_domains(domains)
                .terms_facet('countries', 'countries', size=LARGE_ES_NUMBER))

        c = len(countries.run().facet('countries', 'terms'))
        if c > 0:
            histo_data.append(get_data_point(c, timestamp))

    return format_return_data(histo_data, 0, datespan)


def domains_matching_plan(software_plan_edition, start, end):
    matching_subscriptions = Subscription.objects.filter(
        ((Q(date_start__gte=start) & Q(date_start__lte=end))
            | (Q(date_end__gte=start) & Q(date_end__lte=end))),
        plan_version__plan__edition=software_plan_edition,
    )

    return {
        subscription.subscriber.domain
        for subscription in matching_subscriptions
    }


def get_subscription_stats_data(domains, datespan, interval,
        software_plan_edition=None):
    return [
        get_data_point(
            len(set(domains) & set(domains_matching_plan(
                software_plan_edition, timestamp, timestamp))),
            timestamp
        )
        for timestamp in daterange(
            interval, datespan.startdate, datespan.enddate
        )
    ]


def get_active_domain_stats_data(domains, datespan, interval,
        datefield='received_on', software_plan_edition=None,
        add_form_domains=True, add_sms_domains=True,
        restrict_to_mobile_submissions=False):
    """
    Returns list of timestamps and how many domains were active in the 30 days
    before the timestamp
    """
    histo_data = []
    for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
        t = timestamp
        f = timestamp - relativedelta(days=30)
        domains_in_interval = (
            domains
            if software_plan_edition is None else
            list((set(domains) & set(domains_matching_plan(software_plan_edition, f,
                t))))
        )
        active_domains = set()
        if add_form_domains:
            form_query = (FormES()<commit_msg>Remove duplicate set call<commit_after>import datetime
from dateutil.relativedelta import relativedelta
from django.db.models import Q
from django.utils.translation import ugettext as _

from corehq.apps.accounting.models import Subscription, SoftwarePlanEdition
from corehq.apps.domain.models import Domain
from corehq.apps.es.cases import CaseES
from corehq.apps.es.domains import DomainES
from corehq.apps.es.forms import FormES
from corehq.apps.es.sms import SMSES
from corehq.apps.es.users import UserES
from corehq.apps.groups.models import Group
from corehq.apps.hqadmin.reporting.exceptions import (
    HistoTypeNotFoundException,
    IntervalNotFoundException,
)
from corehq.apps.sms.mixin import SMSBackend
from corehq.elastic import (
    ADD_TO_ES_FILTER,
    DATE_FIELDS,
    es_histogram,
    ES_MAX_CLAUSE_COUNT,
    es_query,
    ES_URLS,
)

from casexml.apps.stock.models import StockReport, StockTransaction
from corehq.util.dates import get_timestamp_millis

LARGE_ES_NUMBER = 10 ** 6


def add_params_to_query(query, params):
    if params:
        for k in params:
            query = query.filter({"terms": {k: params[k]}})
    return query


def get_data_point(count, date):
    return {
        "count": count,
        "time": get_timestamp_millis(date),
    }


def format_return_data(shown_data, initial_value, datespan):
    return {
        'histo_data': {
            "All Domains": add_blank_data(
                shown_data, datespan.startdate, datespan.enddate),
        },
        'initial_values': {"All Domains": initial_value},
        'startdate': datespan.startdate_key_utc,
        'enddate': datespan.enddate_key_utc,
    }


def add_blank_data(histo_data, start, end):
    if not histo_data:
        return [get_data_point(0, date) for date in [start, end]]
    return histo_data


def get_timestep(interval):
    if interval == 'day':
        return relativedelta(days=1)
    elif interval == 'week':
        return relativedelta(weeks=1)
    elif interval == 'month':
        return relativedelta(months=1)
    elif interval == 'year':
        return relativedelta(years=1)
    raise IntervalNotFoundException(unicode(interval))


def daterange(interval, start_date, end_date):
    """
    Generator that yields dates from start_date to end_date in the interval
    specified
    """
    cur_date = start_date
    step = get_timestep(interval)

    while cur_date <= end_date:
        yield cur_date
        cur_date += step


def intervals(interval, start_date, end_date):
    for starting_date in daterange(interval, start_date, end_date):
        closing_date = (
            starting_date + get_timestep(interval) - relativedelta(days=1)
        )
        ending_date = closing_date if closing_date < end_date else end_date
        yield (starting_date, ending_date)


def get_real_project_spaces(facets=None):
    """
    Returns a set of names of real domains
    """
    real_domain_query = (
        DomainES()
        .terms_facet('name', 'name')
        .size(0)
    )
    if facets:
        real_domain_query = add_params_to_query(real_domain_query, facets)
    real_domain_query_results = real_domain_query.run().facets.name.result
    return [_['term'] for _ in real_domain_query_results]


def get_sms_query(begin, end, facet_name, facet_terms, domains,
        size=LARGE_ES_NUMBER):
    """
    Returns query in domains from the begin to end
    """
    return (SMSES()
            .domain(domains)
            .received(gte=begin, lte=end)
            .terms_facet(facet_name, facet_terms, size)
            .size(0))


def get_active_countries_stats_data(domains, datespan, interval,
        datefield='received_on'):
    """
    Returns list of timestamps and how many countries were active in the 30
    days before the timestamp
    """
    histo_data = []
    for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
        t = timestamp
        f = timestamp - relativedelta(days=30)
        form_query = (FormES()
            .domain(domains)
            .submitted(gte=f, lte=t)
            .terms_facet('domains', 'domain', size=LARGE_ES_NUMBER)
            .size(0))

        domains = form_query.run().facet('domains', "terms")
        domains = [x['term'] for x in domains]
        countries = (DomainES()
                .in_domains(domains)
                .terms_facet('countries', 'countries', size=LARGE_ES_NUMBER))

        c = len(countries.run().facet('countries', 'terms'))
        if c > 0:
            histo_data.append(get_data_point(c, timestamp))

    return format_return_data(histo_data, 0, datespan)


def domains_matching_plan(software_plan_edition, start, end):
    matching_subscriptions = Subscription.objects.filter(
        ((Q(date_start__gte=start) & Q(date_start__lte=end))
            | (Q(date_end__gte=start) & Q(date_end__lte=end))),
        plan_version__plan__edition=software_plan_edition,
    )

    return {
        subscription.subscriber.domain
        for subscription in matching_subscriptions
    }


def get_subscription_stats_data(domains, datespan, interval,
        software_plan_edition=None):
    return [
        get_data_point(
            len(set(domains) & domains_matching_plan(
                software_plan_edition, timestamp, timestamp)),
            timestamp
        )
        for timestamp in daterange(
            interval, datespan.startdate, datespan.enddate
        )
    ]


def get_active_domain_stats_data(domains, datespan, interval,
        datefield='received_on', software_plan_edition=None,
        add_form_domains=True, add_sms_domains=True,
        restrict_to_mobile_submissions=False):
    """
    Returns list of timestamps and how many domains were active in the 30 days
    before the timestamp
    """
    histo_data = []
    for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
        t = timestamp
        f = timestamp - relativedelta(days=30)
        domains_in_interval = (
            domains
            if software_plan_edition is None else
            list((set(domains) & domains_matching_plan(software_plan_edition, f, t)))
        )
        active_domains = set()
        if add_form_domains:
            form_query = (FormES()
{'fraction_non_alphanumeric': 0.09083627042614546, 'fraction_numerical': 0.003524511374559436, 'mean_word_length': 2.9459544879898862, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3386498', 'n_tokens_mistral': 4014, 'n_tokens_neox': 3844, 'n_words': 856}