<commit_before>from collections import defaultdict
from datetime import date, datetime, timedelta
from dateutil.relativedelta import relativedelta

from django.template.loader import render_to_string
from django.utils.translation import ugettext as _
from corehq.apps.app_manager.dbaccessors import domain_has_apps
from corehq.apps.hqcase.dbaccessors import get_number_of_cases_in_domain, \
    get_number_of_cases_per_domain
from corehq.util.dates import iso_string_to_datetime
from corehq.apps.app_manager.models import ApplicationBase
from corehq.apps.users.util import WEIRD_USER_IDS
from corehq.apps.es.sms import SMSES
from corehq.apps.es.forms import FormES
from corehq.apps.hqadmin.reporting.reports import (
    USER_COUNT_UPPER_BOUND,
    get_mobile_users,
)
from couchforms.analytics import get_number_of_forms_per_domain, \
    get_number_of_forms_in_domain, domain_has_submission_in_last_30_days, \
    get_first_form_submission_received, get_last_form_submission_received

from corehq.apps.domain.models import Domain
from corehq.apps.reminders.models import CaseReminderHandler
from corehq.apps.reports.util import make_form_couch_key
from corehq.apps.users.models import CouchUser
from corehq.elastic import es_query, ADD_TO_ES_FILTER
from dimagi.utils.parsing import json_format_datetime


def num_web_users(domain, *args):
    key = ["active", domain, 'WebUser']
    row = CouchUser.get_db().view('users/by_domain', startkey=key, endkey=key+[{}]).one()
    return row["value"] if row else 0

def num_mobile_users(domain, *args):
    row = CouchUser.get_db().view('users/by_domain', startkey=[domain], endkey=[domain, {}]).one()
    return row["value"] if row else 0

DISPLAY_DATE_FORMAT = '%Y/%m/%d %H:%M:%S'

def active_mobile_users(domain, *args):
    """
    Returns the number of mobile users who have submitted a form or SMS in the
    last 30 days
    """
    now = datetime.utcnow()
    then = (now - timedelta(days=30))

    user_ids = get_mobile_users(domain)

    form_users = {q['term'] for q in (
        FormES()
        .domain(domain)
        .user_facet(size=USER_COUNT_UPPER_BOUND)
        .submitted(gte=then)
        .user_id(user_ids)
        .size(0)
        .run()
        .facets.user.result
    )}

    sms_users = {q['term'] for q in (
        SMSES()
        .incoming_messages()
        .user_facet(size=USER_COUNT_UPPER_BOUND)
        .to_commcare_user()
        .domain(domain)
        .received(gte=then)
        .size(0)
        .run()
        .facets.user.result
    )}

    num_users = len(form_users | sms_users)
    return num_users if 'inactive' not in args else len(user_ids) - num_users


def cases(domain, *args):
    return get_number_of_cases_in_domain(domain)


def cases_in_last(domain, days):
    """
    Returns the number of open cases that have been modified in the last <days> days
    """
    now = datetime.utcnow()
    then = json_format_datetime(now - timedelta(days=int(days)))
    now = json_format_datetime(now)

    q = {"query": {
        "range": {
            "modified_on": {
                "from": then,
                "to": now}}}}
    data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_index='cases', size=1)
    return data['hits']['total'] if data.get('hits') else 0

def inactive_cases_in_last(domain, days):
    """
    Returns the number of open cases that have been modified in the last <days> days
    """
    now = datetime.utcnow()
    then = json_format_datetime(now - timedelta(days=int(days)))
    now = json_format_datetime(now)

    q = {"query":
             {"bool": {
                 "must_not": {
                     "range": {
                         "modified_on": {
                             "from": then,
                             "to": now }}}}}}
    data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_index='cases', size=1)
    return data['hits']['total'] if data.get('hits') else 0


def forms(domain, *args):
    return get_number_of_forms_in_domain(domain)


def forms_in_last(domain, days):
    """
    Returns the number of forms submitted in the last given number of days
    """
    now = datetime.utcnow()
    then = json_format_datetime(now - timedelta(days=int(days)))
    now = json_format_datetime(now)
    q = {
        'query': {
            'range': {
                'received_on': {'from': then, 'to': now}},
            'filter': {
                'and': ADD_TO_ES_FILTER['forms'][:]}}}
    data = es_query(params={'domain.exact': domain}, q=q, es_index='forms', size=1)
    return data['hits']['total'] if data.get('hits') else 0


def _sms_helper(domain, direction=None, days=None):
    query = SMSES().domain(domain).size(0)

    if direction:
        query = query.direction(direction)

    if days:
        query = query.received(date.today() - relativedelta(days=30))

    return query.run().total


def sms(domain, direction):
    return _sms_helper(domain, direction=direction)
<commit_msg>Use FormES instead of es_query<commit_after>from collections import defaultdict
from datetime import date, datetime, timedelta
from dateutil.relativedelta import relativedelta

from django.template.loader import render_to_string
from django.utils.translation import ugettext as _
from corehq.apps.app_manager.dbaccessors import domain_has_apps
from corehq.apps.hqcase.dbaccessors import get_number_of_cases_in_domain, \
    get_number_of_cases_per_domain
from corehq.util.dates import iso_string_to_datetime
from corehq.apps.app_manager.models import ApplicationBase
from corehq.apps.users.util import WEIRD_USER_IDS
from corehq.apps.es.sms import SMSES
from corehq.apps.es.forms import FormES
from corehq.apps.hqadmin.reporting.reports import (
    USER_COUNT_UPPER_BOUND,
    get_mobile_users,
)
from couchforms.analytics import get_number_of_forms_per_domain, \
    get_number_of_forms_in_domain, domain_has_submission_in_last_30_days, \
    get_first_form_submission_received, get_last_form_submission_received

from corehq.apps.domain.models import Domain
from corehq.apps.reminders.models import CaseReminderHandler
from corehq.apps.reports.util import make_form_couch_key
from corehq.apps.users.models import CouchUser
from corehq.elastic import es_query, ADD_TO_ES_FILTER
from dimagi.utils.parsing import json_format_datetime


def num_web_users(domain, *args):
    key = ["active", domain, 'WebUser']
    row = CouchUser.get_db().view('users/by_domain', startkey=key, endkey=key+[{}]).one()
    return row["value"] if row else 0

def num_mobile_users(domain, *args):
    row = CouchUser.get_db().view('users/by_domain', startkey=[domain], endkey=[domain, {}]).one()
    return row["value"] if row else 0

DISPLAY_DATE_FORMAT = '%Y/%m/%d %H:%M:%S'

def active_mobile_users(domain, *args):
    """
    Returns the number of mobile users who have submitted a form or SMS in the
    last 30 days
    """
    now = datetime.utcnow()
    then = (now - timedelta(days=30))

    user_ids = get_mobile_users(domain)

    form_users = {q['term'] for q in (
        FormES()
        .domain(domain)
        .user_facet(size=USER_COUNT_UPPER_BOUND)
        .submitted(gte=then)
        .user_id(user_ids)
        .size(0)
        .run()
        .facets.user.result
    )}

    sms_users = {q['term'] for q in (
        SMSES()
        .incoming_messages()
        .user_facet(size=USER_COUNT_UPPER_BOUND)
        .to_commcare_user()
        .domain(domain)
        .received(gte=then)
        .size(0)
        .run()
        .facets.user.result
    )}

    num_users = len(form_users | sms_users)
    return num_users if 'inactive' not in args else len(user_ids) - num_users


def cases(domain, *args):
    return get_number_of_cases_in_domain(domain)


def cases_in_last(domain, days):
    """
    Returns the number of open cases that have been modified in the last <days> days
    """
    now = datetime.utcnow()
    then = json_format_datetime(now - timedelta(days=int(days)))
    now = json_format_datetime(now)

    q = {"query": {
        "range": {
            "modified_on": {
                "from": then,
                "to": now}}}}
    data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_index='cases', size=1)
    return data['hits']['total'] if data.get('hits') else 0

def inactive_cases_in_last(domain, days):
    """
    Returns the number of open cases that have been modified in the last <days> days
    """
    now = datetime.utcnow()
    then = json_format_datetime(now - timedelta(days=int(days)))
    now = json_format_datetime(now)

    q = {"query":
             {"bool": {
                 "must_not": {
                     "range": {
                         "modified_on": {
                             "from": then,
                             "to": now }}}}}}
    data = es_query(params={"domain.exact": domain, 'closed': False}, q=q, es_index='cases', size=1)
    return data['hits']['total'] if data.get('hits') else 0


def forms(domain, *args):
    return get_number_of_forms_in_domain(domain)


def forms_in_last(domain, days):
    """
    Returns the number of forms submitted in the last given number of days
    """
    then = datetime.utcnow() - timedelta(days=int(days))
    return FormES().domain(domain).submitted(gte=then).size(0).run().total


def _sms_helper(domain, direction=None, days=None):
    query = SMSES().domain(domain).size(0)

    if direction:
        query = query.direction(direction)

    if days:
        query = query.received(date.today() - relativedelta(days=30))

    return query.run().total


def sms(domain, direction):
    return _sms_helper(domain, direction=direction)

{'fraction_non_alphanumeric': 0.11737868104520946, 'fraction_numerical': 0.0038365823309829948, 'mean_word_length': 3.235836627140975, 'pattern_counts': {'":': 28, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5376024', 'n_tokens_mistral': 3229, 'n_tokens_neox': 3118, 'n_words': 713}