<commit_before>from elasticsearch import ElasticsearchException

from corehq.apps.es import CaseES, GroupES, LedgerES
from corehq.apps.es import FormES
from corehq.apps.es.sms import SMSES
from corehq.apps.es.aggregations import AggregationTerm, NestedTermAggregationsHelper
from corehq.elastic import get_es_new


def get_form_export_base_query(domain, app_id, xmlns, include_errors):
    query = (FormES()
            .domain(domain)
            .app(app_id)
            .xmlns(xmlns)
            .sort("received_on")
            .remove_default_filter('has_user'))
    if include_errors:
        query = query.remove_default_filter("is_xform_instance")
        query = query.doc_type(["xforminstance", "xformarchived", "xformdeprecated", "xformduplicate"])
    return query


def get_case_export_base_query(domain, case_type):
    return (CaseES()
            .domain(domain)
            .case_type(case_type)
            .sort("opened_on"))


def get_sms_export_base_query(domain):
    return (SMSES()
            .domain(domain)
            .processed_or_incoming_messages()
            .sort("date"))


def get_groups_user_ids(group_ids):
    q = (GroupES()
         .doc_id(group_ids))
    return [user for user_list in q.values_list("users", flat=True) for user in user_list]


def get_ledger_section_entry_combinations(domain):
    """Get all section / entry combinations in a domain.
    :returns: a generator of namedtuples with fields ``section_id``, ``entry_id``, ``doc_count``
    """
    terms = [
        AggregationTerm('section_id', 'section_id'),
        AggregationTerm('entry_id', 'entry_id'),
    ]
    query = LedgerES().domain(domain)
    return NestedTermAggregationsHelper(base_query=query, terms=terms).get_data()


def get_case_name(case_id):
    from corehq.pillows.mappings.case_mapping import CASE_INDEX_INFO
    try:
        result = get_es_new().get(CASE_INDEX_INFO.index, case_id, _source_include=['name'])
    except ElasticsearchException:
        return None

    return result['_source']['name']
<commit_msg>handle 1 user in a group<commit_after>from elasticsearch import ElasticsearchException

from corehq.apps.es import CaseES, GroupES, LedgerES
from corehq.apps.es import FormES
from corehq.apps.es.sms import SMSES
from corehq.apps.es.aggregations import AggregationTerm, NestedTermAggregationsHelper
from corehq.elastic import get_es_new


def get_form_export_base_query(domain, app_id, xmlns, include_errors):
    query = (FormES()
            .domain(domain)
            .app(app_id)
            .xmlns(xmlns)
            .sort("received_on")
            .remove_default_filter('has_user'))
    if include_errors:
        query = query.remove_default_filter("is_xform_instance")
        query = query.doc_type(["xforminstance", "xformarchived", "xformdeprecated", "xformduplicate"])
    return query


def get_case_export_base_query(domain, case_type):
    return (CaseES()
            .domain(domain)
            .case_type(case_type)
            .sort("opened_on"))


def get_sms_export_base_query(domain):
    return (SMSES()
            .domain(domain)
            .processed_or_incoming_messages()
            .sort("date"))


def get_groups_user_ids(group_ids):
    q = (GroupES()
         .doc_id(group_ids))

    results = []
    for user_list in q.values_list("users", flat=True):
        if isinstance(user_list, basestring):
            results.append(user_list)
        else:
            results.extend(user_list)

    return results


def get_ledger_section_entry_combinations(domain):
    """Get all section / entry combinations in a domain.
    :returns: a generator of namedtuples with fields ``section_id``, ``entry_id``, ``doc_count``
    """
    terms = [
        AggregationTerm('section_id', 'section_id'),
        AggregationTerm('entry_id', 'entry_id'),
    ]
    query = LedgerES().domain(domain)
    return NestedTermAggregationsHelper(base_query=query, terms=terms).get_data()


def get_case_name(case_id):
    from corehq.pillows.mappings.case_mapping import CASE_INDEX_INFO
    try:
        result = get_es_new().get(CASE_INDEX_INFO.index, case_id, _source_include=['name'])
    except ElasticsearchException:
        return None

    return result['_source']['name']

{'fraction_non_alphanumeric': 0.10584629891560585, 'fraction_numerical': 0.00023573785950023574, 'mean_word_length': 3.4151925078043703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5543266', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1325, 'n_words': 264}