<commit_before>def es_query(input_query, doc_type, min_score=0.5, additional_filter=None, source_filter=None):
    """wrap the body to filter by doc_type
    as some full text queries seem to break in ElasticSearch
    if, instead, we pass the doc_type in the URL path."""
    es_filter = {
        'type': {
            'value': doc_type
        }
    }

    if additional_filter:
        es_filter = {
            'bool': {
                'must': [
                    es_filter,
                    additional_filter
                ]
            }
        }
    query = {
        'query': {
            'filtered': {
                'query': input_query,
                'filter': es_filter
            }
        },
        'min_score': min_score,
        '_source': {
            'exclude': ['*.info']
        }
    }

    if source_filter:
        query['_source'] = source_filter

    return query

def es_aggs(doc_type, field_name, additional_filter=None):
    query = {
      "size": 0,
      "aggs": {}
    }
    if additional_filter:
        query['query'] = {
          "filtered": {
            "query": {
              "match_all": {}
            },
            "filter": {
              "term": {
                additional_filter[0] : additional_filter[1]
              }
            }
          }
        }

    query['aggs'][field_name] = {
        "terms": {
            "field": '.'.join((doc_type, field_name, 'exact_'+field_name))
        }<commit_msg>Set max aggs size to 1000<commit_after>def es_query(input_query, doc_type, min_score=0.5, additional_filter=None, source_filter=None):
    """wrap the body to filter by doc_type
    as some full text queries seem to break in ElasticSearch
    if, instead, we pass the doc_type in the URL path."""
    es_filter = {
        'type': {
            'value': doc_type
        }
    }

    if additional_filter:
        es_filter = {
            'bool': {
                'must': [
                    es_filter,
                    additional_filter
                ]
            }
        }
    query = {
        'query': {
            'filtered': {
                'query': input_query,
                'filter': es_filter
            }
        },
        'min_score': min_score,
        '_source': {
            'exclude': ['*.info']
        }
    }

    if source_filter:
        query['_source'] = source_filter

    return query

def es_aggs(doc_type, field_name, additional_filter=None):
    query = {
      "size": 0,
      "aggs": {}
    }
    if additional_filter:
        query['query'] = {
          "filtered": {
            "query": {
              "match_all": {}
            },
            "filter": {
              "term": {
                additional_filter[0] : additional_filter[1]
              }
            }
          }
        }

    query['aggs'][field_name] = {
        "terms": {
            "field": '.'.join((doc_type, field_name, 'exact_'+field_name)),
            'size': 1000
        }
{'fraction_non_alphanumeric': 0.12182061579651941, 'fraction_numerical': 0.006024096385542169, 'mean_word_length': 1.3278816199376946, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1964440', 'n_tokens_mistral': 895, 'n_tokens_neox': 853, 'n_words': 179}