<commit_before>"""Subclass of FacetedSearch for Data explorer datasets."""

import csv
import jsmin
import json
import os

from flask import current_app

from collections import OrderedDict
from elasticsearch import Elasticsearch
from elasticsearch_dsl import HistogramFacet
from elasticsearch_dsl import FacetedSearch
from elasticsearch_dsl import Mapping
from elasticsearch_dsl import TermsFacet


# Keep in sync with convert_to_index_name() in data-explorer-indexers repo.
def convert_to_index_name(s):
    """Converts a string to an Elasticsearch index name."""
    # For Elasticsearch index name restrictions, see
    # https://github.com/DataBiosphere/data-explorer-indexers/issues/5#issue-308168951
    prohibited_chars = [' ', '"', '*', '\\', '<', '|', ',', '>', '/', '?']
    for char in prohibited_chars:
        s = s.replace(char, '_')
    s = s.lower()
    # Remove leading underscore.
    if s.find('_', 0, 1) == 0:
        s = s.lstrip('_')
    print('Index name: %s' % s)
    return s


def get_index_name():
    """Gets Elasticsearch index name."""
    return convert_to_index_name(get_dataset_name())


def get_dataset_name():
    """Gets dataset name from dataset.json.

    current_app.config['DATASET_CONFIG_DIR'] must be set before this is called.
    """
    file_path = os.path.join(current_app.config['DATASET_CONFIG_DIR'],
                             'dataset.json')
    with open(file_path) as f:
        # Remove comments using jsmin, as recommended by JSON creator
        # (https://plus.google.com/+DouglasCrockfordEsq/posts/RK8qyGVaGSr).
        dataset = json.loads(jsmin.jsmin(f.read()))
        return dataset['name']


def get_facets():
    """Gets facets from facet_fields.csv.

    current_app.config['ELASTICSEARCH_URL'], current_app.config['INDEX_NAME']
    current_app.config['DATASET_CONFIG_DIR'] must be set before this is called.
    """
    f = open(
        os.path.join(current_app.config['DATASET_CONFIG_DIR'],
                     'facet_fields.csv'))
    # Remove comments using jsmin.
    csv_str = jsmin.jsmin(f.read())
    facet_rows = csv.DictReader(
        iter(csv_str.splitlines()), skipinitialspace=True)

    using = Elasticsearch(current_app.config['ELASTICSEARCH_URL'])
    mapping = Mapping.from_es(
        current_app.config['INDEX_NAME'], 'type', using=using).to_dict()
    # Preserve order, so facets are returned in the same order as facet_fields.csv
    facets = OrderedDict()
    for facet_row in facet_rows:
        field_name = facet_row['readable_field_name']
        field_type = mapping['type']['properties'][field_name]['type']
        if field_type == 'text':
            # Use ".keyword" because we want aggregation on keyword field, not
            # term field. See
            # https://www.elastic.co/guide/en/elasticsearch/reference/6.2/fielddata.html#before-enabling-fielddata
            facets[field_name] = TermsFacet(field=field_name + '.keyword')
        else:
            # Assume numeric type.
            # TODO: Handle other types.
            # TODO: Automatically figure out bucket intervals. Unfortunately
            # Elasticsearch won't do this for us
            # (https://github.com/elastic/elasticsearch/issues/9572). Make the
            # ranges easy to read (10-19,20-29 instead of 10-17,18-25).
            facets[field_name] = HistogramFacet(field=field_name, interval=10)
    current_app.logger.info('dataset_faceted_search facets: %s' % facets)
    return facets<commit_msg>Change max facet size to 20 (#83)<commit_after>"""Subclass of FacetedSearch for Data explorer datasets."""

import csv
import jsmin
import json
import os

from flask import current_app

from collections import OrderedDict
from elasticsearch import Elasticsearch
from elasticsearch_dsl import HistogramFacet
from elasticsearch_dsl import FacetedSearch
from elasticsearch_dsl import Mapping
from elasticsearch_dsl import TermsFacet


# Keep in sync with convert_to_index_name() in data-explorer-indexers repo.
def convert_to_index_name(s):
    """Converts a string to an Elasticsearch index name."""
    # For Elasticsearch index name restrictions, see
    # https://github.com/DataBiosphere/data-explorer-indexers/issues/5#issue-308168951
    prohibited_chars = [' ', '"', '*', '\\', '<', '|', ',', '>', '/', '?']
    for char in prohibited_chars:
        s = s.replace(char, '_')
    s = s.lower()
    # Remove leading underscore.
    if s.find('_', 0, 1) == 0:
        s = s.lstrip('_')
    print('Index name: %s' % s)
    return s


def get_index_name():
    """Gets Elasticsearch index name."""
    return convert_to_index_name(get_dataset_name())


def get_dataset_name():
    """Gets dataset name from dataset.json.

    current_app.config['DATASET_CONFIG_DIR'] must be set before this is called.
    """
    file_path = os.path.join(current_app.config['DATASET_CONFIG_DIR'],
                             'dataset.json')
    with open(file_path) as f:
        # Remove comments using jsmin, as recommended by JSON creator
        # (https://plus.google.com/+DouglasCrockfordEsq/posts/RK8qyGVaGSr).
        dataset = json.loads(jsmin.jsmin(f.read()))
        return dataset['name']


def get_facets():
    """Gets facets from facet_fields.csv.

    current_app.config['ELASTICSEARCH_URL'], current_app.config['INDEX_NAME']
    current_app.config['DATASET_CONFIG_DIR'] must be set before this is called.
    """
    f = open(
        os.path.join(current_app.config['DATASET_CONFIG_DIR'],
                     'facet_fields.csv'))
    # Remove comments using jsmin.
    csv_str = jsmin.jsmin(f.read())
    facet_rows = csv.DictReader(
        iter(csv_str.splitlines()), skipinitialspace=True)

    using = Elasticsearch(current_app.config['ELASTICSEARCH_URL'])
    mapping = Mapping.from_es(
        current_app.config['INDEX_NAME'], 'type', using=using).to_dict()
    # Preserve order, so facets are returned in the same order as facet_fields.csv
    facets = OrderedDict()
    for facet_row in facet_rows:
        field_name = facet_row['readable_field_name']
        field_type = mapping['type']['properties'][field_name]['type']
        if field_type == 'text':
            # Use ".keyword" because we want aggregation on keyword field, not
            # term field. See
            # https://www.elastic.co/guide/en/elasticsearch/reference/6.2/fielddata.html#before-enabling-fielddata
            facets[field_name] = TermsFacet(
                field=field_name + '.keyword', size=20)
        else:
            # Assume numeric type.
            # TODO: Handle other types.
            # TODO: Automatically figure out bucket intervals. Unfortunately
            # Elasticsearch won't do this for us
            # (https://github.com/elastic/elasticsearch/issues/9572). Make the
            # ranges easy to read (10-19,20-29 instead of 10-17,18-25).
            facets[field_name] = HistogramFacet(field=field_name, interval=10)
    current_app.logger.info('dataset_faceted_search facets: %s' % facets)
    return facets
{'fraction_non_alphanumeric': 0.11530786773090079, 'fraction_numerical': 0.011687571265678449, 'mean_word_length': 3.144713526284702, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4253076', 'n_tokens_mistral': 2198, 'n_tokens_neox': 2113, 'n_words': 521}