<commit_before>from gdcdatamodel import node_avsc_json
import re
from mapped_entities import (
    file_tree, file_traversal,
    participant_tree, participant_traversal,
    annotation_tree, annotation_traversal,
    project_tree, annotation_tree,
    ONE_TO_MANY, ONE_TO_ONE
)


MULTIFIELDS = re.compile("|".join([
    "submitter_id", "name", "code", "primary_site",
    "disease_type", "project_name"
]))


def index_settings():
    return {"settings":
            {"analysis":
             {"analyzer":
              {"id_search":
               {"tokenizer": "whitespace",
                "filter": ["lowercase"],
                "type": "custom"},
               "id_index": {
                   "tokenizer": "whitespace",
                   "filter": [
                       "lowercase",
                       "edge_ngram"
                   ],
                   "type": "custom"
               }},
              "filter": {
                  "edge_ngram": {
                      "side": "front",
                      "max_gram": 20,
                      "min_gram": 2,
                      "type": "edge_ngram"
                  }}}}}


def _get_header():
    return {
        "dynamic": "true",
        "dynamic_templates": [{
            "template_1": {
                "mapping": {
                    "index": "not_analyzed"
                },
                "match": "*",
                "match_mapping_type": "string"
            }
        }],
        "_all": {
            "enabled": False
        },
        "_source": {
            "compress": True,
            "excludes": ["__comment__"]
        },
    }


def _get_es_type(_type):
    if 'long' in _type or 'int' in _type:
        return 'long'
    else:
        return 'string'


def _munge_properties(source):
    a = [n['fields'] for n in node_avsc_json if n['name'] == source]
    if not a:
        return
    fields = [b['type'] for b in a[0] if b['name'] == 'properties']
    doc = _multfield_template('{}_id'.format(source))
    for field in fields[0][0]['fields']:
        name = field['name']
        if MULTIFIELDS.match(name):
            doc.update(_multfield_template(name))
        else:
            _type = _get_es_type(
                [c['type'] if isinstance(c['type'], (unicode, str))
                 else 'string' for c in b['type'][0]['fields']
                 if c['name'] == name])
            doc[name] = {'type': _type}
            if str(_type) == 'string':
                doc[name]['index'] = 'not_analyzed'
    return doc


def _multfield_template(name):
    return {name: {
        "type": "string", "fields": {
            "raw": {
                "index": "not_analyzed",
                "store": "yes",
                "type": "string"
            }, "analyzed": {
                "index": "analyzed",
                "index_analyzer": "id_index",
                "search_analyzer": "id_search",
                "type": "string"
            }, "search": {
                "index": "analyzed",
                "analyzer": "id_search",
                "type": "string"
            }}}}


def _walk_tree(tree, mapping, level=0):
    for k, v in [(k, v) for k, v in tree.items() if k != 'corr']:
        corr, name = v['corr']
        # print '|--'*level, name
        if name not in mapping:
            mapping[name] = {'properties': {}}
        mapping[name]['properties'].update(_munge_properties(k))
        _walk_tree(tree[k], mapping[name]['properties'], level+1)
        if corr == ONE_TO_MANY:
            mapping[name]['type'] = 'nested'
    return mapping


def get_file_es_mapping(include_participant=True):
    files = _get_header()
    files["_id"] = {"path": "file_id"}
    files["properties"] = _walk_tree(file_tree, _munge_properties("file"))
    files["properties"]['related_files'] = {<commit_msg>fix nesting of es mapping fields to be under properties key<commit_after>from gdcdatamodel import node_avsc_json
import re
from mapped_entities import (
    file_tree, file_traversal,
    participant_tree, participant_traversal,
    annotation_tree, annotation_traversal,
    project_tree, annotation_tree,
    ONE_TO_MANY, ONE_TO_ONE
)


MULTIFIELDS = re.compile("|".join([
    "submitter_id", "name", "code", "primary_site",
    "disease_type", "project_name"
]))


def index_settings():
    return {"settings":
            {"analysis":
             {"analyzer":
              {"id_search":
               {"tokenizer": "whitespace",
                "filter": ["lowercase"],
                "type": "custom"},
               "id_index": {
                   "tokenizer": "whitespace",
                   "filter": [
                       "lowercase",
                       "edge_ngram"
                   ],
                   "type": "custom"
               }},
              "filter": {
                  "edge_ngram": {
                      "side": "front",
                      "max_gram": 20,
                      "min_gram": 2,
                      "type": "edge_ngram"
                  }}}}}


def _get_header():
    return {
        "dynamic": "true",
        "dynamic_templates": [{
            "template_1": {
                "mapping": {
                    "index": "not_analyzed"
                },
                "match": "*",
                "match_mapping_type": "string"
            }
        }],
        "_all": {
            "enabled": False
        },
        "_source": {
            "compress": True,
            "excludes": ["__comment__"]
        },
    }


def _get_es_type(_type):
    if 'long' in _type or 'int' in _type:
        return 'long'
    else:
        return 'string'


def _munge_properties(source):
    a = [n['fields'] for n in node_avsc_json if n['name'] == source]
    if not a:
        return
    fields = [b['type'] for b in a[0] if b['name'] == 'properties']
    doc = _multfield_template('{}_id'.format(source))
    for field in fields[0][0]['fields']:
        name = field['name']
        if MULTIFIELDS.match(name):
            doc.update(_multfield_template(name))
        else:
            _type = _get_es_type(
                [c['type'] if isinstance(c['type'], (unicode, str))
                 else 'string' for c in b['type'][0]['fields']
                 if c['name'] == name])
            doc[name] = {'type': _type}
            if str(_type) == 'string':
                doc[name]['index'] = 'not_analyzed'
    return doc


def _multfield_template(name):
    return {name: {
        "type": "string", "fields": {
            "raw": {
                "index": "not_analyzed",
                "store": "yes",
                "type": "string"
            }, "analyzed": {
                "index": "analyzed",
                "index_analyzer": "id_index",
                "search_analyzer": "id_search",
                "type": "string"
            }, "search": {
                "index": "analyzed",
                "analyzer": "id_search",
                "type": "string"
            }}}}


def _walk_tree(tree, mapping, level=0):
    for k, v in [(k, v) for k, v in tree.items() if k != 'corr']:
        corr, name = v['corr']
        if name not in mapping:
            mapping[name] = {'properties': {}}
        mapping[name]['properties'].update(_munge_properties(k))
        _walk_tree(tree[k], mapping[name]['properties'], level+1)
        if corr == ONE_TO_MANY:
            mapping[name]['type'] = 'nested'
    return mapping


def get_file_es_mapping(include_participant=True):
    files = _get_header()
    files["_id"] = {"path": "file_id"}
    files["properties"] = _walk_tree(file_tree, _munge_properties("file"))
    files["properties"]['related_files'] = {
{'fraction_non_alphanumeric': 0.1493082746019316, 'fraction_numerical': 0.0026102845210127902, 'mean_word_length': 1.6561525129982668, 'pattern_counts': {'":': 90, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '648416', 'n_tokens_mistral': 2337, 'n_tokens_neox': 2222, 'n_words': 485}