<commit_before>import re
from pyramid.view import view_config
from contentbase import (
    Collection,
    TYPES,
    collection_view_listing_db,
)
from contentbase.elasticsearch import ELASTIC_SEARCH
from pyramid.security import effective_principals
from urllib.parse import urlencode
from collections import OrderedDict


def includeme(config):
    config.add_route('search', '/search{slash:/?}')
    config.scan(__name__)


sanitize_search_string_re = re.compile(r'[\\\+\-\&\|\!\(\)\{\}\[\]\^\~\:\/\\\*\?]')

hgConnect = ''.join([
    'http://genome.ucsc.edu/cgi-bin/hgHubConnect',
    '?hgHub_do_redirect=on',
    '&hgHubConnect.remakeTrackHub=on',
    '&hgHub_do_firstDb=1',
    '&hubUrl=',
])

audit_facets = [
    ('audit.ERROR.category', {'title': 'Audit category: ERROR'}),
    ('audit.NOT_COMPLIANT.category', {'title': 'Audit category: NOT COMPLIANT'}),
    ('audit.WARNING.category', {'title': 'Audit category: WARNING'}),
    ('audit.DCC_ACTION.category', {'title': 'Audit category: DCC ACTION'})
]


def get_filtered_query(term, search_fields, result_fields, principals):
    return {
        'query': {
            'query_string': {
                'query': term,
                'fields': search_fields,
                'default_operator': 'AND'
            }
        },
        'filter': {
            'and': {
                'filters': [
                    {
                        'terms': {
                            'principals_allowed.view': principals
                        }
                    }
                ]
            }
        },
        'aggs': {},
        '_source': list(result_fields),
    }


def sanitize_search_string(text):
    return sanitize_search_string_re.sub(r'\\\g<0>', text)


def get_sort_order():
    """
    specifies sort order for elasticsearch results
    """
    return {
        'embedded.date_created': {
            'order': 'desc',
            'ignore_unmapped': True,
        },
        'embedded.label': {
            'order': 'asc',
            'missing': '_last',
            'ignore_unmapped': True,
        },
    }


def get_search_fields(request, doc_types):
    """
    Returns set of columns that are being searched and highlights
    """
    fields = set()
    highlights = {}
    for doc_type in (doc_types or request.root.by_item_type.keys()):
        collection = request.root[doc_type]
        for value in collection.type_info.schema.get('boost_values', ()):
            fields.add('embedded.' + value)
            highlights['embedded.' + value] = {}
    return fields, highlights


def load_columns(request, doc_types, result):
    """<commit_msg>autosort in elasticsearch using correct field<commit_after>import re
from pyramid.view import view_config
from contentbase import (
    Collection,
    TYPES,
    collection_view_listing_db,
)
from contentbase.elasticsearch import ELASTIC_SEARCH
from pyramid.security import effective_principals
from urllib.parse import urlencode
from collections import OrderedDict


def includeme(config):
    config.add_route('search', '/search{slash:/?}')
    config.scan(__name__)


sanitize_search_string_re = re.compile(r'[\\\+\-\&\|\!\(\)\{\}\[\]\^\~\:\/\\\*\?]')

hgConnect = ''.join([
    'http://genome.ucsc.edu/cgi-bin/hgHubConnect',
    '?hgHub_do_redirect=on',
    '&hgHubConnect.remakeTrackHub=on',
    '&hgHub_do_firstDb=1',
    '&hubUrl=',
])

audit_facets = [
    ('audit.ERROR.category', {'title': 'Audit category: ERROR'}),
    ('audit.NOT_COMPLIANT.category', {'title': 'Audit category: NOT COMPLIANT'}),
    ('audit.WARNING.category', {'title': 'Audit category: WARNING'}),
    ('audit.DCC_ACTION.category', {'title': 'Audit category: DCC ACTION'})
]


def get_filtered_query(term, search_fields, result_fields, principals):
    return {
        'query': {
            'query_string': {
                'query': term,
                'fields': search_fields,
                'default_operator': 'AND'
            }
        },
        'filter': {
            'and': {
                'filters': [
                    {
                        'terms': {
                            'principals_allowed.view': principals
                        }
                    }
                ]
            }
        },
        'aggs': {},
        '_source': list(result_fields),
    }


def sanitize_search_string(text):
    return sanitize_search_string_re.sub(r'\\\g<0>', text)


def get_sort_order():
    """
    specifies sort order for elasticsearch results
    """
    return {
        'embedded.dateTime': {
            'order': 'desc',
            'ignore_unmapped': True,
        },
        'embedded.label': {
            'order': 'asc',
            'missing': '_last',
            'ignore_unmapped': True,
        },
    }


def get_search_fields(request, doc_types):
    """
    Returns set of columns that are being searched and highlights
    """
    fields = set()
    highlights = {}
    for doc_type in (doc_types or request.root.by_item_type.keys()):
        collection = request.root[doc_type]
        for value in collection.type_info.schema.get('boost_values', ()):
            fields.add('embedded.' + value)
            highlights['embedded.' + value] = {}
    return fields, highlights


def load_columns(request, doc_types, result):
    """
{'fraction_non_alphanumeric': 0.14501510574018128, 'fraction_numerical': 0.0007552870090634441, 'mean_word_length': 2.206416464891041, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1646080', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1597, 'n_words': 311}