<commit_before>import os
import json
from ..logger import logger

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import NotFoundError

_engine = None

PACKAGES_INDEX_NAME = os.environ.get('OS_ES_PACKAGES_INDEX_NAME', 'packages')

ENABLED_SEARCHES = {
    # 'user': {
    #     'index': 'users',
    #     'doc_type': 'user_profile',
    #     '_source': ['idhash', 'name', 'avatar_url', 'datasets'],
    #     'q_fields': ['name']
    # },
    'package': {
        'index': PACKAGES_INDEX_NAME,
        'doc_type': 'package',
        '_source': ['id', 'model', 'package',
                    'origin_url', 'loaded', 'last_update'],
        'owner': 'package.owner',
        'private': 'package.private',
        'q_fields': ['package.title',
                     'package.author',
                     'package.owner',
                     'package.description',
                     'package.regionCode',
                     'package.countryCode',
                     'package.cityCode'],
    }
}


def _get_engine():
    global _engine
    if _engine is None:
        es_host = os.environ['OS_ELASTICSEARCH_ADDRESS']
        _engine = Elasticsearch(hosts=[es_host], use_ssl='https' in es_host)
    return _engine


def build_dsl(kind_params, userid, kw):
    dsl = {'bool': {'should': [], 'must': [], 'minimum_should_match': 1}}
    # All Datasets:
    all_datasets = {
        'bool': {
            'should': [{'match': {kind_params['private']: False}},
                       {'bool': {
                            'must_not': {
                                'exists': {
                                    'field': kind_params['private']
                                }
                            }
                        }},
                       ],
            'must_not': {'match': {'loaded': False}},
            'minimum_should_match': 1
        }
    }
    dsl['bool']['should'].append(all_datasets)

    # User datasets
    if userid is not None:
        user_datasets = \
            {'bool': {'must': {'match': {kind_params['owner']: userid}}}}
        dsl['bool']['should'].append(user_datasets)

    # Query parameters
    q = kw.get('q')
    if q is not None:
        dsl['bool']['must'].append({
                'multi_match': {
                    'query': json.loads(q[0]),
                    'fields': kind_params['q_fields']
                }
            })
    for k, v_arr in kw.items():
        if k.split('.')[0] in kind_params['_source']:
            if not isinstance(v_arr, list):
                v_arr = [v_arr]
            dsl['bool']['must'].append({<commit_msg>Ensure q arg is a list before iterating<commit_after>import os
import json
from ..logger import logger

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import NotFoundError

_engine = None

PACKAGES_INDEX_NAME = os.environ.get('OS_ES_PACKAGES_INDEX_NAME', 'packages')

ENABLED_SEARCHES = {
    # 'user': {
    #     'index': 'users',
    #     'doc_type': 'user_profile',
    #     '_source': ['idhash', 'name', 'avatar_url', 'datasets'],
    #     'q_fields': ['name']
    # },
    'package': {
        'index': PACKAGES_INDEX_NAME,
        'doc_type': 'package',
        '_source': ['id', 'model', 'package',
                    'origin_url', 'loaded', 'last_update'],
        'owner': 'package.owner',
        'private': 'package.private',
        'q_fields': ['package.title',
                     'package.author',
                     'package.owner',
                     'package.description',
                     'package.regionCode',
                     'package.countryCode',
                     'package.cityCode'],
    }
}


def _get_engine():
    global _engine
    if _engine is None:
        es_host = os.environ['OS_ELASTICSEARCH_ADDRESS']
        _engine = Elasticsearch(hosts=[es_host], use_ssl='https' in es_host)
    return _engine


def build_dsl(kind_params, userid, kw):
    dsl = {'bool': {'should': [], 'must': [], 'minimum_should_match': 1}}
    # All Datasets:
    all_datasets = {
        'bool': {
            'should': [{'match': {kind_params['private']: False}},
                       {'bool': {
                            'must_not': {
                                'exists': {
                                    'field': kind_params['private']
                                }
                            }
                        }},
                       ],
            'must_not': {'match': {'loaded': False}},
            'minimum_should_match': 1
        }
    }
    dsl['bool']['should'].append(all_datasets)

    # User datasets
    if userid is not None:
        user_datasets = \
            {'bool': {'must': {'match': {kind_params['owner']: userid}}}}
        dsl['bool']['should'].append(user_datasets)

    # Query parameters
    q = kw.get('q')
    if q is not None:
        if not isinstance(q, list):
            q = [q]
        dsl['bool']['must'].append({
                'multi_match': {
                    'query': json.loads(q[0]),
                    'fields': kind_params['q_fields']
                }
            })
    for k, v_arr in kw.items():
        if k.split('.')[0] in kind_params['_source']:
            if not isinstance(v_arr, list):
                v_arr = [v_arr]
            dsl['bool']['must'].append({
{'fraction_non_alphanumeric': 0.1504872563718141, 'fraction_numerical': 0.0014992503748125937, 'mean_word_length': 1.6591928251121075, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '914577', 'n_tokens_mistral': 1579, 'n_tokens_neox': 1533, 'n_words': 312}