<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import


def _retrieve_int(querydict, key):
    """
    Retrieve an int from a dict or None
    :param querydict: query dictionary
    :param key: key to retrieve
    :return: integer or None
    """
    try:
        return int(querydict.pop(key)[0])
    except ValueError or KeyError:
        return None


def sanitize(text):
    """
    Sanitize text for Lucene
    :param text: text
    :return: sanitized text
    """
    if text:
        return text.lower().replace(":", "").encode('utf-8')
    raise ValueError


valid_fields = {
    'title',
    'genre',
    'language',
    'country',
    'director',
    'writer',
    'cast',
    'year_start',
    'year_end'
}


def check_field(field):
    """
    Check if the given field is valid
    :param field: field
    :return: field or KeyError
    """
    if field.lower() in valid_fields:
        return field.lower()
    raise KeyError(field + " is not a valid field")


def prepare_query(request):
    """
    Parse GET parameters to generate a query for Lucene
    :param request: request object
    :return: [(field1, text1), (field2, text2), ...]
    """
    # Get a mutable copy of GET parameters
    query = request.GET.copy()

    # Remove pagination parameters
    query.pop('last_id', None)
    query.pop('last_score', None)

    # Manage search bar parameters
    if 'query' in query:
        for term in query.pop('query'):
            try:
                idx = term.index(":")
                field = check_field(term[:idx])
                text = term[idx + 1:].strip()
                query.appendlist(field, sanitize(text))
            except (KeyError, ValueError):
                # Invalid field, treat as a title search<commit_msg>Strip white spaces in queries<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import


def _retrieve_int(querydict, key):
    """
    Retrieve an int from a dict or None
    :param querydict: query dictionary
    :param key: key to retrieve
    :return: integer or None
    """
    try:
        return int(querydict.pop(key)[0])
    except ValueError or KeyError:
        return None


def sanitize(text):
    """
    Sanitize text for Lucene
    :param text: text
    :return: sanitized text
    """
    if text:
        return text.lower().strip().replace(":", "").encode('utf-8')
    raise ValueError


valid_fields = {
    'title',
    'genre',
    'language',
    'country',
    'director',
    'writer',
    'cast',
    'year_start',
    'year_end'
}


def check_field(field):
    """
    Check if the given field is valid
    :param field: field
    :return: field or KeyError
    """
    sanitized_field = sanitize(field)
    if sanitized_field in valid_fields:
        return sanitized_field
    raise KeyError(field + " is not a valid field")


def prepare_query(request):
    """
    Parse GET parameters to generate a query for Lucene
    :param request: request object
    :return: [(field1, text1), (field2, text2), ...]
    """
    # Get a mutable copy of GET parameters
    query = request.GET.copy()

    # Remove pagination parameters
    query.pop('last_id', None)
    query.pop('last_score', None)

    # Manage search bar parameters
    if 'query' in query:
        for term in query.pop('query'):
            try:
                idx = term.index(":")
                field = check_field(term[:idx])
                text = term[idx + 1:].strip()
                query.appendlist(field, sanitize(text))
            except (KeyError, ValueError):
                # Invalid field, treat as a title search
{'fraction_non_alphanumeric': 0.11699779249448124, 'fraction_numerical': 0.004415011037527594, 'mean_word_length': 2.310502283105023, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6404394', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1068, 'n_words': 337}