<commit_before>from skylines.model import User, Club, Airport
from skylines.model.search import (
    combined_search_query, escape_tokens, text_to_tokens
)

MODELS = [User, Club, Airport]


def search(text):
    # Split the search text into tokens and escape them properly
    tokens = text_to_tokens(text)
    tokens = escape_tokens(tokens)

    # Create combined search query
    return combined_search_query(MODELS, tokens)


def test_tokenizer():
    # Check that this does not throw exceptions
    text_to_tokens('\\')
    text_to_tokens('blabla \\')
    text_to_tokens('"')
    text_to_tokens('"blabla \\')

    # Check that the tokenizer returns expected results
    assert text_to_tokens('a b c') == ['a', 'b', 'c']
    assert text_to_tokens('a \'b c\'') == ['a', 'b c']
    assert text_to_tokens('a "b c" d') == ['a', 'b c', 'd']
    assert text_to_tokens('old "mac donald" has a FARM') == \
        ['old', 'mac donald', 'has', 'a', 'FARM']


def test_escaping():
    assert escape_tokens(['hello!']) == ['hello!']
    assert escape_tokens(['hello *!']) == ['hello %!']
    assert escape_tokens(['hello %!']) == ['hello \\%!']
    assert escape_tokens(['hello _!']) == ['hello \\_!']


def test_search(bootstrapped_db):
    assert search('example').count() == 2
    assert search('user').count() == 1
    assert search('man').count() == 1
    assert search('man*er').count() == 1
    assert search('*er').count() == 2<commit_msg>tests/model/test_search: Use new 'test_user/admin' fixtures<commit_after>from skylines.model import User, Club, Airport
from skylines.model.search import (
    combined_search_query, escape_tokens, text_to_tokens
)

MODELS = [User, Club, Airport]


def search(text):
    # Split the search text into tokens and escape them properly
    tokens = text_to_tokens(text)
    tokens = escape_tokens(tokens)

    # Create combined search query
    return combined_search_query(MODELS, tokens)


def test_tokenizer():
    # Check that this does not throw exceptions
    text_to_tokens('\\')
    text_to_tokens('blabla \\')
    text_to_tokens('"')
    text_to_tokens('"blabla \\')

    # Check that the tokenizer returns expected results
    assert text_to_tokens('a b c') == ['a', 'b', 'c']
    assert text_to_tokens('a \'b c\'') == ['a', 'b c']
    assert text_to_tokens('a "b c" d') == ['a', 'b c', 'd']
    assert text_to_tokens('old "mac donald" has a FARM') == \
        ['old', 'mac donald', 'has', 'a', 'FARM']


def test_escaping():
    assert escape_tokens(['hello!']) == ['hello!']
    assert escape_tokens(['hello *!']) == ['hello %!']
    assert escape_tokens(['hello %!']) == ['hello \\%!']
    assert escape_tokens(['hello _!']) == ['hello \\_!']


def test_search(test_user, test_admin):
    assert search('example').count() == 2
    assert search('user').count() == 1
    assert search('man').count() == 1
    assert search('man*er').count() == 1
    assert search('*er').count() == 2
{'fraction_non_alphanumeric': 0.17496589358799455, 'fraction_numerical': 0.0034106412005457027, 'mean_word_length': 3.880199667221298, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3682043', 'n_tokens_mistral': 975, 'n_tokens_neox': 943, 'n_words': 256}