<commit_before># -*- coding: utf-8 -*- 
import re

from django.db.models import Q

def normalize_query(query_string,
                    findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
                    normspace=re.compile(r'\s{2,}').sub):
    return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)] 



#fields = [f for f in table._meta.fields if isinstance(f, CharField)]
#queries = [Q(**{f.name: SEARCH_TERM}) for f in fields]

#qs = Q()
#for query in queries:
    #qs = qs | query

#table.objects.filter(qs)

def get_query(query_string, model, search_fields):
    query = None
    terms = normalize_query(query_string)
    for term in terms:
        or_query = None
        for field_name in search_fields:
            q = Q(**{(u'{}__icontains'.format(field): term})
            if or_query is None:
                or_query = q
            else:
                or_query = or_query | q
        if query is None:
            query = or_query
        else:
            query = query & or_query
    return query

def model_search(model, query_string):
	model_string = model.str()
	# change this to Model.objects.filter
	search_fields = [field for field in model._meta.get_all_field_names()]
	query_results = get_query(query_string, model search_fields)
	return model.filter(query_results)

def multi_model_search(model_list, query_string):
	results = []
	for model in model_list:
		results += model_search(model, query_string)
	return results<commit_msg>worked on search ideas a bit<commit_after># -*- coding: utf-8 -*- 
import re

from django.db.models import Q

def normalize_query(query_string,
                    findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
                    normspace=re.compile(r'\s{2,}').sub):
    return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)] 



#fields = [f for f in table._meta.fields if isinstance(f, CharField)]
#queries = [Q(**{f.name: SEARCH_TERM}) for f in fields]

#qs = Q()
#for query in queries:
    #qs = qs | query

#table.objects.filter(qs)

def get_query(query_string, search_fields):
    query = None
    terms = normalize_query(query_string)
    for term in terms:
        or_query = None
        for field_name in search_fields:
            q = Q(**{(u'{}__icontains'.format(field): term})
            if or_query is None:
                or_query = q
            else:
                or_query = or_query | q
        if query is None:
            query = or_query
        else:
            query = query & or_query
    return query

def model_search(model, query_string):
	model_string = model.str()
	# change this to Model.objects.filter
	search_fields = [field for field in model._meta.get_all_field_names()]
	query_results = get_query(query_string, search_fields)
	return model.filter(query_results)

def multi_model_search(model_list, query_string):
	results = []
	for model in model_list:
		results += model_search(model, query_string)
	return results
{'fraction_non_alphanumeric': 0.12063172043010753, 'fraction_numerical': 0.002688172043010753, 'mean_word_length': 2.84625322997416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5294293', 'n_tokens_mistral': 980, 'n_tokens_neox': 950, 'n_words': 252}