<commit_before>from retrieval.boolean_parser import BooleanParser
from retrieval.ranking import score
from retrieval.index import Index
from preprocess.words import getstem
from common.funcfun import lmap
from common.string import normalize_text

class SearchEngine:
	def __init__(self, index_folder, stopwords):
		self.parser = BooleanParser()
		self.index = Index(index_folder)
		self.stopwords = stopwords
	
	def search(self, query):
		normQuery = normalize_text(query)
		parsedQuery, terms = self._parse_query(normQuery)
		documents = self.index.get_documents(parsedQuery)
		rankedResults = score(terms, documents, self.index)
		sortedResults = sorted(rankedResults, key=lambda doc: doc['score'], reverse=True)
		return sortedResults
		
		
	def _parse_query(self, query):
		self.lastQuery = self.parser.parse(query, self.stopwords)
		terms = self.parser.terms(self.lastQuery)
		return self.lastQuery, lmap(getstem, terms)<commit_msg>bug fixed<commit_after>from retrieval.boolean_parser import BooleanParser, Node
from retrieval.ranking import score
from retrieval.index import Index
from preprocess.words import getstem
from common.funcfun import lmap
from common.string import remove_nonletters


class SearchEngine:
	def __init__(self, index_folder, stopwords):
		self.parser = BooleanParser()
		self.index = Index(index_folder)
		self.stopwords = stopwords
	
	def search(self, query):
		normQuery = remove_nonletters(query, ' ')
		parsedQuery, terms = self._parse_query(normQuery)
		documents = self.index.get_documents(parsedQuery)
		rankedResults = score(terms, documents, self.index)
		sortedResults = sorted(rankedResults, key=lambda doc: doc['score'], reverse=True)
		return sortedResults
		
		
	def _parse_query(self, query):
		self.lastQuery = self.parser.parse(query, self.stopwords)
		self.lastQuery = self._query_to_stems(self.lastQuery)
		terms = self.parser.terms(self.lastQuery)
		return self.lastQuery, lmap(getstem, terms)
	
	def _query_to_stems(self, query):
		if isinstance(query, Node):
			query.children = lmap(self._query_to_stems, query.children)
			return query
		else:
			if query:
				return getstem(query)
			else:
				return query
{'fraction_non_alphanumeric': 0.09061488673139159, 'fraction_numerical': 0.0, 'mean_word_length': 5.440476190476191, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2951380', 'n_tokens_mistral': 729, 'n_tokens_neox': 648, 'n_words': 162}