<commit_before>from retrieval.search_engine import SearchEngine
from other.data import getStopWords
from preprocess.index_manager import IndexManager
import sys
from common.io import readfile
from other.constants import DATA_FOLDER, DATABASES_FOLDER

## functions 
def getCommand(text):
	spaceIndex = text.find(' ')
	if spaceIndex == -1:
		return text
	
	return text[:spaceIndex]

def getQuery(text):
	spaceIndex = text.find(' ')
	return str(text[spaceIndex + 1:])

def printSearch(query, nosco):
	result = nosco.search(query)
	for item in result:
		item['score'] = round(item['score'], 3)
		item['url'] = item['url'].replace('http://', '')
		item['keywords'] = item['keywords'][:3]
		print(item)
	print('Search query: ' + str(nosco.lastQuery))
	print('Number results: ' + str(len(result)))


## main

try:
	database = DATABASES_FOLDER + str(sys.argv[1]) + '/'
except IndexError:
	print('Too many arguments:')
	print('search.py [database folder]')
	exit()<commit_msg>refactoring<commit_after>from retrieval.search_engine import SearchEngine
from other.data import getStopWords
from preprocess.index_manager import IndexManager
import sys
from common.io import readfile
from other.constants import DATA_FOLDER, DATABASES_FOLDER

## functions 
def getCommand(text):
	spaceIndex = text.find(' ')
	if spaceIndex == -1:
		return text
	
	return text[:spaceIndex]

def getQuery(text):
	spaceIndex = text.find(' ')
	return str(text[spaceIndex + 1:])

def printSearch(query, nosco):
	result = nosco.search(query)
	for item in result:
		item['score'] = round(item['score'], 3)
		item['url'] = item['url'].replace('http://', '')
		item['keywords'] = item['keywords'][:3]
		print(item)
	print('Search query: ' + str(nosco.lastQuery))
	print('Number results: ' + str(len(result)))


## main

try:
	database = DATABASES_FOLDER + str(sys.argv[1]) + '/'
except IndexError:
	print('Too few arguments:')
	print('search.py [database folder]')
	sys.exit()
{'fraction_non_alphanumeric': 0.15038759689922482, 'fraction_numerical': 0.00516795865633075, 'mean_word_length': 5.54054054054054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '861171', 'n_tokens_mistral': 671, 'n_tokens_neox': 640, 'n_words': 157}