<commit_before>__author__ = '<NAME>'


from invdx import build_data_structures
from rank import *
from collections import OrderedDict
import operator


class QueryProcessor:
	def __init__(self, queries, idx, dlt, ft):
		self.queries = queries
		#self.index, self.ft, self.dlt = build_data_structures(corpus)
		self.idx_file = idx
		self.dlt = dlt
		self.ft = ft

	def run_query_likelihood(self, query):
		mu = 2000
		print 'query:', query
		print "running query with mu=%d" % mu
		result = OrderedDict()  # collect document rankings for this value of mu
		for term in query:
			with open(self.idx_file, buffering=1000000000) as idx:
				print 'searching index'
				for line in idx:
					tmp = line.split()
					word = tmp.pop(0)
					freq = [tuple(x.split('-')) for x in tmp]
					if term == word:
						print 'found word in index:', word
						docs = set()
<commit_msg>Remove queries param<commit_after>__author__ = '<NAME>'


from invdx import build_data_structures
from rank import *
from collections import OrderedDict
import operator


class QueryProcessor:
	def __init__(self, idx, dlt, ft):
		#self.index, self.ft, self.dlt = build_data_structures(corpus)
		self.idx_file = idx
		self.dlt = dlt
		self.ft = ft

	def run_query_likelihood(self, query):
		mu = 2000
		print 'query:', query
		print "running query with mu=%d" % mu
		result = OrderedDict()  # collect document rankings for this value of mu
		for term in query:
			with open(self.idx_file, buffering=1000000000) as idx:
				print 'searching index'
				for line in idx:
					tmp = line.split()
					word = tmp.pop(0)
					freq = [tuple(x.split('-')) for x in tmp]
					if term == word:
						print 'found word in index:', word
						docs = set()

{'fraction_non_alphanumeric': 0.1044950379451255, 'fraction_numerical': 0.017513134851138354, 'mean_word_length': 3.4404145077720205, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2705688', 'n_tokens_mistral': 682, 'n_tokens_neox': 560, 'n_words': 191}