<commit_before>__author__ = 'leif'

import os
#from ifind.common.query_generation import SingleQueryGeneration, BiTermQueryGeneration, TriTermQueryGeneration
#from ifind.common.language_model import LanguageModel
#from ifind.common.smoothed_language_model import SmoothedLanguageModel, BayesLanguageModel
#from ifind.common.query_ranker import QueryRanker, OddsRatioQueryRanker

from ifind.search.query import Query
from search_context import SearchContext
from search_interface import WhooshInterface, Document, Topic
from text_classifier import iFindTextClassifier
from sim_user import SimUser

from query_producer import QueryProducer, SmarterQueryProducer
import random
from exp_log import ExpLog

work_dir = os.getcwd()
my_whoosh_doc_index_dir = os.path.join(work_dir, 'data/fullindex')

si = WhooshInterface(my_whoosh_doc_index_dir)

#action_handler = { 'Q': do_query, 'D': do_assess}

stopword_file = 'data/stopwords.txt'
bg_file = 'data/vocab.txt'
topic_file = 'topic.307'


t = Topic('307')
t.read_topic_from_file(topic_file)

qp = SmarterQueryProducer(stopword_file)

query_list = qp.produce_query_list(t)

for q in query_list:
    print q


log = ExpLog(limit=750)

tc = iFindTextClassifier(stopword_file, bg_file)
tc.threshold = -0.20

bu = SimUser(si, qp, tc, log, t)

while log.is_finished():
    bu.decide_action()

al =  bu.get_actions_performed()

bu.report()

bu.save_rel_judgements('test.out')<commit_msg>minor change<commit_after>__author__ = 'leif'

import os
#from ifind.common.query_generation import SingleQueryGeneration, BiTermQueryGeneration, TriTermQueryGeneration
#from ifind.common.language_model import LanguageModel
#from ifind.common.smoothed_language_model import SmoothedLanguageModel, BayesLanguageModel
#from ifind.common.query_ranker import QueryRanker, OddsRatioQueryRanker

from ifind.search.query import Query
from search_context import SearchContext
from search_interface import WhooshInterface, Document, Topic
from text_classifier import iFindTextClassifier
from sim_user import SimUser

from query_producer import QueryProducer, SmarterQueryProducer
import random
from exp_log import ExpLog

work_dir = os.getcwd()
my_whoosh_doc_index_dir = os.path.join(work_dir, 'data/fullindex')

si = WhooshInterface(my_whoosh_doc_index_dir)

#action_handler = { 'Q': do_query, 'D': do_assess}

stopword_file = 'data/stopwords.txt'
bg_file = 'data/vocab.txt'
topic_file = 'topic.307'


t = Topic('307')
t.read_topic_from_file(topic_file)

qp = SmarterQueryProducer(stopword_file)

query_list = qp.produce_query_list(t)

for q in query_list:
    print q


log = ExpLog(limit=750)

tc = iFindTextClassifier(stopword_file, bg_file)
tc.threshold = -0.15

bu = SimUser(si, qp, tc, log, t)

while log.is_finished():
    bu.decide_action()

al =  bu.get_actions_performed()

bu.report()

bu.save_rel_judgements('test.out')
{'fraction_non_alphanumeric': 0.08640674394099052, 'fraction_numerical': 0.008429926238145416, 'mean_word_length': 6.739130434782608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4606545', 'n_tokens_mistral': 1011, 'n_tokens_neox': 983, 'n_words': 216}