<commit_before>#!/opt/python-2.7/bin/python2.7
# script can be run without invoking python because of shebang
# will be run with correct version on Patas
# LING 573 Question Answering System
# Last edited on 4/17/14 by <NAME>
#
# This is the wrapper script.


import sys
from general_classes import *
from query_processing.query_processing import *
from info_retrieval.info_retrieval import *
from answer_processing.answer_processing import *
# from lxml import etree
from pymur import *
from os import listdir, path, walk


def main():
	# script needs to have passed in: path to TREC question file, path to index
	# first argument is the TREC question file
	q_file = open(sys.argv[1],'r')
	# second argument is the index path
	index_path = sys.argv[2]
	# should the output file be a third argument?
	# or should we output the stdout?

	# comments taken from our shared Google Doc
	# instantiate an InfoRetriever using a set of documents
	# actually, no need, index is stored in set location, InfoRetriever can access it there
	# instead, index document collection

	# do XML stripping of TREC question file
	questions = generate_q_list(q_file)
	q_file.close()
	
	# filter for only factoid questions
	questions = filter(lambda x: x.type=='FACTOID', questions)

	# for a given Question object:
	for question in questions:

		# instantiate a QueryProcessor and use to generate a set of searches and an AnswerTemplate object
		qp = QueryProcessor(question)

		# use the InfoRetriever and the search set to generate a set of passages

		# instantiate an AnswerProcessor that takes set of passages and the AnswerTemplate object and returns a ranked list of answers

		# do formatting on answer list
		



# This method takes an XML file of questions as input, creates relevant Question objects,
# and returns them in a list.

def generate_q_list(xml_file):
	# parse the xml file (using Beautiful Soup?)
	# create relevant Question objects from the parse tree
	# return the Question objects in a sequential list

	soup = BeautifulSoup(xml_file)

	soup_targets = soup.find_all('target')
	questions = []
	
	for soup_target in soup_targets[:1]:
		target = soup_target.get('text')
		soup_questions = soup_target.find_all('q')

		for soup_question in soup_questions:
			id = soup_question.get('id')
			type = soup_question.get('type')
			q = soup_question.get_text().strip()
			question = Question(id, type, q, target)
			questions.append(question)
	return questions


if __name__ == '__main__':
	main()
<commit_msg>Add import line to wrapper file<commit_after>#!/opt/python-2.7/bin/python2.7
# script can be run without invoking python because of shebang
# will be run with correct version on Patas
# LING 573 Question Answering System
# Last edited on 4/17/14 by <NAME>
#
# This is the wrapper script.


import sys
from general_classes import *
from query_processing.query_processing import *
from info_retrieval.info_retrieval import *
from answer_processing.answer_processing import *
from bs4 import BeautifulSoup
from pymur import *
from os import listdir, path, walk


def main():
	# script needs to have passed in:
	# path to documents, path to TREC question file, optionally path to output file

	# first argument is the folder with the document collection
	# for the AQUAINT corpus, this is "/corpora/LDC/LDC02T31"
	document_collection = sys.argv[1]

	# second argument is the TREC question file
	q_file = open(sys.argv[2],'r')
	
	# should the output file be a third argument?
	# or should we output the stdout?

	# comments taken from our shared Google Doc
	# instantiate an InfoRetriever using a set of documents
	# actually, no need, index is stored in set location, InfoRetriever can access it there
	# instead, index document collection
	index_documents(document_collection)

	# do XML stripping of TREC question file
	questions = generate_q_list(q_file)
	q_file.close()
	
	# filter for only factoid questions
	questions = filter(lambda x: x.type=='FACTOID', questions)

	# for a given Question object:
	for question in questions:

		# instantiate a QueryProcessor and use to generate a set of searches and an AnswerTemplate object
		qp = QueryProcessor(question)

		# use the InfoRetriever and the search set to generate a set of passages

		# instantiate an AnswerProcessor that takes set of passages and the AnswerTemplate object and returns a ranked list of answers

		# do formatting on answer list
		

def index_documents(document_collection):
	# the document collection is a folder
	# first, find the subfolders (these correspond to document sources)
	# we don't want to start looking for files here because there are documentation files
	document_folders = [ path.join(document_collection,x) for x in listdir(document_collection) if path.isdir(path.join(document_collection,x)) ]
	# then find all the documents by traversing document folders
	all_documents = []
	for document_folder in document_folders:
		for (dirpath, dirnames, filenames) in walk(document_folder):
			all_documents.extend(path.join(dirpath,filename) for filename in filenames)

	# for now this is taken from http://findingscience.com/pymur/examples.html
	i = IndexEnvironment()

	# use 1000 bytes - I'm guessing we'll actually need more?
	i.setMemory(1000) 

	# do normalize - this performs case normalization - we probably want it
	i.setNormalization(True)

	# set stemmer
	# two possible stemmers - "porter" (overgeneralizes), "krovetz" (doesn't overgeneralize)
	# indri/lemur doc says this is optional, so I think we can just omit to not stem
	i.setStemmer("porter")

	# make sure that the metadata we just added is indexed and searchable
	# we will make sure "title" is indexed for both forward and backward searches
	# is this the place to make sure we'll be able to access doc IDs?
	i.setMetadataIndexedFields(["title"], ["title"])

	# make index - the InfoRetriever can access this by knowing the path
	i.create("./index")

	# add files
	for document in all_documents:
		print("Adding file: "+document) # for testing purposes
		i.addFile(document, "trectext") # untested, I think this is the correct format

	i.close()


# This method takes an XML file of questions as input, creates relevant Question objects,
# and returns them in a list.

def generate_q_list(xml_file):
	# parse the xml file (using Beautiful Soup?)
	# create relevant Question objects from the parse tree
	# return the Question objects in a sequential list

	soup = BeautifulSoup(xml_file)

	soup_targets = soup.find_all('target')
	questions = []
	
	for soup_target in soup_targets[:1]:
		target = soup_target.get('text')
		soup_questions = soup_target.find_all('q')

		for soup_question in soup_questions:
			id = soup_question.get('id')
			type = soup_question.get('type')
			q = soup_question.get_text().strip()
			question = Question(id, type, q, target)
			questions.append(question)
	return questions


if __name__ == '__main__':
	main()
{'fraction_non_alphanumeric': 0.07172873496594696, 'fraction_numerical': 0.006230981017243877, 'mean_word_length': 4.49085123309467, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7455302', 'n_tokens_mistral': 2146, 'n_tokens_neox': 1958, 'n_words': 828}