<commit_before>package QueryEvaluation;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.Map.Entry;

import mitos.stemmer.Stemmer;

public class QueryResults {
	private static String query;
	private static ArrayList<String> results;
	private static HashMap<String,VocabularyEntry> voc;
	private static RetrievalModel model;
	private static HashMap<String,String> docsIDPathMap;					/* key: docID, value: path */
	
	private static HashMap<String,ArrayList<Long>> wordPos;					/* key: docID, value: positions of this word in it 	*/
	private static HashMap<String, HashMap<String, ArrayList<Long>>> pos;	/* key: word, value: map<DocID,list<pos>> */
	
	private static HashMap<String,Float> termDFs;						/* key: term string, value: its df 					*/
	private static HashMap<String,HashMap<String,Float>> termTFs;		/* key: term , value: map<docID,tf>					*/
	private static int sumDocLength;

	public QueryResults(HashMap<String,VocabularyEntry> voc, RetrievalModel model, String query) throws IOException {
		QueryResults.voc = voc;
		QueryResults.model = model;
		QueryResults.query = query;
		QueryResults.results = new ArrayList<String>();
		QueryResults.docsIDPathMap = new HashMap<String,String>();
		QueryResults.wordPos = new HashMap<String,ArrayList<Long>>();
		QueryResults.pos = new HashMap<String, HashMap<String, ArrayList<Long>>>();
        QueryResults.setTermDFs(new HashMap<String,Float>());
        QueryResults.setTermTFs(new HashMap<String, HashMap<String,Float> >());
        QueryResults.sumDocLength = 0;
	}

	public ArrayList<String> getResults() {
		return results;
	}

    private String[] tokenizeQuery(String query) {
    	Stemmer.Initialize();
        StringTokenizer tokenized = new StringTokenizer(query.toLowerCase());
        int numTokens = tokenized.countTokens();
        String[] tokens = new String[numTokens];
        int i = 0;
        while (tokenized.hasMoreTokens()) {
            String token = tokenized.nextToken();
            if(Vocabulary.getStopWords().get(token) ==null)
            {
            	token = Stemmer.Stem(token);
            	tokens[i] = token;
            	i++;
            }
        }
        return tokens;
    }
    
	public ArrayList<String> createResults() throws IOException {
		
		String[] terms = tokenizeQuery(query.toLowerCase());
		
		// tokenized query terms
		for ( int i=0; i<terms.length; i++ ) {<commit_msg>query results minor<commit_after>package QueryEvaluation;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.Map.Entry;

import mitos.stemmer.Stemmer;

public class QueryResults {
	private static String query;
	private static ArrayList<String> results;
	private static HashMap<String,VocabularyEntry> voc;
	private static RetrievalModel model;
	private static HashMap<String,String> docsIDPathMap;					/* key: docID, value: path */
	
	private static HashMap<String,ArrayList<Long>> wordPos;					/* key: docID, value: positions of this word in it 	*/
	private static HashMap<String, HashMap<String, ArrayList<Long>>> pos;	/* key: word, value: map<DocID,list<pos>> */
	
	private static HashMap<String,Float> termDFs;						/* key: term string, value: its df 					*/
	private static HashMap<String,HashMap<String,Float>> termTFs;		/* key: term , value: map<docID,tf>					*/
	private static int sumDocLength;

	public QueryResults(HashMap<String,VocabularyEntry> voc, RetrievalModel model, String query) throws IOException {
		QueryResults.voc = voc;
		QueryResults.model = model;
		QueryResults.query = query;
		QueryResults.results = new ArrayList<String>();
		QueryResults.docsIDPathMap = new HashMap<String,String>();
		QueryResults.wordPos = new HashMap<String,ArrayList<Long>>();
		QueryResults.pos = new HashMap<String, HashMap<String, ArrayList<Long>>>();
        QueryResults.setTermDFs(new HashMap<String,Float>());
        QueryResults.setTermTFs(new HashMap<String, HashMap<String,Float> >());
        QueryResults.sumDocLength = 0;
	}

	public ArrayList<String> getResults() {
		return results;
	}

    private String[] tokenizeQuery(String query) {
    	Stemmer.Initialize();
        StringTokenizer tokenized = new StringTokenizer(query.toLowerCase());
        int numTokens = tokenized.countTokens();
        String[] tokens = new String[numTokens];
        int i = 0;
        while (tokenized.hasMoreTokens()) {
            String token = tokenized.nextToken();
            token = Stemmer.Stem(token);
            tokens[i] = token;
            i++;
        }
        return tokens;
    }
    
	public ArrayList<String> createResults() throws IOException {
		
		String[] terms = tokenizeQuery(query);
		
		// tokenized query terms
		for ( int i=0; i<terms.length; i++ ) {
{'fraction_non_alphanumeric': 0.11560579020424351, 'fraction_numerical': 0.001189767995240928, 'mean_word_length': 3.845341018251681, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2401014', 'n_tokens_mistral': 1542, 'n_tokens_neox': 1470, 'n_words': 368}