<commit_before>package lucene4ir;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;

import javax.xml.bind.JAXB;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;

import java.io.File;
import java.io.IOException;
import java.util.*;

import static lucene4ir.QETerm.QETermComparator;

/**
 * Created by colin on 17/08/2017.
 * Implements query expansion using Rocchio's expansion by taking the initial query and results list
 * then expanding the query with the top t terms from the top d documents.
 */

//TODO: Stop the inclusion of original query terms in expansion terms.

public class QERetrievalApp extends RetrievalApp{

    protected QERetrievalParams qep;
    protected float qeBeta;
    protected int feedbackDocs;
    protected int feedbackTerms;
    protected long doc_count;

    private HashMap<Integer, String> id2num = new HashMap<Integer, String>();
    private HashMap<String, Integer> num2id = new HashMap<String, Integer>();

    /**
     * Instantiates the QERetrievalApp, setting up RetrievalApp from the retrievalParamFile
     * @param retrievalParamFile
     */
    public QERetrievalApp(String retrievalParamFile){
        super(retrievalParamFile);
        System.out.println("Query Expansion");
        this.readQEParamsFromFile(qeFile);
        mapDocid();

        try {
            reader = DirectoryReader.open(FSDirectory.open( new File(p.indexName).toPath()) );
            searcher = new IndexSearcher(reader);

            // create similarity function and parameter
            selectSimilarityFunction(sim);
            searcher.setSimilarity(simfn);
//            parser = new QueryParser("content", analyzer);
            CollectionStatistics collectionStats = searcher.collectionStatistics(Lucene4IRConstants.FIELD_ALL);
            doc_count = collectionStats.docCount();
        } catch (Exception e){
            System.out.println(" caught a " + e.getClass() +
                    "\n with message: " + e.getMessage());
        }
    }

    /**
     * Performs the scoring of documents from the query issued
     * @param qno
     * @param queryTerms
     * @return
     */
    public ScoreDoc[] runQuery(String qno, String queryTerms){
        ScoreDoc[] hits = null;

        // Original Query
        System.out.println("Query No.: " + qno + " " + queryTerms);
        try {
            Query query = parser.parse(QueryParser.escape(queryTerms));
            try {
                TopDocs results = searcher.search(query, p.maxResults);
                hits = results.scoreDocs;

                Vector<Terms> terms = getDocsTerms(getDocs(queryTerms,hits));
                // will crash if there were no hits for a query.
                // What should happen if there are no hits? Exit and return regular retrieval?
                if (terms.size()>1) {
                    String[] split = Arrays.stream(query.toString().split("all:")).map(String::trim).toArray(String[]::new);

                    String oldQueryString="";
                    for(String s: split)
                        oldQueryString+= s.trim() + " ";
                    List<String> rankedTerms = rankTerms(combineTerms(terms));

                    List<String> oldTerms = Arrays.asList(split);
                    String qString = "";
                    int numterms = 0;
                    int i =0;
                    while(i <= terms.size() && numterms<=feedbackTerms){
                        String rankedTerm = rankedTerms.get(i).trim();
                        if (!oldTerms.contains(rankedTerm)) {
                            qString += rankedTerm + " ";
                            numterms ++;
                        }
                        i++;
                    }

                    Query oq = parser.parse(QueryParser.escape(oldQueryString));
                    Query nq = parser.parse(QueryParser.escape(qString.trim()));

                    BoostQuery obq = new BoostQuery(oq,(1-qeBeta));
                    BoostQuery nbq = new BoostQuery(nq,(qeBeta));
                    BooleanClause obc=new BooleanClause(obq,BooleanClause.Occur.SHOULD);
                    BooleanClause nbc=new BooleanClause(nbq,BooleanClause.Occur.SHOULD);
                    BooleanQuery.Builder bqb = new BooleanQuery.Builder();
                    bqb.add(obc);
                    bqb.add(nbc);
                    BooleanQuery q = bqb.build();
                    System.out.println("ExpandedQuery: " + q.toString());
                    results = searcher.search(q, p.maxResults);
                    hits = results.scoreDocs;
                }
            }
            catch (IOException ioe){
                System.out.println(" caught a " + ioe.getClass() +
                        "\n with message: " + ioe.getMessage());
            }
        } catch (ParseException pe){
            System.out.println("Can't parse query");
        }

        // BEFORE returning, this should re-run the query with the expanded terms.
        return hits;
    }


<commit_msg>Flipped Rocchio's beta calculation (higher value more weighting on original terms).<commit_after>package lucene4ir;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;

import javax.xml.bind.JAXB;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;

import java.io.File;
import java.io.IOException;
import java.util.*;

import static lucene4ir.QETerm.QETermComparator;

/**
 * Created by colin on 17/08/2017.
 * Implements query expansion using Rocchio's expansion by taking the initial query and results list
 * then expanding the query with the top t terms from the top d documents.
 */

//TODO: Stop the inclusion of original query terms in expansion terms.

public class QERetrievalApp extends RetrievalApp{

    protected QERetrievalParams qep;
    protected float qeBeta;
    protected int feedbackDocs;
    protected int feedbackTerms;
    protected long doc_count;

    private HashMap<Integer, String> id2num = new HashMap<Integer, String>();
    private HashMap<String, Integer> num2id = new HashMap<String, Integer>();

    /**
     * Instantiates the QERetrievalApp, setting up RetrievalApp from the retrievalParamFile
     * @param retrievalParamFile
     */
    public QERetrievalApp(String retrievalParamFile){
        super(retrievalParamFile);
        System.out.println("Query Expansion");
        this.readQEParamsFromFile(qeFile);
        mapDocid();

        try {
            reader = DirectoryReader.open(FSDirectory.open( new File(p.indexName).toPath()) );
            searcher = new IndexSearcher(reader);

            // create similarity function and parameter
            selectSimilarityFunction(sim);
            searcher.setSimilarity(simfn);
//            parser = new QueryParser("content", analyzer);
            CollectionStatistics collectionStats = searcher.collectionStatistics(Lucene4IRConstants.FIELD_ALL);
            doc_count = collectionStats.docCount();
        } catch (Exception e){
            System.out.println(" caught a " + e.getClass() +
                    "\n with message: " + e.getMessage());
        }
    }

    /**
     * Performs the scoring of documents from the query issued
     * @param qno
     * @param queryTerms
     * @return
     */
    public ScoreDoc[] runQuery(String qno, String queryTerms){
        ScoreDoc[] hits = null;

        // Original Query
        System.out.println("Query No.: " + qno + " " + queryTerms);
        try {
            Query query = parser.parse(QueryParser.escape(queryTerms));
            try {
                TopDocs results = searcher.search(query, p.maxResults);
                hits = results.scoreDocs;

                Vector<Terms> terms = getDocsTerms(getDocs(queryTerms,hits));
                // will crash if there were no hits for a query.
                // What should happen if there are no hits? Exit and return regular retrieval?
                if (terms.size()>1) {
                    String[] split = Arrays.stream(query.toString().split("all:")).map(String::trim).toArray(String[]::new);

                    String oldQueryString="";
                    for(String s: split)
                        oldQueryString+= s.trim() + " ";
                    List<String> rankedTerms = rankTerms(combineTerms(terms));

                    List<String> oldTerms = Arrays.asList(split);
                    String qString = "";
                    int numterms = 0;
                    int i =0;
                    while(i <= terms.size() && numterms<=feedbackTerms){
                        String rankedTerm = rankedTerms.get(i).trim();
                        if (!oldTerms.contains(rankedTerm)) {
                            qString += rankedTerm + " ";
                            numterms ++;
                        }
                        i++;
                    }

                    Query oq = parser.parse(QueryParser.escape(oldQueryString));
                    Query nq = parser.parse(QueryParser.escape(qString.trim()));

                    BoostQuery obq = new BoostQuery(oq,(qeBeta));
                    BoostQuery nbq = new BoostQuery(nq,(1-qeBeta));
                    BooleanClause obc=new BooleanClause(obq,BooleanClause.Occur.SHOULD);
                    BooleanClause nbc=new BooleanClause(nbq,BooleanClause.Occur.SHOULD);
                    BooleanQuery.Builder bqb = new BooleanQuery.Builder();
                    bqb.add(obc);
                    bqb.add(nbc);
                    BooleanQuery q = bqb.build();
                    System.out.println("ExpandedQuery: " + q.toString());
                    results = searcher.search(q, p.maxResults);
                    hits = results.scoreDocs;
                }
            }
            catch (IOException ioe){
                System.out.println(" caught a " + ioe.getClass() +
                        "\n with message: " + ioe.getMessage());
            }
        } catch (ParseException pe){
            System.out.println("Can't parse query");
        }

        // BEFORE returning, this should re-run the query with the expanded terms.
        return hits;
    }



{'fraction_non_alphanumeric': 0.10126353790613718, 'fraction_numerical': 0.0030685920577617327, 'mean_word_length': 2.040889132821076, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7239036', 'n_tokens_mistral': 3080, 'n_tokens_neox': 2921, 'n_words': 692}