<commit_before>/*
 *  BSD License (http://lemurproject.org/galago-license)
 */
package org.lemurproject.galago.core.retrieval.prf;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.lemurproject.galago.core.parse.Document;
import org.lemurproject.galago.core.parse.Document.DocumentComponents;
import org.lemurproject.galago.core.parse.TagTokenizer;
import org.lemurproject.galago.core.parse.stem.Porter2Stemmer;
import org.lemurproject.galago.core.parse.stem.Stemmer;
import org.lemurproject.galago.core.retrieval.GroupRetrieval;
import org.lemurproject.galago.core.retrieval.Retrieval;
import org.lemurproject.galago.core.retrieval.ScoredDocument;
import org.lemurproject.galago.core.retrieval.query.AnnotatedNode;
import org.lemurproject.galago.core.retrieval.query.Node;
import org.lemurproject.galago.core.retrieval.query.StructuredQuery;
import org.lemurproject.galago.core.util.MathUtils;
import org.lemurproject.galago.core.util.WordLists;
import org.lemurproject.galago.tupleflow.Parameters;

/**
 *
 * @author sjh
 */
public class RelevanceModel1 implements ExpansionModel {

  private static final Logger logger = Logger.getLogger("RM3");
  private final Retrieval retrieval;
  private int defaultFbDocs;
  private int defaultFbTerms;
  private Set<String> exclusionTerms;
  private Set<String> inclusionTerms;
  private Stemmer stemmer;
  private TagTokenizer tokenizer;

  public RelevanceModel1(Retrieval r) throws Exception {
    this.retrieval = r;
    defaultFbDocs = (int) Math.round(r.getGlobalParameters().get("fbDocs", 10.0));
    defaultFbTerms = (int) Math.round(r.getGlobalParameters().get("fbTerm", 5.0));
    exclusionTerms = WordLists.getWordList(r.getGlobalParameters().get("rmstopwords", "rmstop"));
    inclusionTerms = null;
    if (r.getGlobalParameters().isString("rmwhitelist")){
        inclusionTerms = WordLists.getWordList(r.getGlobalParameters().getString("rmwhitelist"));
    }
    tokenizer = new TagTokenizer();
    if (r.getGlobalParameters().isString("rmStemmer")) {
      String rmstemmer = r.getGlobalParameters().getString("rmStemmer");
      stemmer = (Stemmer) Class.forName(rmstemmer).getConstructor().newInstance();
    } else {
      stemmer = new Porter2Stemmer();
    }
  }

  @Override
  public Node expand(Node root, Parameters queryParameters) throws Exception {

    int fbDocs = (int) Math.round(root.getNodeParameters().get("fbDocs", queryParameters.get("fbDocs", (double) defaultFbDocs)));
    int fbTerms = (int) Math.round(root.getNodeParameters().get("fbTerm", queryParameters.get("fbTerm", (double) defaultFbTerms)));

    if (fbDocs <= 0 || fbTerms <= 0) {
      logger.info("fbDocs, or fbTerms is invalid, no expansion possible. (<= 0)");
      return root;
    }

    // transform query to ensure it will run
    Parameters fbParams = new Parameters();
    fbParams.set("requested", fbDocs);
    // first pass is asserted to be document level
    fbParams.set("passageQuery", false);
    fbParams.set("extentQuery", false);
    fbParams.setBackoff(queryParameters);

    Node transformed = retrieval.transformQuery(root.clone(), fbParams);

    // get some initial results
    List<ScoredDocument> initialResults = collectInitialResults(transformed, fbParams);

    // extract grams from results
    Set<String> stemmedQueryTerms = stemTerms(StructuredQuery.findQueryTerms(transformed));
    Set<String> exclusions = (fbParams.isString("rmstopwords")) ? WordLists.getWordList(fbParams.getString("rmstopwords")) : exclusionTerms;
    Set<String> inclusions = null;
    if (fbParams.isString("rmwhitelist")){
        inclusions = WordLists.getWordList(fbParams.getString("rmwhitelist"));
    } else {
        inclusions = inclusionTerms;
    }
    System.err.println((inclusionTerms == null) + " is null");
    List<WeightedTerm> weightedTerms = extractGrams(initialResults, fbParams, stemmedQueryTerms, exclusions, inclusions);

    // select some terms to form exp query node
    Node expNode = generateExpansionQuery(weightedTerms, fbTerms);

    return expNode;
  }

  public List<ScoredDocument> collectInitialResults(Node transformed, Parameters fbParams) throws Exception {
    ScoredDocument[] res = retrieval.runQuery(transformed, fbParams);
    return Arrays.asList(res);
  }

  public List<WeightedTerm> extractGrams(List<ScoredDocument> initialResults, Parameters fbParams, Set<String> queryTerms, Set<String> exclusionTerms, Set<String> inclusionTerms) throws IOException {
    // convert documentScores to posterior probs
    Map<ScoredDocument, Double> scores = logstoposteriors(initialResults);

    // get term frequencies in documents
    Map<String, Map<ScoredDocument, Integer>> counts = countGrams(initialResults, fbParams, queryTerms, exclusionTerms, inclusionTerms);

    // compute term weights
    List<WeightedTerm> scored = scoreGrams(counts, scores);

    // sort by weight<commit_msg>Remove extraneous println.<commit_after>/*
 *  BSD License (http://lemurproject.org/galago-license)
 */
package org.lemurproject.galago.core.retrieval.prf;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.lemurproject.galago.core.parse.Document;
import org.lemurproject.galago.core.parse.Document.DocumentComponents;
import org.lemurproject.galago.core.parse.TagTokenizer;
import org.lemurproject.galago.core.parse.stem.Porter2Stemmer;
import org.lemurproject.galago.core.parse.stem.Stemmer;
import org.lemurproject.galago.core.retrieval.GroupRetrieval;
import org.lemurproject.galago.core.retrieval.Retrieval;
import org.lemurproject.galago.core.retrieval.ScoredDocument;
import org.lemurproject.galago.core.retrieval.query.AnnotatedNode;
import org.lemurproject.galago.core.retrieval.query.Node;
import org.lemurproject.galago.core.retrieval.query.StructuredQuery;
import org.lemurproject.galago.core.util.MathUtils;
import org.lemurproject.galago.core.util.WordLists;
import org.lemurproject.galago.tupleflow.Parameters;

/**
 *
 * @author sjh
 */
public class RelevanceModel1 implements ExpansionModel {

  private static final Logger logger = Logger.getLogger("RM3");
  private final Retrieval retrieval;
  private int defaultFbDocs;
  private int defaultFbTerms;
  private Set<String> exclusionTerms;
  private Set<String> inclusionTerms;
  private Stemmer stemmer;
  private TagTokenizer tokenizer;

  public RelevanceModel1(Retrieval r) throws Exception {
    this.retrieval = r;
    defaultFbDocs = (int) Math.round(r.getGlobalParameters().get("fbDocs", 10.0));
    defaultFbTerms = (int) Math.round(r.getGlobalParameters().get("fbTerm", 5.0));
    exclusionTerms = WordLists.getWordList(r.getGlobalParameters().get("rmstopwords", "rmstop"));
    inclusionTerms = null;
    if (r.getGlobalParameters().isString("rmwhitelist")){
        inclusionTerms = WordLists.getWordList(r.getGlobalParameters().getString("rmwhitelist"));
    }
    tokenizer = new TagTokenizer();
    if (r.getGlobalParameters().isString("rmStemmer")) {
      String rmstemmer = r.getGlobalParameters().getString("rmStemmer");
      stemmer = (Stemmer) Class.forName(rmstemmer).getConstructor().newInstance();
    } else {
      stemmer = new Porter2Stemmer();
    }
  }

  @Override
  public Node expand(Node root, Parameters queryParameters) throws Exception {

    int fbDocs = (int) Math.round(root.getNodeParameters().get("fbDocs", queryParameters.get("fbDocs", (double) defaultFbDocs)));
    int fbTerms = (int) Math.round(root.getNodeParameters().get("fbTerm", queryParameters.get("fbTerm", (double) defaultFbTerms)));

    if (fbDocs <= 0 || fbTerms <= 0) {
      logger.info("fbDocs, or fbTerms is invalid, no expansion possible. (<= 0)");
      return root;
    }

    // transform query to ensure it will run
    Parameters fbParams = new Parameters();
    fbParams.set("requested", fbDocs);
    // first pass is asserted to be document level
    fbParams.set("passageQuery", false);
    fbParams.set("extentQuery", false);
    fbParams.setBackoff(queryParameters);

    Node transformed = retrieval.transformQuery(root.clone(), fbParams);

    // get some initial results
    List<ScoredDocument> initialResults = collectInitialResults(transformed, fbParams);

    // extract grams from results
    Set<String> stemmedQueryTerms = stemTerms(StructuredQuery.findQueryTerms(transformed));
    Set<String> exclusions = (fbParams.isString("rmstopwords")) ? WordLists.getWordList(fbParams.getString("rmstopwords")) : exclusionTerms;
    Set<String> inclusions = null;
    if (fbParams.isString("rmwhitelist")){
        inclusions = WordLists.getWordList(fbParams.getString("rmwhitelist"));
    } else {
        inclusions = inclusionTerms;
    }
    
    List<WeightedTerm> weightedTerms = extractGrams(initialResults, fbParams, stemmedQueryTerms, exclusions, inclusions);

    // select some terms to form exp query node
    Node expNode = generateExpansionQuery(weightedTerms, fbTerms);

    return expNode;
  }

  public List<ScoredDocument> collectInitialResults(Node transformed, Parameters fbParams) throws Exception {
    ScoredDocument[] res = retrieval.runQuery(transformed, fbParams);
    return Arrays.asList(res);
  }

  public List<WeightedTerm> extractGrams(List<ScoredDocument> initialResults, Parameters fbParams, Set<String> queryTerms, Set<String> exclusionTerms, Set<String> inclusionTerms) throws IOException {
    // convert documentScores to posterior probs
    Map<ScoredDocument, Double> scores = logstoposteriors(initialResults);

    // get term frequencies in documents
    Map<String, Map<ScoredDocument, Integer>> counts = countGrams(initialResults, fbParams, queryTerms, exclusionTerms, inclusionTerms);

    // compute term weights
    List<WeightedTerm> scored = scoreGrams(counts, scores);

    // sort by weight
{'fraction_non_alphanumeric': 0.11471882640586797, 'fraction_numerical': 0.00254278728606357, 'mean_word_length': 4.959207459207459, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6747276', 'n_tokens_mistral': 3209, 'n_tokens_neox': 3074, 'n_words': 627}