<commit_before>package com.senseidb.search.query;

import it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet;
import it.unimi.dsi.fastutil.floats.FloatOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import javassist.CannotCompileException;
import javassist.ClassPool;
import javassist.CtClass;
import javassist.CtMethod;
import javassist.CtNewMethod;
import javassist.NotFoundException;

import org.apache.log4j.Logger;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import com.browseengine.bobo.api.BoboIndexReader;
import com.browseengine.bobo.facets.data.FacetDataCache;
import com.browseengine.bobo.facets.data.TermDoubleList;
import com.browseengine.bobo.facets.data.TermFloatList;
import com.browseengine.bobo.facets.data.TermIntList;
import com.browseengine.bobo.facets.data.TermLongList;
import com.browseengine.bobo.facets.data.TermShortList;
import com.browseengine.bobo.facets.data.TermStringList;
import com.browseengine.bobo.facets.data.TermValueList;
import com.browseengine.bobo.util.BigSegmentedArray;

public class RelevanceQuery extends AbstractScoreAdjuster
{
  
  /* JSON keywords*/
  
  // (1) json keys;
  public static final String           KW_VALUES               = "values";
  public static final String           KW_VARIABLES            = "variables";
  public static final String           KW_FACETS               = "facets";
  public static final String           KW_FUNC_PARAMETERS      = "function_params";
  public static final String           KW_FUNCTION             = "function";

  // (2) supported types in json:
  // set type: [set_int, set_float, set_string, set_double, set_long]
  public static final String           KW_TYPE_SET_INT         = "set_int";
  public static final String           KW_TYPE_SET_FLOAT       = "set_float";
  public static final String           KW_TYPE_SET_STRING      = "set_string";
  public static final String           KW_TYPE_SET_DOUBLE      = "set_double";
  public static final String           KW_TYPE_SET_LONG        = "set_long";

  // normal type: [int, double, float, long, bool, string]
  public static final String           KW_TYPE_INT             = "int";
  public static final String           KW_TYPE_FLOAT           = "float";
  public static final String           KW_TYPE_STRING          = "string";
  public static final String           KW_TYPE_DOUBLE          = "double";
  public static final String           KW_TYPE_LONG            = "long";
  public static final String           KW_TYPE_BOOL            = "bool";

  // facet type support: [double, float, int, long, short, string]
  public static final String           KW_TYPE_FACET_INT       = "int";
  public static final String           KW_TYPE_FACET_FLOAT     = "float";
  public static final String           KW_TYPE_FACET_STRING    = "string";
  public static final String           KW_TYPE_FACET_DOUBLE    = "double";
  public static final String           KW_TYPE_FACET_LONG      = "long";
  public static final String           KW_TYPE_FACET_SHORT     = "short";

  // constant type:
  public static final String           KW_INNER_SCORE          = "_INNER_SCORE";
  public static final String           KW_NOW                  = "_NOW";
  
  
  /* Type Strings; */
  
  // (1) inner score type;
  private final String                 TYPE_INNER_SCORE  = "INNER_SCORE";  //actually a float value;
  
  // (2) general types:
  private final String                 TYPE_INT          = "INT";
  private final String                 TYPE_LONG         = "LONG";
  private final String                 TYPE_DOUBLE       = "DOUBLE";
  private final String                 TYPE_FLOAT        = "FLOAT";
  private final String                 TYPE_BOOLEAN      = "BOOLEAN";
  private final String                 TYPE_STRING       = "STRING";

  // hashset container types:
  private final String                 TYPE_SET_INT          = "SET_INT";
  private final String                 TYPE_SET_LONG         = "SET_LONG";
  private final String                 TYPE_SET_DOUBLE       = "SET_DOUBLE";
  private final String                 TYPE_SET_FLOAT        = "SET_FLOAT";
  private final String                 TYPE_SET_STRING       = "SET_STRING";
  
  private final String                 TYPE_SET_HEAD         = "SET";

  // (3) facet types:
  private final String                 TYPE_FACET_INT    = "FACET_INT";
  private final String                 TYPE_FACET_LONG   = "FACET_LONG";
  private final String                 TYPE_FACET_DOUBLE = "FACET_DOUBLE";
  private final String                 TYPE_FACET_FLOAT  = "FACET_FLOAT";
  private final String                 TYPE_FACET_SHORT  = "FACET_SHORT";
  private final String                 TYPE_FACET_STRING = "FACET_STRING";
  
  private final String                 TYPE_FACET_HEAD   = "FACET";  
  
  
  /* Type Numbers */
  
  // (1) inner score type number;
  private final int                 TYPENUMBER_INNER_SCORE  = 0;
  
  // (2) general type numbers:
  private final int                 TYPENUMBER_INT          = 1;
  private final int                 TYPENUMBER_LONG         = 2;
  private final int                 TYPENUMBER_DOUBLE       = 3;
  private final int                 TYPENUMBER_FLOAT        = 4;
  private final int                 TYPENUMBER_BOOLEAN      = 5;
  private final int                 TYPENUMBER_STRING       = 6;  
  
  private final int                 TYPENUMBER_SET          = 7;
  
  // (3) facet type numbers;
  private final int                 TYPENUMBER_FACET_INT    = 10;
  private final int                 TYPENUMBER_FACET_LONG   = 11;
  private final int                 TYPENUMBER_FACET_DOUBLE = 12;
  private final int                 TYPENUMBER_FACET_FLOAT  = 13;
  private final int                 TYPENUMBER_FACET_SHORT  = 14;
  private final int                 TYPENUMBER_FACET_STRING = 15;
  
  
  
  private static final long serialVersionUID = 1L;
  
  private static Logger logger = Logger.getLogger(RelevanceQuery.class);
  
  protected final Query _query;
  
  private HashMap<String, Object> hm_var = new HashMap<String, Object>();
  private HashMap<String, String> hm_type = new HashMap<String, String>();
  private HashMap<String, String> hm_symbol_facet = new HashMap<String, String>();
  private HashMap<String, Integer> hm_facet_index = new HashMap<String, Integer>();
  private LinkedList<String> lls_params = new LinkedList<String>();
  private String funcBody = null;
  private String classIDString = null;
  private CustomScorer cscorer = null;
  private int facetIndex = 0;
  
  
  static ClassPool pool = ClassPool.getDefault();
  static
  {
    pool.importPackage("java.util");

    pool.importPackage("it.unimi.dsi.fastutil.ints.IntOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.longs.LongOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.shorts.ShortOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.booleans.BooleanOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.floats.FloatOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.objects.ObjectOpenHashSet");
  }
  


  //white list of safe classes;
  static HashSet<String>    hs_safe = new HashSet<String>();      
  static
  {
    hs_safe.add("it.unimi.dsi.fastutil.ints.IntOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.longs.LongOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.shorts.ShortOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.booleans.BooleanOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.floats.FloatOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.objects.ObjectOpenHashSet");
    
    hs_safe.add("com.senseidb.search.query.RelevanceQuery");
    hs_safe.add("com.senseidb.search.query.CustomScorer");
    hs_safe.add("com.senseidb.search.query.RelevanceQuery$CustomLoader");
    
    hs_safe.add("java.lang.Object");
  }
  
  
  static HashMap<String, CustomScorer> hmModels = new HashMap<String, CustomScorer>();
  
//  "relevance":{
//               
//                  "variables": {
//                                 "set_int":["c","d"],  // supported hashset types: [set_int, set_float, set_string, set_double, set_long]
//                                 "int":["e","f"],       // supported normal variables: [int, double, float, long, bool, string]
//                                 "long":["g","h"]
//                                },
//                  "facets":{
//                               "int":["year","age"],   // facet type support: [double, float, int, long, short, string];
//                               "long":["time"]         // facet variable has the same name as the facet name, and they are defined inside this json;
//                            },
//                  
//                  "values":{
//                              "c":[1996,1997],
//                              "e":0.98
//                            },
//         
//                   // (2) scoring function and function input parameters in Java;
//                   //     A scoring function and its parameters are the model. A model changes when the function body or signature changes;
//                   
//                  //  params for the function. Symbol order matters, and symbols must be those defined above. innerScore MUST be used, otherwise, makes no sense to use the custom relevance;
//                  //  reserved keyword for internal parameters are:  "_INNER_SCORE" and "_NOW"     
//
//                   "function_params":["_INNER_SCORE", "timeVal", "_timeWeight", "_waterworldWeight", "_half_time"],               
//<commit_msg>minor update add more whitelist class<commit_after>package com.senseidb.search.query;

import it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet;
import it.unimi.dsi.fastutil.floats.FloatOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import javassist.CannotCompileException;
import javassist.ClassPool;
import javassist.CtClass;
import javassist.CtMethod;
import javassist.CtNewMethod;
import javassist.NotFoundException;

import org.apache.log4j.Logger;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import com.browseengine.bobo.api.BoboIndexReader;
import com.browseengine.bobo.facets.data.FacetDataCache;
import com.browseengine.bobo.facets.data.TermDoubleList;
import com.browseengine.bobo.facets.data.TermFloatList;
import com.browseengine.bobo.facets.data.TermIntList;
import com.browseengine.bobo.facets.data.TermLongList;
import com.browseengine.bobo.facets.data.TermShortList;
import com.browseengine.bobo.facets.data.TermStringList;
import com.browseengine.bobo.facets.data.TermValueList;
import com.browseengine.bobo.util.BigSegmentedArray;

public class RelevanceQuery extends AbstractScoreAdjuster
{
  
  /* JSON keywords*/
  
  // (1) json keys;
  public static final String           KW_VALUES               = "values";
  public static final String           KW_VARIABLES            = "variables";
  public static final String           KW_FACETS               = "facets";
  public static final String           KW_FUNC_PARAMETERS      = "function_params";
  public static final String           KW_FUNCTION             = "function";

  // (2) supported types in json:
  // set type: [set_int, set_float, set_string, set_double, set_long]
  public static final String           KW_TYPE_SET_INT         = "set_int";
  public static final String           KW_TYPE_SET_FLOAT       = "set_float";
  public static final String           KW_TYPE_SET_STRING      = "set_string";
  public static final String           KW_TYPE_SET_DOUBLE      = "set_double";
  public static final String           KW_TYPE_SET_LONG        = "set_long";

  // normal type: [int, double, float, long, bool, string]
  public static final String           KW_TYPE_INT             = "int";
  public static final String           KW_TYPE_FLOAT           = "float";
  public static final String           KW_TYPE_STRING          = "string";
  public static final String           KW_TYPE_DOUBLE          = "double";
  public static final String           KW_TYPE_LONG            = "long";
  public static final String           KW_TYPE_BOOL            = "bool";

  // facet type support: [double, float, int, long, short, string]
  public static final String           KW_TYPE_FACET_INT       = "int";
  public static final String           KW_TYPE_FACET_FLOAT     = "float";
  public static final String           KW_TYPE_FACET_STRING    = "string";
  public static final String           KW_TYPE_FACET_DOUBLE    = "double";
  public static final String           KW_TYPE_FACET_LONG      = "long";
  public static final String           KW_TYPE_FACET_SHORT     = "short";

  // constant type:
  public static final String           KW_INNER_SCORE          = "_INNER_SCORE";
  public static final String           KW_NOW                  = "_NOW";
  
  
  /* Type Strings; */
  
  // (1) inner score type;
  private final String                 TYPE_INNER_SCORE  = "INNER_SCORE";  //actually a float value;
  
  // (2) general types:
  private final String                 TYPE_INT          = "INT";
  private final String                 TYPE_LONG         = "LONG";
  private final String                 TYPE_DOUBLE       = "DOUBLE";
  private final String                 TYPE_FLOAT        = "FLOAT";
  private final String                 TYPE_BOOLEAN      = "BOOLEAN";
  private final String                 TYPE_STRING       = "STRING";

  // hashset container types:
  private final String                 TYPE_SET_INT          = "SET_INT";
  private final String                 TYPE_SET_LONG         = "SET_LONG";
  private final String                 TYPE_SET_DOUBLE       = "SET_DOUBLE";
  private final String                 TYPE_SET_FLOAT        = "SET_FLOAT";
  private final String                 TYPE_SET_STRING       = "SET_STRING";
  
  private final String                 TYPE_SET_HEAD         = "SET";

  // (3) facet types:
  private final String                 TYPE_FACET_INT    = "FACET_INT";
  private final String                 TYPE_FACET_LONG   = "FACET_LONG";
  private final String                 TYPE_FACET_DOUBLE = "FACET_DOUBLE";
  private final String                 TYPE_FACET_FLOAT  = "FACET_FLOAT";
  private final String                 TYPE_FACET_SHORT  = "FACET_SHORT";
  private final String                 TYPE_FACET_STRING = "FACET_STRING";
  
  private final String                 TYPE_FACET_HEAD   = "FACET";  
  
  
  /* Type Numbers */
  
  // (1) inner score type number;
  private final int                 TYPENUMBER_INNER_SCORE  = 0;
  
  // (2) general type numbers:
  private final int                 TYPENUMBER_INT          = 1;
  private final int                 TYPENUMBER_LONG         = 2;
  private final int                 TYPENUMBER_DOUBLE       = 3;
  private final int                 TYPENUMBER_FLOAT        = 4;
  private final int                 TYPENUMBER_BOOLEAN      = 5;
  private final int                 TYPENUMBER_STRING       = 6;  
  
  private final int                 TYPENUMBER_SET          = 7;
  
  // (3) facet type numbers;
  private final int                 TYPENUMBER_FACET_INT    = 10;
  private final int                 TYPENUMBER_FACET_LONG   = 11;
  private final int                 TYPENUMBER_FACET_DOUBLE = 12;
  private final int                 TYPENUMBER_FACET_FLOAT  = 13;
  private final int                 TYPENUMBER_FACET_SHORT  = 14;
  private final int                 TYPENUMBER_FACET_STRING = 15;
  
  
  
  private static final long serialVersionUID = 1L;
  
  private static Logger logger = Logger.getLogger(RelevanceQuery.class);
  
  protected final Query _query;
  
  private HashMap<String, Object> hm_var = new HashMap<String, Object>();
  private HashMap<String, String> hm_type = new HashMap<String, String>();
  private HashMap<String, String> hm_symbol_facet = new HashMap<String, String>();
  private HashMap<String, Integer> hm_facet_index = new HashMap<String, Integer>();
  private LinkedList<String> lls_params = new LinkedList<String>();
  private String funcBody = null;
  private String classIDString = null;
  private CustomScorer cscorer = null;
  private int facetIndex = 0;
  
  
  static ClassPool pool = ClassPool.getDefault();
  static
  {
    pool.importPackage("java.util");

    pool.importPackage("it.unimi.dsi.fastutil.ints.IntOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.longs.LongOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.shorts.ShortOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.booleans.BooleanOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.floats.FloatOpenHashSet");
    pool.importPackage("it.unimi.dsi.fastutil.objects.ObjectOpenHashSet");
  }
  


  //white list of safe classes;
  static HashSet<String>    hs_safe = new HashSet<String>();      
  static
  {
    hs_safe.add("it.unimi.dsi.fastutil.ints.IntOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.longs.LongOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.shorts.ShortOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.booleans.BooleanOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.floats.FloatOpenHashSet");
    hs_safe.add("it.unimi.dsi.fastutil.objects.ObjectOpenHashSet");
    
    hs_safe.add("com.senseidb.search.query.RelevanceQuery");
    hs_safe.add("com.senseidb.search.query.CustomScorer");
    hs_safe.add("com.senseidb.search.query.RelevanceQuery$CustomLoader");
    
    hs_safe.add("java.lang.Object");
    hs_safe.add("java.lang.Exception");
    hs_safe.add("java.lang.Boolean");
    hs_safe.add("java.lang.Byte");
    hs_safe.add("java.lang.Double");
    hs_safe.add("java.lang.Float");
    hs_safe.add("java.lang.Integer");
    hs_safe.add("java.lang.Long");
    hs_safe.add("java.lang.Math");
    hs_safe.add("java.lang.Number");
    hs_safe.add("java.lang.Short");
    hs_safe.add("java.lang.String");
    hs_safe.add("java.lang.StringBuffer");
    hs_safe.add("java.lang.StringBuilder");
    hs_safe.add("java.math.BigDecimal");
    hs_safe.add("java.math.BigInteger");
    hs_safe.add("java.math.MathContext");
    hs_safe.add("java.math.RoundingMode");
    
  }
  
  
  static HashMap<String, CustomScorer> hmModels = new HashMap<String, CustomScorer>();
  
//  "relevance":{
//               
//                  "variables": {
//                                 "set_int":["c","d"],  // supported hashset types: [set_int, set_float, set_string, set_double, set_long]
//                                 "int":["e","f"],       // supported normal variables: [int, double, float, long, bool, string]
//                                 "long":["g","h"]
//                                },
//                  "facets":{
//                               "int":["year","age"],   // facet type support: [double, float, int, long, short, string];
//                               "long":["time"]         // facet variable has the same name as the facet name, and they are defined inside this json;
//                            },
//                  
//                  "values":{
//                              "c":[1996,1997],
//                              "e":0.98
//                            },
//         
//                   // (2) scoring function and function input parameters in Java;
//                   //     A scoring function and its parameters are the model. A model changes when the function body or signature changes;
//                   
//                  //  params for the function. Symbol order matters, and symbols must be those defined above. innerScore MUST be used, otherwise, makes no sense to use the custom relevance;
//                  //  reserved keyword for internal parameters are:  "_INNER_SCORE" and "_NOW"     
//
//                   "function_params":["_INNER_SCORE", "timeVal", "_timeWeight", "_waterworldWeight", "_half_time"],               
//
{'fraction_non_alphanumeric': 0.09717588459188947, 'fraction_numerical': 0.004041163479159814, 'mean_word_length': 2.224056961066505, 'pattern_counts': {'":': 24, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6988951', 'n_tokens_mistral': 6517, 'n_tokens_neox': 6301, 'n_words': 1410}