<commit_before>package org.vitrivr.cineast.core.features.abstracts;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.config.Config;
import org.vitrivr.cineast.core.config.QueryConfig;
import org.vitrivr.cineast.core.data.SegmentContainer;
import org.vitrivr.cineast.core.data.StringDoublePair;
import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
import org.vitrivr.cineast.core.db.ADAMproSelector;
import org.vitrivr.cineast.core.db.DBSelector;
import org.vitrivr.cineast.core.db.DBSelectorSupplier;
import org.vitrivr.cineast.core.decode.subtitle.SubtitleItem;
import org.vitrivr.cineast.core.features.retriever.Retriever;
import org.vitrivr.cineast.core.setup.EntityCreator;
import org.vitrivr.cineast.core.util.MathHelper;

/**
 * This is a proof of concept class and will probably be replaced by a more general solution to text
 * retrieval in the future
 * 
 *
 */
public abstract class SolrTextRetriever implements Retriever {

  private ADAMproSelector selector = null; // this is necessary since there is no abstraction for
                                           // the way external providers are handled in ADAMpro
                                           // (yet)

  private static final Logger LOGGER = LogManager.getLogger();

  @Override
  public void initalizePersistentLayer(Supplier<EntityCreator> supply) {
  }

  protected abstract String getEntityName();
  protected abstract float getMaxDist();

  @Override
  public void init(DBSelectorSupplier selectorSupply) {
    DBSelector s = selectorSupply.get();
    if (s instanceof ADAMproSelector) {
      this.selector = (ADAMproSelector) s;
      this.selector.open(getEntityName());
    } else {
      LOGGER.warn(
          "SolrTextRetriever only works with ADAMproSelectors, {} is currently not supported",
          s.getClass().getSimpleName());
    }

  }

  @Override
  public List<StringDoublePair> getSimilar(SegmentContainer sc, QueryConfig qc) {
    if (this.selector == null) {
      return new ArrayList<>(0);
    }

    HashMap<String, String> parameters = new HashMap<>();
    parameters.put("rows", Integer.toString(Config.getRetrieverConfig().getMaxResultsPerModule()));

    List<SubtitleItem> subItems = sc.getSubtitleItems();
    
    if(subItems.isEmpty()){
      return new ArrayList<>(0);
    }
    
    StringBuilder sb = new StringBuilder();
    sb.append('(');
    for (SubtitleItem subItem : subItems) {
      sb.append(subItem.getText());
      sb.append(' ');
    }
    sb.append(')');
    String query = sb.toString();
   

    parameters.put("query", "feature:" + query);

    List<Map<String, PrimitiveTypeProvider>> resultList = this.selector.getFromExternal("solr",
        parameters);
    
    ArrayList<StringDoublePair> pairs = new ArrayList<>(resultList.size());
    
    for(Map<String, PrimitiveTypeProvider> result : resultList){
      String id = result.get("id").getString();
      float dist = result.get("ap_score").getFloat();
      
      pairs.add(new StringDoublePair(id, MathHelper.getScore(dist, getMaxDist())));
    }
    
    return pairs;
  }

  @Override
  public List<StringDoublePair> getSimilar(String shotId, QueryConfig qc) {
    return new ArrayList<>(0); // currently not supported
  }

  @Override
  public void finish() {
    if (this.selector != null) {
      this.selector.close();<commit_msg>changed scoring function in SolrTextRetriever<commit_after>package org.vitrivr.cineast.core.features.abstracts;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.config.Config;
import org.vitrivr.cineast.core.config.QueryConfig;
import org.vitrivr.cineast.core.data.SegmentContainer;
import org.vitrivr.cineast.core.data.StringDoublePair;
import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider;
import org.vitrivr.cineast.core.db.ADAMproSelector;
import org.vitrivr.cineast.core.db.DBSelector;
import org.vitrivr.cineast.core.db.DBSelectorSupplier;
import org.vitrivr.cineast.core.decode.subtitle.SubtitleItem;
import org.vitrivr.cineast.core.features.retriever.Retriever;
import org.vitrivr.cineast.core.setup.EntityCreator;
import org.vitrivr.cineast.core.util.MathHelper;

/**
 * This is a proof of concept class and will probably be replaced by a more general solution to text
 * retrieval in the future
 * 
 *
 */
public abstract class SolrTextRetriever implements Retriever {

  private ADAMproSelector selector = null; // this is necessary since there is no abstraction for
                                           // the way external providers are handled in ADAMpro
                                           // (yet)

  private static final Logger LOGGER = LogManager.getLogger();

  @Override
  public void initalizePersistentLayer(Supplier<EntityCreator> supply) {
  }

  protected abstract String getEntityName();
  protected abstract float getMaxDist();

  @Override
  public void init(DBSelectorSupplier selectorSupply) {
    DBSelector s = selectorSupply.get();
    if (s instanceof ADAMproSelector) {
      this.selector = (ADAMproSelector) s;
      this.selector.open(getEntityName());
    } else {
      LOGGER.warn(
          "SolrTextRetriever only works with ADAMproSelectors, {} is currently not supported",
          s.getClass().getSimpleName());
    }

  }

  @Override
  public List<StringDoublePair> getSimilar(SegmentContainer sc, QueryConfig qc) {
    if (this.selector == null) {
      return new ArrayList<>(0);
    }

    HashMap<String, String> parameters = new HashMap<>();
    parameters.put("rows", Integer.toString(Config.getRetrieverConfig().getMaxResultsPerModule()));

    List<SubtitleItem> subItems = sc.getSubtitleItems();
    
    if(subItems.isEmpty()){
      return new ArrayList<>(0);
    }
    
    StringBuilder sb = new StringBuilder();
    sb.append('(');
    for (SubtitleItem subItem : subItems) {
      sb.append(subItem.getText());
      sb.append(' ');
    }
    sb.append(')');
    String query = sb.toString();
   

    parameters.put("query", "feature:" + query);

    List<Map<String, PrimitiveTypeProvider>> resultList = this.selector.getFromExternal("solr",
        parameters);
    
    ArrayList<StringDoublePair> pairs = new ArrayList<>(resultList.size());
    
    int words = query.split("\\s+").length;
    
    for(Map<String, PrimitiveTypeProvider> result : resultList){
      String id = result.get("id").getString();
      float score = result.get("ap_score").getFloat();
      
      pairs.add(new StringDoublePair(id, MathHelper.limit(score / words / 10f, 0f, 1f)));
    }
    
    return pairs;
  }

  @Override
  public List<StringDoublePair> getSimilar(String shotId, QueryConfig qc) {
    return new ArrayList<>(0); // currently not supported
  }

  @Override
  public void finish() {
    if (this.selector != null) {
      this.selector.close();
{'fraction_non_alphanumeric': 0.11412742382271468, 'fraction_numerical': 0.0019390581717451524, 'mean_word_length': 3.5216030056355665, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '680183', 'n_tokens_mistral': 2198, 'n_tokens_neox': 2123, 'n_words': 467}