<commit_before>package edu.umass.ciir.strepsirank.collector

import scala.collection.mutable.ListBuffer
import scala.collection.mutable
import edu.umass.ciir.strepsi.{StringTools, SeqTools}
import java.io.FileWriter

/**
 *
 * queryid -> {document -> (Option[Label], features)}
 * where features = Seq[(String,Double)]
 * User: dietz
 * Date: 12/9/13
 * Time: 6:00 PM
 */

class StrepsiKeyTypedFeatureCollector[Key](val featureSetName: String = "") extends mutable.HashMap[String, mutable.HashMap[Key, (Option[Int], mutable.Buffer[(String, Double)])]] {
  val defaultFeatures = new ListBuffer[(String, Double)]()

  def addDefaultFeatures(features: Seq[(String, Double)]) {
    for ((fname, fvalue) <- features) {
      defaultFeatures.find(_._1 == featureSetName + fname) match {
        case None => {
          defaultFeatures += (featureSetName + fname) -> fvalue
        } // new default feature, fill!
        case Some((_, `fvalue`)) => {
          // value already filled, but with same value - no need to complain
        }
        case Some((_, othervalue)) =>
          throw new RuntimeException("FeatureCollector#addDefaultFeatures: " +
            "Tried to set default feature " + featureSetName + fname + " with value " + fvalue + ", " +
            "but it was already set to default value " + othervalue)
      }
    }
  }

  def prependFeatureSetName(features: Seq[(String, Double)]): Seq[(String, Double)] = {
    features.map(feat => (featureSetName + feat._1, feat._2))
  }

  def addEntry(query: String, entry: Key, classLabel: Option[Int]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    if (!queryMap.contains(entry)) {
      queryMap.update(entry, (classLabel, new ListBuffer[(String, Double)]))
    }
  }

  def setClassLabel(query: String, entry: Key, classLabel: Option[Int]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, features) = queryMap.getOrElseUpdate(entry, (classLabel, new ListBuffer[(String, Double)]))
    queryMap.update(entry, (classLabel, features))
  }


  def addFeatures(query: String, entry: Key, features: Seq[(String, Double)]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures ++= prependFeatureSetName(features)
  }

  def addFeaturesNoPrepend(query: String, entry: Key, features: Seq[(String, Double)]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures ++= features
  }

  def getFeatures(query: String, entry: Key): Option[Seq[(String, Double)]] = {
    // here we exploit that an option is a sequence of length 0 or 1, map.get(..) gives and Option
    for (queryMap <- this.get(query); (_, features) <- queryMap.get(entry)) yield {
      features
    }
  }

  def getFeaturesWithDefault(query:String, entry:Key):Seq[(String,Double)] = {
    val localFeatures = getFeatures(query, entry).getOrElse(Seq.empty).toMap
    for((fname, defvalue) <- defaultFeatures) yield {
      fname -> localFeatures.getOrElse(fname, defvalue)
    }
  }


  def addClassAndFeatures(query: String, entry: Key, classLabel: Option[Int], features: Seq[(String, Double)]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures ++= prependFeatureSetName(features)
    queryMap.update(entry, (classLabel, oldFeatures))
  }

  def addClassAndFeaturesNoPrepend(query: String,
                                   entry: Key,
                                   classLabel: Option[Int],
                                   features: Seq[(String, Double)]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures ++= features
    queryMap.update(entry, (classLabel, oldFeatures))
  }

  def addFeature(query: String, entry: Key, featureName: String, featureValue: Double) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures += (featureSetName + featureName) -> featureValue
  }
<commit_msg>removing fcDoc/fcPgs prefix (to match against predefined domainMap)<commit_after>package edu.umass.ciir.strepsirank.collector

import scala.collection.mutable.ListBuffer
import scala.collection.mutable
import edu.umass.ciir.strepsi.{StringTools, SeqTools}
import java.io.FileWriter

/**
 *
 * queryid -> {document -> (Option[Label], features)}
 * where features = Seq[(String,Double)]
 * User: dietz
 * Date: 12/9/13
 * Time: 6:00 PM
 */

class StrepsiKeyTypedFeatureCollector[Key](val featureSetName: String = "") extends mutable.HashMap[String, mutable.HashMap[Key, (Option[Int], mutable.Buffer[(String, Double)])]] {
  val defaultFeatures = new ListBuffer[(String, Double)]()

  def addDefaultFeatures(features: Seq[(String, Double)]) {
    for ((fname, fvalue) <- features) {
      defaultFeatures.find(_._1 == featureSetName + fname) match {
        case None => {
          defaultFeatures += (featureSetName + fname) -> fvalue
        } // new default feature, fill!
        case Some((_, `fvalue`)) => {
          // value already filled, but with same value - no need to complain
        }
        case Some((_, othervalue)) =>
          throw new RuntimeException("FeatureCollector#addDefaultFeatures: " +
            "Tried to set default feature " + featureSetName + fname + " with value " + fvalue + ", " +
            "but it was already set to default value " + othervalue)
      }
    }
  }

  def prependFeatureSetName(features: Seq[(String, Double)]): Seq[(String, Double)] = {
    features.map(feat => (featureSetName + feat._1, feat._2))
  }

  def addEntry(query: String, entry: Key, classLabel: Option[Int]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    if (!queryMap.contains(entry)) {
      queryMap.update(entry, (classLabel, new ListBuffer[(String, Double)]))
    }
  }

  def setClassLabel(query: String, entry: Key, classLabel: Option[Int]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, features) = queryMap.getOrElseUpdate(entry, (classLabel, new ListBuffer[(String, Double)]))
    queryMap.update(entry, (classLabel, features))
  }


  def addFeatures(query: String, entry: Key, features: Seq[(String, Double)]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures ++= prependFeatureSetName(features)
  }

  def addFeaturesNoPrepend(query: String, entry: Key, features: Seq[(String, Double)]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures ++= features
  }

  def getFeatures(query: String, entry: Key): Option[Seq[(String, Double)]] = {
    // here we exploit that an option is a sequence of length 0 or 1, map.get(..) gives and Option
    for (queryMap <- this.get(query); (_, features) <- queryMap.get(entry)) yield {
      features
    }
  }

  def getFeaturesWithDefault(query:String, entry:Key):Seq[(String,Double)] = {
    val localFeatures = getFeatures(query, entry).getOrElse(Seq.empty).toMap
    for((fname, defvalue) <- defaultFeatures) yield {
      val value = localFeatures.getOrElse(fname, defvalue)
      fname -> value
    }
  }


  def addClassAndFeatures(query: String, entry: Key, classLabel: Option[Int], features: Seq[(String, Double)]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures ++= prependFeatureSetName(features)
    queryMap.update(entry, (classLabel, oldFeatures))
  }

  def addClassAndFeaturesNoPrepend(query: String,
                                   entry: Key,
                                   classLabel: Option[Int],
                                   features: Seq[(String, Double)]) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures ++= features
    queryMap.update(entry, (classLabel, oldFeatures))
  }

  def addFeature(query: String, entry: Key, featureName: String, featureValue: Double) {
    val queryMap = this.getOrElseUpdate(query, new mutable.HashMap())
    val (_, oldFeatures) = queryMap.getOrElseUpdate(entry, (None, new ListBuffer[(String, Double)]))
    oldFeatures += (featureSetName + featureName) -> featureValue
  }

{'fraction_non_alphanumeric': 0.1338670224594174, 'fraction_numerical': 0.00289081609962197, 'mean_word_length': 3.7367035281727223, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '143273', 'n_tokens_mistral': 2757, 'n_tokens_neox': 2692, 'n_words': 652}