<commit_before>package org.goobs.truth.scripts

import scala.collection.JavaConversions._
import scala.io.Source
import gnu.trove.map.TObjectIntMap
import gnu.trove.map.TLongFloatMap
import gnu.trove.map.TObjectFloatMap
import gnu.trove.map.hash.TObjectIntHashMap
import gnu.trove.map.hash.TLongFloatHashMap
import gnu.trove.map.hash.TObjectFloatHashMap
import gnu.trove.procedure.TObjectFloatProcedure

import java.io._
import java.util.zip.GZIPInputStream
import java.sql.Connection
import java.sql.PreparedStatement
import java.sql.ResultSet

import edu.stanford.nlp.io.IOUtils._
import edu.stanford.nlp.util.logging.Redwood
import edu.stanford.nlp.util.logging.Redwood.Util._

import org.goobs.truth._
import org.goobs.truth.Postgres._
import org.goobs.truth.Implicits._
import org.goobs.truth.EdgeType._
import org.goobs.truth.Utils._


//
// SQL prerequisite statements for this script:
// CREATE TABLE facts ( left_arg INTEGER[], rel INTEGER[], right_arg INTEGER[], weight REAL );
// CREATE INDEX index_fact_left_arg on "facts" USING GIN ("left_arg");
// CREATE INDEX index_fact_right_arg on "facts" USING GIN ("right_arg");
// CREATE INDEX index_fact_rel on "facts" USING GIN ("rel");
//

/**
 * The entry point for reading and indexing the ReVerb facts.
 *
 * @author <NAME>
 */
object IndexFacts {
  import scala.language.implicitConversions
  
  private val logger = Redwood.channels("Facts")

  case class MinimalFact(leftArg:Array[Int], rel:Array[Int], rightArg:Array[Int]) {
    override def hashCode:Int = {
      leftArg.foldLeft(31) { case (h:Int, term:Int) => h ^ (31 * term); } ^
        rel.foldLeft(31) { case (h:Int, term:Int) => h ^ (31 * term); } ^
        rightArg.foldLeft(31) { case (h:Int, term:Int) => h ^ (31 * term); }
    }

    def equals(a:Array[Int], b:Array[Int]):Boolean = {
      if (a.length != b.length) { return false; }
      for (i <- 0 until a.length) {
        if (a(i) != b(i)) { return false; }
      }
      return true;
    }

    override def equals(other:Any):Boolean = {
      other match {
        case (o:MinimalFact) =>
          equals(leftArg, o.leftArg) && equals(rel, o.rel) && equals(rightArg, o.rightArg)
        case _ => false
      }
    }

    def hash:Long = {
      val fact = leftArg ++ rel ++ rightArg
      val (hash, shift) = fact.foldLeft( (0, 0) ) { 
            case ( (hash, shift), word ) =>
          (hash ^ (word << shift), shift + (64 / fact.length))
        }
      hash
    }
  }
  implicit def fn2troveFloatFn[E](fn:(E,Float)=>Any):TObjectFloatProcedure[E] = {
    new TObjectFloatProcedure[E]() {
      override def execute(key:E, value:Float):Boolean = {
        fn( key, value )
        true
      }
    }
  }

  def index(rawPhrase:String, allowEmpty:Boolean=false)
           (implicit wordIndexer:TObjectIntMap[String]):Option[Array[Int]] = {
    if (!allowEmpty && rawPhrase.trim.equals("")) { return None }
    val phrase:Array[String] = tokenizeWithCase(rawPhrase)
    // Create object to store result
    val indexResult:Array[Int] = new Array[Int](phrase.length)
    for (i <- 0 until indexResult.length) { indexResult(i) = -1 }

    // Find largest phrases to index into (case sensitive)
    for (length <- phrase.length until 0 by -1;
         start <- 0 to phrase.length - length) {
      if ( (start until (start + length)) forall (indexResult(_) < 0) ) {
        val candidate:String = phrase.slice(start, start+length).mkString(" ")
        if (wordIndexer.containsKey(candidate)) {
          val index = wordIndexer.get(candidate)
          for (i <- start until start + length) { indexResult(i) = index; }
        }
      }
    }
    
    // Find largest phrases to index into (case insensitive)
    for (length <- phrase.length until 0 by -1;
         start <- 0 to phrase.length - length) {
      if ( (start until (start + length)) forall (indexResult(_) < 0) ) {
        val candidate:String = phrase.slice(start, start+length).mkString(" ")
        if (wordIndexer.containsKey(candidate)) {
          val index = wordIndexer.get(candidate)
          for (i <- start until start + length) { indexResult(i) = index; }
        }
      }
    }

    // Create resulting array
    var lastElem:Int = -999
    var rtn = List[Int]()
    for (i <- indexResult.length - 1 to 0 by -1) {
      if (indexResult(i) < 0) { return None }
      if (indexResult(i) != lastElem) {
        lastElem = indexResult(i)
        rtn = indexResult(i) :: rtn
      }
    }
    return Some(rtn.toArray)
  }
      
  var factCumulativeWeight = List( new TLongFloatHashMap )
  
  def updateWeight(key:Long, confidence:Float):Float = {
    this.synchronized {
      // Find the map which likely contains this key
      val candidateMap = factCumulativeWeight
        .collectFirst{ case (map:TLongFloatMap) if (map.containsKey(key)) => map }
        .getOrElse(factCumulativeWeight.head)
      try {
        // Try adding the weight
        candidateMap.adjustOrPutValue(key, confidence, confidence)
      } catch {
        case (e:IllegalStateException) =>
          // Could nto add; try making more maps (can overflow hashmap?)
          warn("Could not write to hash map (" + e.getMessage + ")")
          factCumulativeWeight = new TLongFloatHashMap :: factCumulativeWeight;
          try {
            // Add to new hash map
            factCumulativeWeight.head.adjustOrPutValue(key, confidence, confidence)
          } catch {
            // Still can't write? Give up.
            case (e:IllegalStateException) => e.printStackTrace
            confidence
          }
      }
    }
  }

  def getWeight(key:Long):Float = {
    factCumulativeWeight
      .collectFirst{ case (map:TLongFloatMap) if (map.containsKey(key)) => map }
      .map{ _.get(key) }.getOrElse(0.0f)
  }


  def main(args:Array[String]):Unit = {
    edu.stanford.nlp.NLPConfig.caseless  // set caseless models
    Props.exec(() => {
      // Read word index
      forceTrack("Reading words")
      implicit val wordInterner = new TObjectIntHashMap[String]
      var count = 0;
      slurpTable(TABLE_WORD_INTERN, {(r:ResultSet) =>
        val key:Int = r.getInt("index")
        val gloss:String = r.getString("gloss")
        wordInterner.put(gloss, key)
        count += 1;
        if (count % 1000000 == 0) { 
          logger.log("read " + (count / 1000000) + "M words; " + (Runtime.getRuntime.freeMemory / 1000000) + " MB of memory free")
        }
      })
      endTrack("Reading words")
      
      // Read facts
      val toInsertAllPending:TObjectFloatMap[MinimalFact] = new TObjectFloatHashMap[MinimalFact]
      startTrack("Adding Facts (parallel)")
      for (file <- iterFilesRecursive(Props.SCRIPT_REVERB_RAW_DIR).par) { try {
        val toInsert:TObjectFloatMap[MinimalFact] = new TObjectFloatHashMap[MinimalFact]
        val toUpdate:TObjectFloatMap[MinimalFact] = new TObjectFloatHashMap[MinimalFact]
        for (line <-
             try {
              { if (Props.SCRIPT_REVERB_RAW_GZIP) Source.fromInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(file))))
                else Source.fromFile(file) }.getLines
              } catch {
                case (e:java.util.zip.ZipException) =>
                  try {
                    Source.fromFile(file).getLines
                  } catch {
                    case (e:Exception) => Nil
                  }
              } ) { try {
          val fact = Fact(file.getName, line.split("\t"))
          if (fact.confidence >= 0.25) {
            for (leftArg:Array[Int] <- index(fact.leftArg.trim);
                 rightArg:Array[Int] <- index(fact.rightArg.trim, true);  // can be empty
                 relation:Array[Int] <- index(fact.relation.trim)) {
              // vv add fact vv
              // Get cumulative confidence<commit_msg>Forgot to threadsafe a map<commit_after>package org.goobs.truth.scripts

import scala.collection.JavaConversions._
import scala.io.Source
import gnu.trove.map.TObjectIntMap
import gnu.trove.map.TLongFloatMap
import gnu.trove.map.TObjectFloatMap
import gnu.trove.map.hash.TObjectIntHashMap
import gnu.trove.map.hash.TLongFloatHashMap
import gnu.trove.map.hash.TObjectFloatHashMap
import gnu.trove.procedure.TObjectFloatProcedure
import gnu.trove.TCollections._

import java.io._
import java.util.zip.GZIPInputStream
import java.sql.Connection
import java.sql.PreparedStatement
import java.sql.ResultSet

import edu.stanford.nlp.io.IOUtils._
import edu.stanford.nlp.util.logging.Redwood
import edu.stanford.nlp.util.logging.Redwood.Util._

import org.goobs.truth._
import org.goobs.truth.Postgres._
import org.goobs.truth.Implicits._
import org.goobs.truth.EdgeType._
import org.goobs.truth.Utils._


//
// SQL prerequisite statements for this script:
// CREATE TABLE facts ( left_arg INTEGER[], rel INTEGER[], right_arg INTEGER[], weight REAL );
// CREATE INDEX index_fact_left_arg on "facts" USING GIN ("left_arg");
// CREATE INDEX index_fact_right_arg on "facts" USING GIN ("right_arg");
// CREATE INDEX index_fact_rel on "facts" USING GIN ("rel");
//

/**
 * The entry point for reading and indexing the ReVerb facts.
 *
 * @author <NAME>
 */
object IndexFacts {
  import scala.language.implicitConversions
  
  private val logger = Redwood.channels("Facts")

  case class MinimalFact(leftArg:Array[Int], rel:Array[Int], rightArg:Array[Int]) {
    override def hashCode:Int = {
      leftArg.foldLeft(31) { case (h:Int, term:Int) => h ^ (31 * term); } ^
        rel.foldLeft(31) { case (h:Int, term:Int) => h ^ (31 * term); } ^
        rightArg.foldLeft(31) { case (h:Int, term:Int) => h ^ (31 * term); }
    }

    def equals(a:Array[Int], b:Array[Int]):Boolean = {
      if (a.length != b.length) { return false; }
      for (i <- 0 until a.length) {
        if (a(i) != b(i)) { return false; }
      }
      return true;
    }

    override def equals(other:Any):Boolean = {
      other match {
        case (o:MinimalFact) =>
          equals(leftArg, o.leftArg) && equals(rel, o.rel) && equals(rightArg, o.rightArg)
        case _ => false
      }
    }

    def hash:Long = {
      val fact = leftArg ++ rel ++ rightArg
      val (hash, shift) = fact.foldLeft( (0, 0) ) { 
            case ( (hash, shift), word ) =>
          (hash ^ (word << shift), shift + (64 / fact.length))
        }
      hash
    }
  }
  implicit def fn2troveFloatFn[E](fn:(E,Float)=>Any):TObjectFloatProcedure[E] = {
    new TObjectFloatProcedure[E]() {
      override def execute(key:E, value:Float):Boolean = {
        fn( key, value )
        true
      }
    }
  }

  def index(rawPhrase:String, allowEmpty:Boolean=false)
           (implicit wordIndexer:TObjectIntMap[String]):Option[Array[Int]] = {
    if (!allowEmpty && rawPhrase.trim.equals("")) { return None }
    val phrase:Array[String] = tokenizeWithCase(rawPhrase)
    // Create object to store result
    val indexResult:Array[Int] = new Array[Int](phrase.length)
    for (i <- 0 until indexResult.length) { indexResult(i) = -1 }

    // Find largest phrases to index into (case sensitive)
    for (length <- phrase.length until 0 by -1;
         start <- 0 to phrase.length - length) {
      if ( (start until (start + length)) forall (indexResult(_) < 0) ) {
        val candidate:String = phrase.slice(start, start+length).mkString(" ")
        if (wordIndexer.containsKey(candidate)) {
          val index = wordIndexer.get(candidate)
          for (i <- start until start + length) { indexResult(i) = index; }
        }
      }
    }
    
    // Find largest phrases to index into (case insensitive)
    for (length <- phrase.length until 0 by -1;
         start <- 0 to phrase.length - length) {
      if ( (start until (start + length)) forall (indexResult(_) < 0) ) {
        val candidate:String = phrase.slice(start, start+length).mkString(" ")
        if (wordIndexer.containsKey(candidate)) {
          val index = wordIndexer.get(candidate)
          for (i <- start until start + length) { indexResult(i) = index; }
        }
      }
    }

    // Create resulting array
    var lastElem:Int = -999
    var rtn = List[Int]()
    for (i <- indexResult.length - 1 to 0 by -1) {
      if (indexResult(i) < 0) { return None }
      if (indexResult(i) != lastElem) {
        lastElem = indexResult(i)
        rtn = indexResult(i) :: rtn
      }
    }
    return Some(rtn.toArray)
  }
      
  var factCumulativeWeight = List( new TLongFloatHashMap )
  
  def updateWeight(key:Long, confidence:Float):Float = {
    this.synchronized {
      // Find the map which likely contains this key
      val candidateMap = factCumulativeWeight
        .collectFirst{ case (map:TLongFloatMap) if (map.containsKey(key)) => map }
        .getOrElse(factCumulativeWeight.head)
      try {
        // Try adding the weight
        candidateMap.adjustOrPutValue(key, confidence, confidence)
      } catch {
        case (e:IllegalStateException) =>
          // Could nto add; try making more maps (can overflow hashmap?)
          warn("Could not write to hash map (" + e.getMessage + ")")
          factCumulativeWeight = new TLongFloatHashMap :: factCumulativeWeight;
          try {
            // Add to new hash map
            factCumulativeWeight.head.adjustOrPutValue(key, confidence, confidence)
          } catch {
            // Still can't write? Give up.
            case (e:IllegalStateException) => e.printStackTrace
            confidence
          }
      }
    }
  }

  def getWeight(key:Long):Float = {
    factCumulativeWeight
      .collectFirst{ case (map:TLongFloatMap) if (map.containsKey(key)) => map }
      .map{ _.get(key) }.getOrElse(0.0f)
  }


  def main(args:Array[String]):Unit = {
    edu.stanford.nlp.NLPConfig.caseless  // set caseless models
    Props.exec(() => {
      // Read word index
      forceTrack("Reading words")
      implicit val wordInterner = new TObjectIntHashMap[String]
      var count = 0;
      slurpTable(TABLE_WORD_INTERN, {(r:ResultSet) =>
        val key:Int = r.getInt("index")
        val gloss:String = r.getString("gloss")
        wordInterner.put(gloss, key)
        count += 1;
        if (count % 1000000 == 0) { 
          logger.log("read " + (count / 1000000) + "M words; " + (Runtime.getRuntime.freeMemory / 1000000) + " MB of memory free")
        }
      })
      endTrack("Reading words")
      
      // Read facts
      val toInsertAllPending:TObjectFloatMap[MinimalFact]
        = synchronizedMap(new TObjectFloatHashMap[MinimalFact])
      startTrack("Adding Facts (parallel)")
      for (file <- iterFilesRecursive(Props.SCRIPT_REVERB_RAW_DIR).par) { try {
        val toInsert:TObjectFloatMap[MinimalFact] = new TObjectFloatHashMap[MinimalFact]
        val toUpdate:TObjectFloatMap[MinimalFact] = new TObjectFloatHashMap[MinimalFact]
        for (line <-
             try {
              { if (Props.SCRIPT_REVERB_RAW_GZIP) Source.fromInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(file))))
                else Source.fromFile(file) }.getLines
              } catch {
                case (e:java.util.zip.ZipException) =>
                  try {
                    Source.fromFile(file).getLines
                  } catch {
                    case (e:Exception) => Nil
                  }
              } ) { try {
          val fact = Fact(file.getName, line.split("\t"))
          if (fact.confidence >= 0.25) {
            for (leftArg:Array[Int] <- index(fact.leftArg.trim);
                 rightArg:Array[Int] <- index(fact.rightArg.trim, true);  // can be empty
                 relation:Array[Int] <- index(fact.relation.trim)) {
              // vv add fact vv
              // Get cumulative confidence
{'fraction_non_alphanumeric': 0.12413837120245086, 'fraction_numerical': 0.008169517487873373, 'mean_word_length': 2.5069382273948073, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1075115', 'n_tokens_mistral': 5117, 'n_tokens_neox': 4870, 'n_words': 1137}