<commit_before>package org.goobs.truth.scripts

import scala.collection.JavaConversions._
import scala.io.Source

import java.io._
import java.util.zip.GZIPInputStream
import java.sql.Connection

import edu.stanford.nlp.io.IOUtils._
import edu.stanford.nlp.util.HashIndex
import edu.stanford.nlp.util.logging.Redwood.Util._

import edu.smu.tspell.wordnet._

import org.goobs.sim._

import org.goobs.truth._
import org.goobs.truth.Postgres._
import org.goobs.truth.Implicits._
import org.goobs.truth.EdgeType._

/**
 *
 * The entry point for constructing the nodes of a graph, indexed to a compact
 * form.
 *
 * @author <NAME>
 */

object CreateIndexers {
  
  val wordIndexer = new HashIndex[String]

  // ( begin, end, log(P(hyper) / P(hypo)) )
  val wordnetGraphUp = new scala.collection.mutable.ArrayBuffer[(Int, Int, Double)]
  val wordnetGraphDown = new scala.collection.mutable.ArrayBuffer[(Int, Int, Double)]
  
  // ( begin, end, [1.0] )
  val wordnetNounAntonym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetVerbAntonym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdjAntonym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdvAntonym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdjPertainym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdvPertainym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdjSimilar = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  
  // ( begin, end, acos(cos_similarity) / π )
  val angleNearestNeighbors = new scala.collection.mutable.ArrayBuffer[(Int, Int, Double)]
  
  // ( begin, end, [1.0] )
  val freebaseGraphUp = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val freebaseGraphDown = new scala.collection.mutable.ArrayBuffer[(Int, Int)]

  def main(args:Array[String]) = {
    Props.exec(() => {
      wordIndexer.indexOf("", true)
      val wordnet = Ontology.load(Props.SCRIPT_WORDNET_PATH)
      withConnection{ (psql:Connection) =>

        // Part 1: Read Wordnet
        // note: we are prohibiting ROOT as a valid node to traverse
        forceTrack("Reading WordNet")
        for ((phrase, nodes) <- wordnet.ontology) {
          val phraseAsString:String = phrase.mkString(" ")
          val source:Int = wordIndexer.indexOf(phraseAsString, true)
          for (node <- nodes;
               hyper <- node.hypernyms) {
            hyper match {
              case (hyperNode:Ontology.RealNode) =>
                val edgeWeight:Double = scala.math.log( hyperNode.count / node.count )
                // is_a ontology
                for (hyperWord <- hyperNode.synset.getWordForms) {
                  val hyperInt:Int = wordIndexer.indexOf(hyperWord, true)
                  wordnetGraphUp.append( (source, hyperInt, edgeWeight) )
                  wordnetGraphDown.append( (hyperInt, source, edgeWeight) )
                }
                // other wordnet relations
                hyperNode.synset match {
                  case (as:NounSynset) =>
                    for (antonym <- as.getAntonyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(antonym.getWordForm, true)
                      wordnetVerbAntonym.append( (source, sink) )
                    }
                  case (as:VerbSynset) =>
                    for (antonym <- as.getAntonyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(antonym.getWordForm, true)
                      wordnetNounAntonym.append( (source, sink) )
                    }
                  case (as:AdjectiveSynset) =>
                    for (related <- as.getSimilar;
                         wordForm <- related.getWordForms) {
                      val sink:Int = wordIndexer.indexOf(wordForm, true)
                      wordnetAdjSimilar.append( (source, sink) )
                    }
                    for (pertainym <- as.getPertainyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(pertainym.getWordForm, true)
                      wordnetAdjPertainym.append( (source, sink) )
                    }
                    for (antonym <- as.getAntonyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(antonym.getWordForm, true)
                      wordnetAdjAntonym.append( (source, sink) )
                    }
                  case (as:AdverbSynset) =>
                    for (pertainym <- as.getPertainyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(pertainym.getWordForm, true)
                      wordnetAdvPertainym.append( (source, sink) )
                    }
                    for (antonym <- as.getAntonyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(antonym.getWordForm, true)
                      wordnetAdvAntonym.append( (source, sink) )
                    }
                  case _ =>
                }
              case _ =>
                debug("sub-root node: " + node + " [" + hyper.getClass + "]")
            }
            
          }
        }
        endTrack("Reading WordNet")

        // Part 2: Read distsim
        forceTrack("Reading Nearest Neighbors")
        for (line <- Source.fromFile(Props.SCRIPT_DISTSIM_COS, "UTF-8").getLines) {
          val fields = line.split("\t")
          val source:Int = wordIndexer.indexOf(fields(0), true)
          for (i <- 1 until fields.length) {
            val scoreAndGloss = fields(i).split(" ")
            assert(scoreAndGloss.length == 2)
            val sink:Int = wordIndexer.indexOf(scoreAndGloss(1), true)
            val angle:Double = scala.math.acos( scoreAndGloss(0).toDouble ) / scala.math.Pi
            angleNearestNeighbors.append( (source, sink, angle) )
          }
        }
        endTrack("Reading Nearest Neighbors")

        // Part 3: Read freebase
        forceTrack("Reading Freebase")
        val fbNames = new scala.collection.mutable.HashMap[String, String]
        val hypernyms = new scala.collection.mutable.ArrayBuffer[ (String, String) ]
        forceTrack("Reading File")
        for (line <- Source.fromInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(Props.SCRIPT_FREEBASE_RAW_PATH)))).getLines) {
//        for (line <- Source.fromFile(Props.SCRIPT_FREEBASE_RAW_PATH).getLines.drop(1)) {
          val fields = line.split("\t")
          if (fields(1) == "fb:type.object.name") {
            fbNames.put(fields(0), fields(2).substring(1, fields(2).length - 5))
            fbNames.put(fields(0), fields(0))
            println("name: " + fields(0) + "  -->  " + fields(2).substring(1, fields(2).length - 5))
          } else if (fields(1) == "fb:type.object.type") {
            val target = if (fields(2).endsWith(".")) fields(2).substring(0, fields(2).length -1) else fields(2)
            hypernyms.append( (fields(0), target) )
            println("hyper: " + fields(0) + "  is_a  " + target)
          } else {
            /* do nothing */
          }
        }
        endTrack("Reading File")
        startTrack("Creating Edges")
        for ( (hypo, hyper) <- hypernyms ) {
          val hypoInt:Int = wordIndexer.indexOf(fbNames(hypo), true)
          val hyperInt:Int = wordIndexer.indexOf(fbNames(hyper), true)
          freebaseGraphUp.append( (hypoInt, hyperInt) )<commit_msg>Removed debugging statements<commit_after>package org.goobs.truth.scripts

import scala.collection.JavaConversions._
import scala.io.Source

import java.io._
import java.util.zip.GZIPInputStream
import java.sql.Connection

import edu.stanford.nlp.io.IOUtils._
import edu.stanford.nlp.util.HashIndex
import edu.stanford.nlp.util.logging.Redwood.Util._

import edu.smu.tspell.wordnet._

import org.goobs.sim._

import org.goobs.truth._
import org.goobs.truth.Postgres._
import org.goobs.truth.Implicits._
import org.goobs.truth.EdgeType._

/**
 *
 * The entry point for constructing the nodes of a graph, indexed to a compact
 * form.
 *
 * @author <NAME>
 */

object CreateIndexers {
  
  val wordIndexer = new HashIndex[String]

  // ( begin, end, log(P(hyper) / P(hypo)) )
  val wordnetGraphUp = new scala.collection.mutable.ArrayBuffer[(Int, Int, Double)]
  val wordnetGraphDown = new scala.collection.mutable.ArrayBuffer[(Int, Int, Double)]
  
  // ( begin, end, [1.0] )
  val wordnetNounAntonym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetVerbAntonym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdjAntonym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdvAntonym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdjPertainym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdvPertainym = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val wordnetAdjSimilar = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  
  // ( begin, end, acos(cos_similarity) / π )
  val angleNearestNeighbors = new scala.collection.mutable.ArrayBuffer[(Int, Int, Double)]
  
  // ( begin, end, [1.0] )
  val freebaseGraphUp = new scala.collection.mutable.ArrayBuffer[(Int, Int)]
  val freebaseGraphDown = new scala.collection.mutable.ArrayBuffer[(Int, Int)]

  def main(args:Array[String]) = {
    Props.exec(() => {
      wordIndexer.indexOf("", true)
      val wordnet = Ontology.load(Props.SCRIPT_WORDNET_PATH)
      withConnection{ (psql:Connection) =>

        // Part 1: Read Wordnet
        // note: we are prohibiting ROOT as a valid node to traverse
        forceTrack("Reading WordNet")
        for ((phrase, nodes) <- wordnet.ontology) {
          val phraseAsString:String = phrase.mkString(" ")
          val source:Int = wordIndexer.indexOf(phraseAsString, true)
          for (node <- nodes;
               hyper <- node.hypernyms) {
            hyper match {
              case (hyperNode:Ontology.RealNode) =>
                val edgeWeight:Double = scala.math.log( hyperNode.count / node.count )
                // is_a ontology
                for (hyperWord <- hyperNode.synset.getWordForms) {
                  val hyperInt:Int = wordIndexer.indexOf(hyperWord, true)
                  wordnetGraphUp.append( (source, hyperInt, edgeWeight) )
                  wordnetGraphDown.append( (hyperInt, source, edgeWeight) )
                }
                // other wordnet relations
                hyperNode.synset match {
                  case (as:NounSynset) =>
                    for (antonym <- as.getAntonyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(antonym.getWordForm, true)
                      wordnetVerbAntonym.append( (source, sink) )
                    }
                  case (as:VerbSynset) =>
                    for (antonym <- as.getAntonyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(antonym.getWordForm, true)
                      wordnetNounAntonym.append( (source, sink) )
                    }
                  case (as:AdjectiveSynset) =>
                    for (related <- as.getSimilar;
                         wordForm <- related.getWordForms) {
                      val sink:Int = wordIndexer.indexOf(wordForm, true)
                      wordnetAdjSimilar.append( (source, sink) )
                    }
                    for (pertainym <- as.getPertainyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(pertainym.getWordForm, true)
                      wordnetAdjPertainym.append( (source, sink) )
                    }
                    for (antonym <- as.getAntonyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(antonym.getWordForm, true)
                      wordnetAdjAntonym.append( (source, sink) )
                    }
                  case (as:AdverbSynset) =>
                    for (pertainym <- as.getPertainyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(pertainym.getWordForm, true)
                      wordnetAdvPertainym.append( (source, sink) )
                    }
                    for (antonym <- as.getAntonyms(phraseAsString)) {
                      val sink:Int = wordIndexer.indexOf(antonym.getWordForm, true)
                      wordnetAdvAntonym.append( (source, sink) )
                    }
                  case _ =>
                }
              case _ =>
                debug("sub-root node: " + node + " [" + hyper.getClass + "]")
            }
            
          }
        }
        endTrack("Reading WordNet")

        // Part 2: Read distsim
        forceTrack("Reading Nearest Neighbors")
        for (line <- Source.fromFile(Props.SCRIPT_DISTSIM_COS, "UTF-8").getLines) {
          val fields = line.split("\t")
          val source:Int = wordIndexer.indexOf(fields(0), true)
          for (i <- 1 until fields.length) {
            val scoreAndGloss = fields(i).split(" ")
            assert(scoreAndGloss.length == 2)
            val sink:Int = wordIndexer.indexOf(scoreAndGloss(1), true)
            val angle:Double = scala.math.acos( scoreAndGloss(0).toDouble ) / scala.math.Pi
            angleNearestNeighbors.append( (source, sink, angle) )
          }
        }
        endTrack("Reading Nearest Neighbors")

        // Part 3: Read freebase
        forceTrack("Reading Freebase")
        val fbNames = new scala.collection.mutable.HashMap[String, String]
        val hypernyms = new scala.collection.mutable.ArrayBuffer[ (String, String) ]
        forceTrack("Reading File")
/        for (line <- Source.fromInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(Props.SCRIPT_FREEBASE_RAW_PATH)))).getLines) {
//        for (line <- Source.fromFile(Props.SCRIPT_FREEBASE_RAW_PATH).getLines.drop(1)) {
          val fields = line.split("\t")
          if (fields(1) == "fb:type.object.name") {
            fbNames.put(fields(0), fields(2).substring(1, fields(2).length - 5))
            fbNames.put(fields(0), fields(0))
          } else if (fields(1) == "fb:type.object.type") {
            val target = if (fields(2).endsWith(".")) fields(2).substring(0, fields(2).length -1) else fields(2)
            hypernyms.append( (fields(0), target) )
          } else {
            /* do nothing */
          }
        }
        endTrack("Reading File")
        startTrack("Creating Edges")
        for ( (hypo, hyper) <- hypernyms ) {
          val hypoInt:Int = wordIndexer.indexOf(fbNames(hypo), true)
          val hyperInt:Int = wordIndexer.indexOf(fbNames(hyper), true)
          freebaseGraphUp.append( (hypoInt, hyperInt) )
{'fraction_non_alphanumeric': 0.11631089217296114, 'fraction_numerical': 0.00451559934318555, 'mean_word_length': 2.0981348028825773, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5318424', 'n_tokens_mistral': 4475, 'n_tokens_neox': 4266, 'n_words': 839}