<commit_before>package epic.dense

import breeze.linalg.DenseVector
import scala.collection.mutable.HashMap
import breeze.util.Index
import breeze.linalg.DenseMatrix
import breeze.linalg.Counter2
import breeze.linalg.Counter
import epic.features.HackyLexicalProductionFeaturizer
import breeze.linalg.sum
import epic.features.RuleBasedHackyHeadFinder
import epic.features.HackyHeadFinder
import epic.parser.RuleTopology
import epic.trees.AnnotatedLabel

class Word2VecIndexed[W](val wordIndex: Index[W],
                         val word2vec: Array[Array[Double]],
                         val converter: W => W) {
  
  def wordRepSize = word2vec.head.size
//  def vectorSize: Int = 6 * wordRepSize
  def vocSize = wordIndex.size
    
  val zeroVector = Array.tabulate(wordRepSize)(i => 0.0)
  
  def fetch(wordIdx: Int): DenseVector[Double] = {
    if (wordIdx != -1 && wordIdx < word2vec.size) DenseVector(word2vec(wordIdx)) else DenseVector(zeroVector)
  }
   
  def assemble(vectors: Seq[Array[Double]]) = vectors.reduce(_ ++ _)
  
  def convertToVector(indexedWords: Array[Int]): Array[Double] = {
    assemble(indexedWords.map(wordIdx => if (wordIdx == -1) zeroVector else word2vec(wordIdx)))
  }
  
  def augment(numSparseFeats: Int, featurizer: W => Array[Int]): Word2VecIndexed[W] = {
    val newWord2Vec = Array.tabulate(word2vec.size)(i => {
      val word = wordIndex.get(i)
      val feats = featurizer(word)
      word2vec(i) ++ Array.tabulate(numSparseFeats)(j => if (feats.contains(j)) 1.0 else 0.0)
    })
    new Word2VecIndexed(wordIndex, newWord2Vec, converter)
  }
}

object Word2VecIndexed {
  
  def apply[W](word2vec: HashMap[W,Array[Double]],
               converter: W => W) = {
    val index = Index[W]
    val arr = new Array[Array[Double]](word2vec.size)
    for (word <- word2vec.keySet) {
      arr(index.index(word)) = word2vec(word)
    }
    new Word2VecIndexed(index, arr, converter)
  }
}

trait WordVectorAnchoringIndexed[String] {
  def featuresForSpan(start: Int, end: Int): Array[Int];
  def featuresForSplit(start: Int, split: Int, end: Int): Array[Int];
}

class Word2VecSurfaceFeaturizerIndexed[W](val word2vecIndexed: Word2VecIndexed[W],
                                          val featureSpec: String) {
  
  def inputSize = {
    anchor(IndexedSeq[W]()).featuresForSplit(0, 0, 0).size * word2vecIndexed.wordRepSize
  }
  
  def anchor(words: IndexedSeq[W]): WordVectorAnchoringIndexed[W] = {
    val convertedWords = words.map(word2vecIndexed.converter(_))
    val indexedWords = convertedWords.map(word2vecIndexed.wordIndex(_))
    new WordVectorAnchoringIndexed[W] {
      
      def featuresForSpan(start: Int, end: Int) = {
//        Array(fetchWord(start - 1), fetchWord(start), -1, -1, fetchWord(end - 1), fetchWord(end))
        if (featureSpec == "") {
          Array(fetchWord(start - 1), fetchWord(start), -1, -1, fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "morecontext") {
          Array(fetchWord(start - 2), fetchWord(start - 1), fetchWord(start), -1, -1, fetchWord(end - 1), fetchWord(end), fetchWord(end + 1))
        } else if (featureSpec == "morefirstlast") {
          Array(fetchWord(start - 1), fetchWord(start), fetchWord(start + 1), -1, -1, fetchWord(end - 2), fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "moresplit") {
          Array(fetchWord(start - 1), fetchWord(start), -1, -1, -1, -1, fetchWord(end - 1), fetchWord(end))
        } else {
          throw new RuntimeException("Unknown featureSpec: " + featureSpec)
        }
      }
        
      def featuresForSplit(start: Int, split: Int, end: Int) = {
//        Array(fetchWord(start - 1), fetchWord(start), fetchWord(split - 1), fetchWord(split), fetchWord(end - 1), fetchWord(end))
        if (featureSpec == "") {
          Array(fetchWord(start - 1), fetchWord(start), fetchWord(split - 1), fetchWord(split), fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "morecontext") {
          Array(fetchWord(start - 2), fetchWord(start - 1), fetchWord(start), fetchWord(split - 1), fetchWord(split), fetchWord(end - 1), fetchWord(end), fetchWord(end + 1))
        } else if (featureSpec == "morefirstlast") {
          Array(fetchWord(start - 1), fetchWord(start), fetchWord(start + 1), fetchWord(split - 1), fetchWord(split), fetchWord(end - 2), fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "moresplit") {
          Array(fetchWord(start - 1), fetchWord(start), fetchWord(split - 2), fetchWord(split - 1), fetchWord(split), fetchWord(split + 1), fetchWord(end - 1), fetchWord(end))
        } else {
          throw new RuntimeException("Unknown featureSpec: " + featureSpec)<commit_msg>Expanded set of possible surface features<commit_after>package epic.dense

import breeze.linalg.DenseVector
import scala.collection.mutable.HashMap
import breeze.util.Index
import breeze.linalg.DenseMatrix
import breeze.linalg.Counter2
import breeze.linalg.Counter
import epic.features.HackyLexicalProductionFeaturizer
import breeze.linalg.sum
import epic.features.RuleBasedHackyHeadFinder
import epic.features.HackyHeadFinder
import epic.parser.RuleTopology
import epic.trees.AnnotatedLabel

class Word2VecIndexed[W](val wordIndex: Index[W],
                         val word2vec: Array[Array[Double]],
                         val converter: W => W) {
  
  def wordRepSize = word2vec.head.size
//  def vectorSize: Int = 6 * wordRepSize
  def vocSize = wordIndex.size
    
  val zeroVector = Array.tabulate(wordRepSize)(i => 0.0)
  
  def fetch(wordIdx: Int): DenseVector[Double] = {
    if (wordIdx != -1 && wordIdx < word2vec.size) DenseVector(word2vec(wordIdx)) else DenseVector(zeroVector)
  }
   
  def assemble(vectors: Seq[Array[Double]]) = vectors.reduce(_ ++ _)
  
  def convertToVector(indexedWords: Array[Int]): Array[Double] = {
    assemble(indexedWords.map(wordIdx => if (wordIdx == -1) zeroVector else word2vec(wordIdx)))
  }
  
  def augment(numSparseFeats: Int, featurizer: W => Array[Int]): Word2VecIndexed[W] = {
    val newWord2Vec = Array.tabulate(word2vec.size)(i => {
      val word = wordIndex.get(i)
      val feats = featurizer(word)
      word2vec(i) ++ Array.tabulate(numSparseFeats)(j => if (feats.contains(j)) 1.0 else 0.0)
    })
    new Word2VecIndexed(wordIndex, newWord2Vec, converter)
  }
}

object Word2VecIndexed {
  
  def apply[W](word2vec: HashMap[W,Array[Double]],
               converter: W => W) = {
    val index = Index[W]
    val arr = new Array[Array[Double]](word2vec.size)
    for (word <- word2vec.keySet) {
      arr(index.index(word)) = word2vec(word)
    }
    new Word2VecIndexed(index, arr, converter)
  }
}

trait WordVectorAnchoringIndexed[String] {
  def featuresForSpan(start: Int, end: Int): Array[Int];
  def featuresForSplit(start: Int, split: Int, end: Int): Array[Int];
}

class Word2VecSurfaceFeaturizerIndexed[W](val word2vecIndexed: Word2VecIndexed[W],
                                          val featureSpec: String) {
  
  def inputSize = {
    anchor(IndexedSeq[W]()).featuresForSplit(0, 0, 0).size * word2vecIndexed.wordRepSize
  }
  
  def anchor(words: IndexedSeq[W]): WordVectorAnchoringIndexed[W] = {
    val convertedWords = words.map(word2vecIndexed.converter(_))
    val indexedWords = convertedWords.map(word2vecIndexed.wordIndex(_))
    new WordVectorAnchoringIndexed[W] {
      
      def featuresForSpan(start: Int, end: Int) = {
//        Array(fetchWord(start - 1), fetchWord(start), -1, -1, fetchWord(end - 1), fetchWord(end))
        if (featureSpec == "") {
          Array(fetchWord(start - 1), fetchWord(start), -1, -1, fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "morecontext") {
          Array(fetchWord(start - 2), fetchWord(start - 1), fetchWord(start), -1, -1, fetchWord(end - 1), fetchWord(end), fetchWord(end + 1))
        } else if (featureSpec == "morefirstlast") {
          Array(fetchWord(start - 1), fetchWord(start), fetchWord(start + 1), -1, -1, fetchWord(end - 2), fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "moresplit") {
          Array(fetchWord(start - 1), fetchWord(start), -1, -1, -1, -1, fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "mcmfl") {
          Array(fetchWord(start - 2), fetchWord(start - 1), fetchWord(start), fetchWord(start + 1), -1, -1, fetchWord(end - 2), fetchWord(end - 1), fetchWord(end), fetchWord(end + 1))
        } else {
          throw new RuntimeException("Unknown featureSpec: " + featureSpec)
        }
      }
        
      def featuresForSplit(start: Int, split: Int, end: Int) = {
//        Array(fetchWord(start - 1), fetchWord(start), fetchWord(split - 1), fetchWord(split), fetchWord(end - 1), fetchWord(end))
        if (featureSpec == "") {
          Array(fetchWord(start - 1), fetchWord(start), fetchWord(split - 1), fetchWord(split), fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "morecontext") {
          Array(fetchWord(start - 2), fetchWord(start - 1), fetchWord(start), fetchWord(split - 1), fetchWord(split), fetchWord(end - 1), fetchWord(end), fetchWord(end + 1))
        } else if (featureSpec == "morefirstlast") {
          Array(fetchWord(start - 1), fetchWord(start), fetchWord(start + 1), fetchWord(split - 1), fetchWord(split), fetchWord(end - 2), fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "moresplit") {
          Array(fetchWord(start - 1), fetchWord(start), fetchWord(split - 2), fetchWord(split - 1), fetchWord(split), fetchWord(split + 1), fetchWord(end - 1), fetchWord(end))
        } else if (featureSpec == "mcmfl") {
          Array(fetchWord(start - 2), fetchWord(start - 1), fetchWord(start), fetchWord(start + 1), fetchWord(split - 1), fetchWord(split), fetchWord(end - 2), fetchWord(end - 1), fetchWord(end), fetchWord(end + 1))
        } else {
          throw new RuntimeException("Unknown featureSpec: " + featureSpec)
{'fraction_non_alphanumeric': 0.14157958199356913, 'fraction_numerical': 0.018388263665594855, 'mean_word_length': 3.4612281488121917, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4937147', 'n_tokens_mistral': 3348, 'n_tokens_neox': 3221, 'n_words': 705}