<commit_before>import Workers.nodeType
import akka.actor.Actor

import scala.collection.mutable
/**
 * Created by Maginecz on 3/16/2015.
 */
package object Workers {
  type nodeType = Vector[Any]
}
case class ChangeSet(positive: Vector[nodeType] = Vector(), negative: Vector[nodeType] = Vector())

class Trimmer(val next: (ChangeSet) => Unit, val selectionVector: Vector[Int]) extends Actor {
  override def receive: Receive = {
    case ChangeSet(positive, negative) => {
      next(ChangeSet(
        positive.map(vec => selectionVector.map(i => vec(i))),
        negative.map(vec => selectionVector.map(i => vec(i)))
        )
      )
    }
  }
}

import scala.collection.mutable.MultiMap

case class Primary(changeSet: ChangeSet)
case class Secondary(changeSet: ChangeSet)

class HashJoiner(val next:(ChangeSet) => Unit,
               val primaryLength: Int, val primarySelector: Vector[Int],
               val secondaryLength: Int, val secondarySelector: Vector[Int])
  extends Actor {
  var primaryValues = new mutable.HashMap[nodeType, mutable.Set[nodeType]] with MultiMap[nodeType, nodeType]
  var secondaryValues = new mutable.HashMap[nodeType, mutable.Set[nodeType]] with MultiMap[nodeType, nodeType]

  val inversePrimarySelector = Vector.range(0,primaryLength) filter( i=> !primarySelector.contains(i))
  val inverseSecondarySelector = Vector.range(0,secondaryLength) filter( i=> !secondarySelector.contains(i))

  override def receive: Actor.Receive = {
    case Primary(ChangeSet(positive, negative)) => {
      val joinedPositive = positive.filter( primaryVec => {
        secondaryValues.contains(primarySelector.map(i => primaryVec(i)))
      }).
      map{
        primaryVec => {
          val key = primarySelector.map(i => primaryVec(i))
          val rest = inversePrimarySelector.map(i => primaryVec(i))
          secondaryValues(key).map(
            secondaryVec => {
              primaryVec ++ inverseSecondarySelector.map( i=>secondaryVec(i) )
            }
          ).toVector
        }
      }
      val joinedNegative = negative.map(
        primaryVec => {
          val key = primarySelector.map(i => primaryVec(i))
          secondaryValues(key).map(
            secondaryVec => {
              primaryVec ++ inverseSecondarySelector.map( i=>secondaryVec(i) )
            }.toVector
          )
        }
      )
      next(ChangeSet(joinedPositive.flatten,joinedNegative.flatten))
      positive.foreach(
        vec => {
          val key = primarySelector.map(i => vec(i))
          primaryValues.addBinding(key,vec)
        }
      )
      negative.foreach(
        vec =>{
          val key = primarySelector.map(i => vec(i))
          primaryValues.removeBinding(key,vec)
        }
      )
    }
    case Secondary(ChangeSet(positive, negative)) => {
      val joinedPositive = positive.map{
        secondaryVec => {
          val key = secondarySelector.map(i => secondaryVec(i))
          val rest = inverseSecondarySelector.map(i => secondaryVec(i))
          primaryValues(key).map(
            primaryVec => {
              primaryVec ++ inverseSecondarySelector.map( i=>secondaryVec(i) )
            }
          ).toVector
        }
      }
      val joinedNegative = negative.map(
        secondaryVec => {
          val key = secondarySelector.map(i => secondaryVec(i))
          val rest = inverseSecondarySelector.map(i => secondaryVec(i))
          primaryValues(key).map(
            primaryVec => {
              primaryVec ++ inverseSecondarySelector.map( i=>secondaryVec(i) )
            }.toVector
          )
        }
      )

      next(ChangeSet(joinedPositive.flatten,joinedNegative.flatten))

      positive.foreach(
        vec => {
          val key = secondarySelector.map(i => vec(i))
          secondaryValues.addBinding(key,vec) //must be used with multimaps
        }
      )
      negative.foreach(
        vec =>{
          val key = secondarySelector.map(i => vec(i))
          secondaryValues.removeBinding(key,vec)
        }
      )
    }
  }
}

class Checker(val next: (ChangeSet)=> Unit, val condition: (nodeType)=> Boolean) extends Actor{
  override def receive: Actor.Receive = {
    case ChangeSet(positive, negative) => {
      next(ChangeSet(
        positive.filter(condition),
        negative.filter(condition)
      ))
    }
  }
}

class InequalityChecker(override val next: (ChangeSet)=> Unit,val nodeIndex: Int, val inequals: Vector[Int]) extends Checker(
  next,
  (node:nodeType)=> { !inequals.map{ i => node(i)}.exists{ value =>value==node(nodeIndex)}}
)

class EqualityChecker(override val next: (ChangeSet)=> Unit,val nodeIndex: Int, val equals: Vector[Int]) extends Checker(
  next,
  (node:nodeType)=> { equals.map{ i => node(i)}.forall{ value =>value==node(nodeIndex)}}
)



class Printer() extends Actor {<commit_msg>replaced maps and filters with for loop in hashjoin<commit_after>import Workers.nodeType
import akka.actor.Actor

import scala.collection.mutable
/**
 * Created by Maginecz on 3/16/2015.
 */
package object Workers {
  type nodeType = Vector[Any]
}
case class ChangeSet(positive: Vector[nodeType] = Vector(), negative: Vector[nodeType] = Vector())

class Trimmer(val next: (ChangeSet) => Unit, val selectionVector: Vector[Int]) extends Actor {
  override def receive: Receive = {
    case ChangeSet(positive, negative) => {
      next(ChangeSet(
        positive.map(vec => selectionVector.map(i => vec(i))),
        negative.map(vec => selectionVector.map(i => vec(i)))
        )
      )
    }
  }
}

import scala.collection.mutable.MultiMap

case class Primary(changeSet: ChangeSet)
case class Secondary(changeSet: ChangeSet)

class HashJoiner(val next:(ChangeSet) => Unit,
               val primaryLength: Int, val primarySelector: Vector[Int],
               val secondaryLength: Int, val secondarySelector: Vector[Int])
  extends Actor {
  var primaryValues = new mutable.HashMap[nodeType, mutable.Set[nodeType]] with MultiMap[nodeType, nodeType]
  var secondaryValues = new mutable.HashMap[nodeType, mutable.Set[nodeType]] with MultiMap[nodeType, nodeType]

  val inversePrimarySelector = Vector.range(0,primaryLength) filter( i=> !primarySelector.contains(i))
  val inverseSecondarySelector = Vector.range(0,secondaryLength) filter( i=> !secondarySelector.contains(i))

  override def receive: Actor.Receive = {
    case Primary(ChangeSet(positive, negative)) => {
      val joinedPositive = for {
          primaryVec <- positive
          key = primarySelector.map(i => primaryVec(i))
          if secondaryValues.contains(key)
          secondaryVec <- secondaryValues(key)
    }
        yield primaryVec ++ inverseSecondarySelector.map( i=>secondaryVec(i) )


      val joinedNegative = for {
        primaryVec <- negative
        key = primarySelector.map(i => primaryVec(i))
        if secondaryValues.contains(key)
        secondaryVec <- secondaryValues(key)
      }
        yield primaryVec ++ inverseSecondarySelector.map( i=>secondaryVec(i) )

      next(ChangeSet(joinedPositive,joinedNegative))
      positive.foreach(
        vec => {
          val key = primarySelector.map(i => vec(i))
          primaryValues.addBinding(key,vec)
        }
      )
      negative.foreach(
        vec =>{
          val key = primarySelector.map(i => vec(i))
          primaryValues.removeBinding(key,vec)
        }
      )
    }
    case Secondary(ChangeSet(positive, negative)) => {
      val joinedPositive = for {
        secondaryVec <- positive
        key = secondarySelector.map(i => secondaryVec(i))
        if primaryValues.contains(key)
        primaryVec <- primaryValues(key)
    }
       yield primaryVec ++ inverseSecondarySelector.map( i=>secondaryVec(i) )


      val joinedNegative = for {
        secondaryVec <- negative
        key = secondarySelector.map(i => secondaryVec(i))
        if primaryValues.contains(key)
        primaryVec <- primaryValues(key)
    }
       yield primaryVec ++ inverseSecondarySelector.map( i=>secondaryVec(i) )

      next(ChangeSet(joinedPositive,joinedNegative))

      positive.foreach(
        vec => {
          val key = secondarySelector.map(i => vec(i))
          secondaryValues.addBinding(key,vec) //must be used with multimaps
        }
      )
      negative.foreach(
        vec =>{
          val key = secondarySelector.map(i => vec(i))
          secondaryValues.removeBinding(key,vec)
        }
      )
    }
  }
}

class Checker(val next: (ChangeSet)=> Unit, val condition: (nodeType)=> Boolean) extends Actor{
  override def receive: Actor.Receive = {
    case ChangeSet(positive, negative) => {
      next(ChangeSet(
        positive.filter(condition),
        negative.filter(condition)
      ))
    }
  }
}

class InequalityChecker(override val next: (ChangeSet)=> Unit,val nodeIndex: Int, val inequals: Vector[Int]) extends Checker(
  next,
  (node:nodeType)=> { !inequals.map{ i => node(i)}.exists{ value =>value==node(nodeIndex)}}
)

class EqualityChecker(override val next: (ChangeSet)=> Unit,val nodeIndex: Int, val equals: Vector[Int]) extends Checker(
  next,
  (node:nodeType)=> { equals.map{ i => node(i)}.forall{ value =>value==node(nodeIndex)}}
)

class Printer() extends Actor {
{'fraction_non_alphanumeric': 0.11965160587915079, 'fraction_numerical': 0.00195971692977681, 'mean_word_length': 2.882502113271344, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 92, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '224128', 'n_tokens_mistral': 2671, 'n_tokens_neox': 2582, 'n_words': 574}