<commit_before>package com.github.kaeluka.spencer.analysis

import com.github.kaeluka.spencer.PostgresSpencerDB
import com.google.common.base.Stopwatch
import org.apache.spark.graphx._
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.DataFrame

case class ObjWithMeta(oid: VertexId,
                       klass: Option[String],
                       allocationSite: Option[String],
                       firstUsage: Long,
                       lastUsage: Long,
                       thread: Option[String])

case class ConnectedComponent() extends VertexIdAnalyser {
  def analyse(implicit g: SpencerDB): DataFrame = {
    import g.sqlContext.implicits._
    val graph: Graph[ObjDesc, EdgeDesc] = g.getGraph()
    val components: Graph[VertexId, EdgeDesc] = graph.subgraph(epred = _.attr.kind == EdgeKind.FIELD).connectedComponents()

    val joined = graph.outerJoinVertices(components.vertices) {
      (vid, _odesc, optCC) => optCC
    }

    val data = joined.vertices.toDF.withColumnRenamed("_1", "id").withColumnRenamed("_2", "connectedComponent")

    data.show(10)

    data
  }

  override def explanation(): String = "are connected"
}

case class WithMetaInformation(inner: VertexIdAnalyser) extends SpencerAnalyser[RDD[ObjWithMeta]] {

  override def analyse(implicit g: SpencerDB): RDD[ObjWithMeta] = {
    import g.sqlContext.implicits._
    val matchingIDs = inner.analyse(g).toDF

    println("getting meta info")
    println("WARNING: GETTING ALL META INFO! USE JOINS!")

    val frame = g.selectFrame("objects", "SELECT id, klass, allocationsitefile, allocationsiteline, firstusage, lastusage, thread " +
      "FROM objects")

    val ret =frame
      .rdd
      .map(row =>
        ObjWithMeta(
          oid = row.getAs[Long]("id"),
          klass = Option(row.getAs[String]("klass")),
          allocationSite = Option(row.getAs[String]("allocationsitefile"))
            .flatMap(file =>
              Option(row.getAs[Int]("allocationsiteline"))
                .map(line => file+":"+line.toString))
            .filter(! _.contains("<")),
          firstUsage = row.getAs[Long]("firstusage"),
          lastUsage = row.getAs[Long]("lastusage"),
          thread = Option(row.getAs[String]("thread")))
        )
    println("gotten meta info!")
    ret
  }

  override def pretty(result: RDD[ObjWithMeta]): String = {
    "Satisfying "+inner+":\n\t"+
      result.collect().mkString("\n")
  }

  def availableVariables : Map[String,String] = {
    Map(
      "klass"              -> "categorical",
      "allocationSite"     -> "categorical",
      "firstUsage"         -> "numerical",
      "lastUsage"          -> "numerical",
      "thread"             -> "categorical"
    )
  }

  override def explanation(): String = inner.explanation()

}

object WithMetaInformationTest extends App {

  implicit val db: SpencerDB = new PostgresSpencerDB("test")
  db.connect()

  val watch: Stopwatch = Stopwatch.createStarted()
  AgeOrderedObj().analyse.show()
  println("analysis took "+watch.stop())<commit_msg>add numFieldReads/FieldWrites/Calls variables<commit_after>package com.github.kaeluka.spencer.analysis

import com.github.kaeluka.spencer.PostgresSpencerDB
import com.google.common.base.Stopwatch
import org.apache.spark.graphx._
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.DataFrame

case class ObjWithMeta(oid: VertexId,
                       klass: Option[String],
                       allocationSite: Option[String],
                       firstUsage: Long,
                       lastUsage: Long,
                       thread: Option[String],
                       numFieldWrites: Long,
                       numFieldReads: Long,
                       numCalls: Long)

case class ConnectedComponent() extends VertexIdAnalyser {
  def analyse(implicit g: SpencerDB): DataFrame = {
    import g.sqlContext.implicits._
    val graph: Graph[ObjDesc, EdgeDesc] = g.getGraph()
    val components: Graph[VertexId, EdgeDesc] = graph.subgraph(epred = _.attr.kind == EdgeKind.FIELD).connectedComponents()

    val joined = graph.outerJoinVertices(components.vertices) {
      (vid, _odesc, optCC) => optCC
    }

    val data = joined.vertices.toDF.withColumnRenamed("_1", "id").withColumnRenamed("_2", "connectedComponent")

    data.show(10)

    data
  }

  override def explanation(): String = "are connected"
}

case class WithMetaInformation(inner: VertexIdAnalyser) extends SpencerAnalyser[RDD[ObjWithMeta]] {

  override def analyse(implicit g: SpencerDB): RDD[ObjWithMeta] = {
    import g.sqlContext.implicits._
    val matchingIDs = inner.analyse(g).toDF

    println("getting meta info")
    println("WARNING: GETTING ALL META INFO! USE JOINS!")

    val frame = g.selectFrame("objects", "SELECT id, klass, allocationsitefile, allocationsiteline, firstusage, lastusage, thread " +
      "FROM objects")

    val ret =frame
      .rdd
      .map(row =>
        ObjWithMeta(
          oid = row.getAs[Long]("id"),
          klass = Option(row.getAs[String]("klass")),
          allocationSite = Option(row.getAs[String]("allocationsitefile"))
            .flatMap(file =>
              Option(row.getAs[Int]("allocationsiteline"))
                .map(line => file+":"+line.toString))
            .filter(! _.contains("<")),
          firstUsage = row.getAs[Long]("firstusage"),
          lastUsage = row.getAs[Long]("lastusage"),
          thread = Option(row.getAs[String]("thread")),
          numFieldWrites = (Math.random()*1000).asInstanceOf[Long],
          numFieldReads = (Math.random()*1000).asInstanceOf[Long],
          numCalls = (Math.random()*1000).asInstanceOf[Long]
          )
        )
    println("gotten meta info!")
    ret
  }

  override def pretty(result: RDD[ObjWithMeta]): String = {
    "Satisfying "+inner+":\n\t"+
      result.collect().mkString("\n")
  }

  def availableVariables : Map[String,String] = {
    Map(
      "klass"              -> "categorical",
      "allocationSite"     -> "categorical",
      "firstUsage"         -> "numerical",
      "lastUsage"          -> "numerical",
      "thread"             -> "categorical",
      "numFieldWrites"     -> "numerical",
      "numFieldReads"      -> "numerical",
      "numCalls"           -> "numerical"
    )
  }

  override def explanation(): String = inner.explanation()

}

object WithMetaInformationTest extends App {

  implicit val db: SpencerDB = new PostgresSpencerDB("test")
  db.connect()

  val watch: Stopwatch = Stopwatch.createStarted()
  AgeOrderedObj().analyse.show()
  println("analysis took "+watch.stop())
{'fraction_non_alphanumeric': 0.12960995598725147, 'fraction_numerical': 0.0030353619669145544, 'mean_word_length': 2.746446844798181, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4745105', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1970, 'n_words': 387}