<commit_before>package edu.berkeley.cs.amplab.carat

import spark._
import spark.SparkContext._
import spark.timeseries._
import edu.berkeley.cs.amplab.carat.plot.PlotUtil
import edu.berkeley.cs.amplab.carat.dynamodb.DynamoAnalysisUtil
import scala.collection.immutable.TreeMap
import scala.collection.mutable.HashMap
import scala.collection.mutable.Map
import collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import edu.berkeley.cs.amplab.carat.dynamodb.DynamoDbEncoder
import edu.berkeley.cs.amplab.carat.dynamodb.DynamoDbDecoder

/**
 * Do the exact same thing as in CaratDynamoDataToPlots, but do not collect() and write plot files and run plotting in the end.
 *
 * @author <NAME>
 */

object CaratNoRDDAnalysis {

  // How many clients do we need to consider data reliable?
  val ENOUGH_USERS = 5

  var DEBUG = false
  val DECIMALS = 3
  var BUCKETS = 100
  // Isolate from the plotting.
  val tmpdir = "/mnt/TimeSeriesSpark-osmodel/spark-temp-plots/"

  var userLimit = Int.MaxValue

  /* Right now we don't remove non-hogs from the bugs table. That should be done.*/
  var bugsToRemove = new ArrayBuffer[(String, String)]
  var hogsToRemove = new ArrayBuffer[(String, String)]

  /**
   * Main program entry point.
   */
  def main(args: Array[String]) {
    var master = "local[16]"
    if (args != null && args.length >= 1) {
      master = args(0)
    }
    if (args != null && args.length > 1)
      userLimit = args(1).toInt

    val start = DynamoAnalysisUtil.start()
    CaratAnalysisGeneric.genericAnalysis(master, tmpdir, userLimit, ENOUGH_USERS, DECIMALS,
      DynamoAnalysisUtil.removeDaemons, storeDists, storeJScores, globalCorrelations)
  }

  /* Generate a gnuplot-readable plot file of the bucketed distribution.
   * Create folders plots/data plots/plotfiles
   * Save it as "plots/data/titleWith-titleWithout".txt.
   * Also generate a plotfile called plots/plotfiles/titleWith-titleWithout.gnuplot
   */

  def storeDists(nature: String, keyValue1: String, keyValue2: String, title: String, titleNeg: String,
    one: Array[CaratRate], two: Array[CaratRate], aPrioriDistribution: Map[Double, Double], isBugOrHog: Boolean,
    filtered: Array[CaratRate], oses: Set[String], models: Set[String],
    totalsByUuid: scala.collection.immutable.TreeMap[String, (Double, Double)], usersWith: Int, usersWithout: Int, uuid: String) = {
    var hasSamples = true
    if (usersWith == 0 && usersWithout == 0) {
      hasSamples = one.take(1) match {
        case Array(t) => true
        case _ => false
      }
      hasSamples = two.take(1) match {
        case Array(t) => hasSamples && true
        case _ => false
      }
    }
    if (hasSamples) {
      val (xmax, probDist, probDistNeg, ev, evNeg, evDistance /*, usersWith, usersWithout*/ ) = DynamoAnalysisUtil.getDistanceAndDistributionsUnBucketed(one, two, aPrioriDistribution)
      if (probDist != null && probDistNeg != null && (!isBugOrHog || evDistance > 0)) {
        if (evDistance > 0) {
          var imprHr = (100.0 / evNeg - 100.0 / ev) / 3600.0
          val imprD = (imprHr / 24.0).toInt
          imprHr -= imprD * 24.0
          printf("%s evWith=%s evWithout=%s evDistance=%s improvement=%s days %s hours (%s vs %s users)\n", title, ev, evNeg, evDistance, imprD, imprHr, usersWith, usersWithout)
        } else {
          printf("%s evWith=%s evWithout=%s evDistance=%s (%s vs %s users)\n", title, ev, evNeg, evDistance, usersWith, usersWithout)
        }
        val (xmax2, distWith, distWithout) = ProbUtil.bucketDistributionsByX(probDist, probDistNeg, BUCKETS, DECIMALS)

        if (nature == "hog" && evDistance > 0) {
          // Need to delete new hogs from the bugs table:
          val (key, items) = DynamoDbDecoder.filterItems(bugsTable, (hogKey, keyValue1))
          for (k <- items) {
            val attr = k.get(resultKey)
            if (attr != null) {
              val uuid = attr.getS()
              DynamoDbDecoder.deleteItem(bugsTable, uuid, keyValue1)
            }
          }
        }
        
        val putFunction = storageFunction(nature, keyValue1, keyValue2)
        putFunction(xmax, distWith.toSeq, distWithout.toSeq, evDistance, ev, evNeg)

        // TODO:Store correlations?
        if (isBugOrHog && filtered != null) {
          val (osCorrelations, modelCorrelations, userCorrelations) = DynamoAnalysisUtil.correlation(title, filtered, aPrioriDistribution, models, oses, totalsByUuid)
          print(title, titleNeg, xmax, probDist, probDistNeg, ev, evNeg, evDistance, osCorrelations, modelCorrelations, userCorrelations, usersWith, usersWithout, uuid)
        } else {
          print(title, titleNeg, xmax, probDist, probDistNeg, ev, evNeg, evDistance, null, null, null, usersWith, usersWithout, uuid)
        }
      } else if (isBugOrHog && evDistance <= 0) {
        /* delete old hogs and bugs that are no longer hoggy/buggy */
        deleteFunction(nature, keyValue1, keyValue2)
      }
      isBugOrHog && evDistance > 0
    } else
      false
  }

  def storageFunction(nature: String, keyValue1: String, keyValue2: String, uuidApps: Seq[String] = null) = {
    val putFunction: (Double, Seq[(Int, Double)], Seq[(Int, Double)], Double, Double, Double) => Unit = nature match {
      case "hog" => { DynamoDbEncoder.put(hogsTable, hogKey, keyValue1, _, _, _, _, _, _) }
      case "similar" => { DynamoDbEncoder.put(similarsTable, similarKey, keyValue1, _, _, _, _, _, _) }
      case "os" => { DynamoDbEncoder.put(osTable, osKey, keyValue1, _, _, _, _, _, _) }
      case "model" => { DynamoDbEncoder.put(modelsTable, modelKey, keyValue1, _, _, _, _, _, _) }
      case "result" => { DynamoDbEncoder.put(resultsTable, resultKey, keyValue1, _, _, _, _, _, _, uuidApps) }
      case "bug" => { DynamoDbEncoder.putBug(bugsTable, (resultKey, hogKey), (keyValue1, keyValue2), _, _, _, _, _, _) }
      case _ => null
    }
    putFunction
  }

  def deleteFunction(nature: String, keyValue1: String, keyValue2: String) {
    nature match {
      case "hog" => {
        DynamoDbDecoder.deleteItem(hogsTable, keyValue1)
      }
      case "similar" => { { println("Delete not implemented for SimilarApps.") } }
      case "os" => { { println("Delete not implemented for OS.") } }
      case "model" => { { println("Delete not implemented for models.") } }
      case "result" => { { println("Delete not implemented for jscores.") } }
      case "bug" => { DynamoDbDecoder.deleteItem(bugsTable, keyValue1, keyValue2) }
      case _ => null
    }
  }

  def print(title: String, titleNeg: String, xmax: Double, distWith: Array[(Double, Double)],
    distWithout: Array[(Double, Double)],
    ev: Double, evNeg: Double, evDistance: Double,
    osCorrelations: scala.collection.immutable.Map[String, Double], modelCorrelations: scala.collection.immutable.Map[String, Double], userCorrelations: scala.collection.immutable.Map[String, Double],
    usersWith: Int, usersWithout: Int, uuid: String) {
    println("Calculated %s vs %s xmax=%s ev=%s evWithout=%s evDistance=%s osCorrelations=%s modelCorrelations=%s userCorrelations=%s uuid=%s".format(
      title, titleNeg, xmax, ev, evNeg, evDistance, osCorrelations, modelCorrelations, userCorrelations, uuid))
  }

  /**
   * The J-Score is the % of people with worse = higher energy use.
   * therefore, it is the size of the set of evDistances that are higher than mine,<commit_msg>Enable pretend mode for new analysis.<commit_after>package edu.berkeley.cs.amplab.carat

import spark._
import spark.SparkContext._
import spark.timeseries._
import edu.berkeley.cs.amplab.carat.plot.PlotUtil
import edu.berkeley.cs.amplab.carat.dynamodb.DynamoAnalysisUtil
import scala.collection.immutable.TreeMap
import scala.collection.mutable.HashMap
import scala.collection.mutable.Map
import collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import edu.berkeley.cs.amplab.carat.dynamodb.DynamoDbEncoder
import edu.berkeley.cs.amplab.carat.dynamodb.DynamoDbDecoder

/**
 * Do the exact same thing as in CaratDynamoDataToPlots, but do not collect() and write plot files and run plotting in the end.
 *
 * @author <NAME>
 */

object CaratNoRDDAnalysis {

  // How many clients do we need to consider data reliable?
  val ENOUGH_USERS = 5

  var pretend = false
  val DECIMALS = 3
  var BUCKETS = 100
  // Isolate from the plotting.
  val tmpdir = "/mnt/TimeSeriesSpark-osmodel/spark-temp-plots/"

  var userLimit = Int.MaxValue

  /* Right now we don't remove non-hogs from the bugs table. That should be done.*/
  var bugsToRemove = new ArrayBuffer[(String, String)]
  var hogsToRemove = new ArrayBuffer[(String, String)]

  /**
   * Main program entry point.
   */
  def main(args: Array[String]) {
    var master = "local[16]"
    if (args != null && args.length >= 1) {
      master = args(0)
    }
    if (args != null && args.length > 1 && args(1) == "true")
      pretend = true

    if (args != null && args.length > 2)
      userLimit = args(2).toInt

    val start = DynamoAnalysisUtil.start()
    CaratAnalysisGeneric.genericAnalysis(master, tmpdir, userLimit, ENOUGH_USERS, DECIMALS,
      DynamoAnalysisUtil.removeDaemons, storeDists, storeJScores, globalCorrelations)
  }

  /* Generate a gnuplot-readable plot file of the bucketed distribution.
   * Create folders plots/data plots/plotfiles
   * Save it as "plots/data/titleWith-titleWithout".txt.
   * Also generate a plotfile called plots/plotfiles/titleWith-titleWithout.gnuplot
   */

  def storeDists(nature: String, keyValue1: String, keyValue2: String, title: String, titleNeg: String,
    one: Array[CaratRate], two: Array[CaratRate], aPrioriDistribution: Map[Double, Double], isBugOrHog: Boolean,
    filtered: Array[CaratRate], oses: Set[String], models: Set[String],
    totalsByUuid: scala.collection.immutable.TreeMap[String, (Double, Double)], usersWith: Int, usersWithout: Int, uuid: String) = {
    var hasSamples = true
    if (usersWith == 0 && usersWithout == 0) {
      hasSamples = one.take(1) match {
        case Array(t) => true
        case _ => false
      }
      hasSamples = two.take(1) match {
        case Array(t) => hasSamples && true
        case _ => false
      }
    }
    if (hasSamples) {
      val (xmax, probDist, probDistNeg, ev, evNeg, evDistance /*, usersWith, usersWithout*/ ) = DynamoAnalysisUtil.getDistanceAndDistributionsUnBucketed(one, two, aPrioriDistribution)
      if (probDist != null && probDistNeg != null && (!isBugOrHog || evDistance > 0)) {
        if (evDistance > 0) {
          var imprHr = (100.0 / evNeg - 100.0 / ev) / 3600.0
          val imprD = (imprHr / 24.0).toInt
          imprHr -= imprD * 24.0
          printf("%s evWith=%s evWithout=%s evDistance=%s improvement=%s days %s hours (%s vs %s users)\n", title, ev, evNeg, evDistance, imprD, imprHr, usersWith, usersWithout)
        } else {
          printf("%s evWith=%s evWithout=%s evDistance=%s (%s vs %s users)\n", title, ev, evNeg, evDistance, usersWith, usersWithout)
        }
        val (xmax2, distWith, distWithout) = ProbUtil.bucketDistributionsByX(probDist, probDistNeg, BUCKETS, DECIMALS)

        if (nature == "hog" && evDistance > 0) {
          // Need to delete new hogs from the bugs table:
          val (key, items) = DynamoDbDecoder.filterItems(bugsTable, (hogKey, keyValue1))
          for (k <- items) {
            val attr = k.get(resultKey)
            if (attr != null) {
              val uuid = attr.getS()
              DynamoDbDecoder.deleteItem(bugsTable, uuid, keyValue1)
            }
          }
        }

        val putFunction = storageFunction(nature, keyValue1, keyValue2)
        putFunction(xmax, distWith.toSeq, distWithout.toSeq, evDistance, ev, evNeg)

        // TODO:Store correlations?
        if (isBugOrHog && filtered != null) {
          val (osCorrelations, modelCorrelations, userCorrelations) = DynamoAnalysisUtil.correlation(title, filtered, aPrioriDistribution, models, oses, totalsByUuid)
          print(title, titleNeg, xmax, probDist, probDistNeg, ev, evNeg, evDistance, osCorrelations, modelCorrelations, userCorrelations, usersWith, usersWithout, uuid)
        } else {
          print(title, titleNeg, xmax, probDist, probDistNeg, ev, evNeg, evDistance, null, null, null, usersWith, usersWithout, uuid)
        }
      } else if (isBugOrHog && evDistance <= 0) {
        /* delete old hogs and bugs that are no longer hoggy/buggy */
        deleteFunction(nature, keyValue1, keyValue2)
      }
      isBugOrHog && evDistance > 0
    } else
      false
  }

  def storageFunction(nature: String, keyValue1: String, keyValue2: String, uuidApps: Seq[String] = null) = {
    if (pretend) {
      val putFunction: (Double, Seq[(Int, Double)], Seq[(Int, Double)], Double, Double, Double) => Unit = (xmax, wDist, woDist, evD, ev, evNeg) => {
        println("Pretending to put %s k1=%s k2=%s: xmax=%s ev=%s evN=%s evD=%s wDist=%s woDist=%s".format(
          nature, keyValue1, keyValue2, xmax, ev, evNeg, evD, wDist, woDist))
      }
      putFunction
    } else {

      val putFunction: (Double, Seq[(Int, Double)], Seq[(Int, Double)], Double, Double, Double) => Unit = nature match {
        case "hog" => { DynamoDbEncoder.put(hogsTable, hogKey, keyValue1, _, _, _, _, _, _) }
        case "similar" => { DynamoDbEncoder.put(similarsTable, similarKey, keyValue1, _, _, _, _, _, _) }
        case "os" => { DynamoDbEncoder.put(osTable, osKey, keyValue1, _, _, _, _, _, _) }
        case "model" => { DynamoDbEncoder.put(modelsTable, modelKey, keyValue1, _, _, _, _, _, _) }
        case "result" => { DynamoDbEncoder.put(resultsTable, resultKey, keyValue1, _, _, _, _, _, _, uuidApps) }
        case "bug" => { DynamoDbEncoder.putBug(bugsTable, (resultKey, hogKey), (keyValue1, keyValue2), _, _, _, _, _, _) }
        case _ => null
      }
      putFunction
    }
  }

  def deleteFunction(nature: String, keyValue1: String, keyValue2: String) {
    if (pretend) {
      println("Pretend to delete %s %s %s".format(hogsTable, keyValue1, keyValue2))
    } else {
      nature match {
        case "hog" => {
          DynamoDbDecoder.deleteItem(hogsTable, keyValue1)
        }
        case "similar" => { { println("Delete not implemented for SimilarApps.") } }
        case "os" => { { println("Delete not implemented for OS.") } }
        case "model" => { { println("Delete not implemented for models.") } }
        case "result" => { { println("Delete not implemented for jscores.") } }
        case "bug" => { DynamoDbDecoder.deleteItem(bugsTable, keyValue1, keyValue2) }
        case _ => null
      }
    }
  }

  def print(title: String, titleNeg: String, xmax: Double, distWith: Array[(Double, Double)],
    distWithout: Array[(Double, Double)],
    ev: Double, evNeg: Double, evDistance: Double,
    osCorrelations: scala.collection.immutable.Map[String, Double], modelCorrelations: scala.collection.immutable.Map[String, Double], userCorrelations: scala.collection.immutable.Map[String, Double],
    usersWith: Int, usersWithout: Int, uuid: String) {
    println("Calculated %s vs %s xmax=%s ev=%s evWithout=%s evDistance=%s osCorrelations=%s modelCorrelations=%s userCorrelations=%s uuid=%s".format(
      title, titleNeg, xmax, ev, evNeg, evDistance, osCorrelations, modelCorrelations, userCorrelations, uuid))
  }

  /**
   * The J-Score is the % of people with worse = higher energy use.
   * therefore, it is the size of the set of evDistances that are higher than mine,
{'fraction_non_alphanumeric': 0.12255060466921038, 'fraction_numerical': 0.008536506499385631, 'mean_word_length': 3.294362677034157, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4227244', 'n_tokens_mistral': 5139, 'n_tokens_neox': 4869, 'n_words': 1292}