<commit_before>package edu.berkeley.cs.amplab.carat.dynamodb

import com.amazonaws.services.dynamodb.AmazonDynamoDBClient
import com.amazonaws.services.dynamodb.model.CreateTableRequest
import com.amazonaws.services.dynamodb.model.DescribeTableRequest
import com.amazonaws.services.dynamodb.model.KeySchemaElement
import com.amazonaws.services.dynamodb.model.KeySchema
import com.amazonaws.services.dynamodb.model.ProvisionedThroughput
import com.amazonaws.services.dynamodb.model.PutItemRequest
import com.amazonaws.services.dynamodb.model.AttributeValue
import com.amazonaws.services.dynamodb.model.DeleteTableRequest
import collection.JavaConversions._
import java.util.HashSet
import com.amazonaws.services.dynamodb.model.UpdateTableRequest
import edu.berkeley.cs.amplab.carat.s3.S3Encoder

object DynamoDbEncoder {
  val dd = new AmazonDynamoDBClient(S3Encoder.cred)

  // For putting data:
  val prob = "prob"
  val probNeg = "probNeg"
  val xmax = "xmax"
  val distanceField = "distance"
  val apps = "reportedApps"

  /**
   * Put a new entry into a table that uses only a HashKey.
   * For `bugsTable`, use `putBug()`.
   */
  def put(table: String, keyName: String, keyValue: String,
    maxX: Double,
    prob1: Seq[(Int, Double)], prob2: Seq[(Int, Double)],
    distance: Double,
    ev:Double, evNeg: Double,
    uuidApps: Seq[String] = new HashSet[String].toSeq) {
    if (uuidApps.size > 0)
      put(table, (keyName, keyValue), (xmax, maxX),
        (prob, prob1.map(x => { x._1 + ";" + x._2 })),
        (probNeg, prob2.map(x => { x._1 + ";" + x._2 })),
        (distanceField, distance),
        (expectedValue, ev),
        (expectedValueNeg, evNeg),
        (apps, uuidApps))
    else
      put(table, (keyName, keyValue), (xmax, maxX),
        (prob, prob1.map(x => { x._1 + ";" + x._2 })),
        (probNeg, prob2.map(x => { x._1 + ";" + x._2 })),
        (distanceField, distance),
        (expectedValue, ev),
        (expectedValueNeg, evNeg))
  }

  /**
   * Function that makes it easy to use the ridiculous DynamoDB put API.
   */
  def put(table: String, vals: (String, Any)*) = {
    val map = getMap(vals: _*)
    println("Going to put into " + table + ":\n" + map.mkString("\n"))
    val putReq = new PutItemRequest(table, map)
    dd.putItem(putReq)
  }

  
  /**
   * Function for putting already proper Maps into DynamoDb.
   */
  def putItems(table: String, stuff: java.util.List[java.util.Map[String, AttributeValue]]) = {
    for (map <- stuff) {
      println("Going to put into " + table + ":\n" + map.mkString("\n"))
      val putReq = new PutItemRequest(table, map)
      dd.putItem(putReq)
    }
  }
  /**
   * Function for putting a ready Map into DynamoDb.
   */
  def putItem(table: String, map: java.util.Map[String, AttributeValue]) = {
    println("Going to put into " + table + ":\n" + map.mkString("\n"))
    val putReq = new PutItemRequest(table, map)
    dd.putItem(putReq)
  }

  /**
   * Helper function used by put(table, vals).
   * Constructs Maps to be put into a table from a variable number of (String, Any) - pairs.
   */
  def getMap(vals: (String, Any)*) = {
    val map:java.util.Map[String, AttributeValue] = new java.util.HashMap[String, AttributeValue]()
    for (k <- vals) {
        map.put(k._1, toAttributeValue(k._2))
    }
    map
  }

  def toAttributeValue(thing: Any) = {
    if (thing.isInstanceOf[Double]
      || thing.isInstanceOf[Int]
      || thing.isInstanceOf[Long]
      || thing.isInstanceOf[Float]
      || thing.isInstanceOf[Short])<commit_msg>Try to make put safe so that records would not be lost in case analysis is too fast.<commit_after>package edu.berkeley.cs.amplab.carat.dynamodb

import com.amazonaws.services.dynamodb.AmazonDynamoDBClient
import com.amazonaws.services.dynamodb.model.CreateTableRequest
import com.amazonaws.services.dynamodb.model.DescribeTableRequest
import com.amazonaws.services.dynamodb.model.KeySchemaElement
import com.amazonaws.services.dynamodb.model.KeySchema
import com.amazonaws.services.dynamodb.model.ProvisionedThroughput
import com.amazonaws.services.dynamodb.model.PutItemRequest
import com.amazonaws.services.dynamodb.model.AttributeValue
import com.amazonaws.services.dynamodb.model.DeleteTableRequest
import collection.JavaConversions._
import java.util.HashSet
import com.amazonaws.services.dynamodb.model.UpdateTableRequest
import edu.berkeley.cs.amplab.carat.s3.S3Encoder

object DynamoDbEncoder {
  val dd = new AmazonDynamoDBClient(S3Encoder.cred)

  // For putting data:
  val prob = "prob"
  val probNeg = "probNeg"
  val xmax = "xmax"
  val distanceField = "distance"
  val apps = "reportedApps"

  /**
   * Put a new entry into a table that uses only a HashKey.
   * For `bugsTable`, use `putBug()`.
   */
  def put(table: String, keyName: String, keyValue: String,
    maxX: Double,
    prob1: Seq[(Int, Double)], prob2: Seq[(Int, Double)],
    distance: Double,
    ev:Double, evNeg: Double,
    uuidApps: Seq[String] = new HashSet[String].toSeq) {
    if (uuidApps.size > 0)
      put(table, (keyName, keyValue), (xmax, maxX),
        (prob, prob1.map(x => { x._1 + ";" + x._2 })),
        (probNeg, prob2.map(x => { x._1 + ";" + x._2 })),
        (distanceField, distance),
        (expectedValue, ev),
        (expectedValueNeg, evNeg),
        (apps, uuidApps))
    else
      put(table, (keyName, keyValue), (xmax, maxX),
        (prob, prob1.map(x => { x._1 + ";" + x._2 })),
        (probNeg, prob2.map(x => { x._1 + ";" + x._2 })),
        (distanceField, distance),
        (expectedValue, ev),
        (expectedValueNeg, evNeg))
  }

  /**
   * Function that makes it easy to use the ridiculous DynamoDB put API.
   */
  def put(table: String, vals: (String, Any)*) = {
    val map = getMap(vals: _*)
    println("Going to put into " + table + ":\n" + map.mkString("\n"))
    val putReq = new PutItemRequest(table, map)
    safePut(putReq)
  }

  
  /**
   * Function for putting already proper Maps into DynamoDb.
   */
  def putItems(table: String, stuff: java.util.List[java.util.Map[String, AttributeValue]]) = {
    for (map <- stuff) {
      println("Going to put into " + table + ":\n" + map.mkString("\n"))
      val putReq = new PutItemRequest(table, map)
      safePut(putReq)
    }
  }
  /**
   * Function for putting a ready Map into DynamoDb.
   */
  def putItem(table: String, map: java.util.Map[String, AttributeValue]) = {
    println("Going to put into " + table + ":\n" + map.mkString("\n"))
    val putReq = new PutItemRequest(table, map)
    safePut(putReq)
  }
  
  // max allowed write caps per sec
  val capMax = 80.0
  // write caps used
  var capUsedPerSec = 0.0
  // time write caps were used in
  var timeSpent = 0L
  // last request write caps used
  var lastCap = 0.0
  
  
  def safePut(req: PutItemRequest) = {
    val now = System.currentTimeMillis
    if (timeSpent == 0){
      timeSpent = now
    }else
      timeSpent = now - timeSpent
    // if LastCap is bigger than capMax, this whole thing makes no sense.
    if (lastCap > capMax)
      println("Error: lastCap=%s is greater than max=%s".format(lastCap, capMax))
    else {
      if (timeSpent < 1000 && capUsedPerSec + lastCap > capMax) {
        wait(1000 - timeSpent)
        timeSpent = now
        capUsedPerSec = 0
      }
    }
    val res = dd.putItem(req)
    lastCap = res.getConsumedCapacityUnits()
    capUsedPerSec += lastCap
    println("Put used %s cap, %s cap left".format(lastCap, (capMax - capUsedPerSec)))
  }

  /**
   * Helper function used by put(table, vals).
   * Constructs Maps to be put into a table from a variable number of (String, Any) - pairs.
   */
  def getMap(vals: (String, Any)*) = {
    val map:java.util.Map[String, AttributeValue] = new java.util.HashMap[String, AttributeValue]()
    for (k <- vals) {
        map.put(k._1, toAttributeValue(k._2))
    }
    map
  }

  def toAttributeValue(thing: Any) = {
    if (thing.isInstanceOf[Double]
      || thing.isInstanceOf[Int]
      || thing.isInstanceOf[Long]
      || thing.isInstanceOf[Float]
      || thing.isInstanceOf[Short])
{'fraction_non_alphanumeric': 0.1348871807587404, 'fraction_numerical': 0.007190676915447558, 'mean_word_length': 3.1369230769230767, 'pattern_counts': {'":': 6, '<': 8, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '677623', 'n_tokens_mistral': 2789, 'n_tokens_neox': 2714, 'n_words': 662}