<commit_before>package robco

import org.apache.spark.{SparkContext, SparkConf}

object SparkSql2 {
  // case class Person(firstName: String, lastName: String, gender: String)
  case class AvcInfo(avcId: String, cvcId: String)
  case class AvcData(avcId: String, octetsIn: String, octetsOut: String)

  def main(args: Array[String]) = {
    val sparkMaster = "spark://" + args(0) + ":7077"
    val hdfsRoot = "hdfs://" + args(1) + ":9000"
    val startTime = System.currentTimeMillis()/1000
    val conf =
        new SparkConf().setMaster(sparkMaster)
            .set("spark.driver.memory", "6G")
            .set("spark.storage.memoryFraction", "0.1")
            .set("spark.executor.memory", "6G")
    val dataset = args(2)
    val sc = new SparkContext(conf)
    sys.ShutdownHookThread { sc.stop() }
    val hc = new org.apache.spark.sql.hive.HiveContext(sc)
    import hc.implicits._
    val avcsRDD = sc.textFile(s"$hdfsRoot/data-schema-$dataset-avcInfo.csv").map(_.split(",")).map(a => AvcInfo(a(0),a(1)))
    val avcs = avcsRDD.toDF
    avcs.registerTempTable("avcs")
    val avcDataRDD = sc.textFile(s"$hdfsRoot/data-schema-$dataset-avcData.csv").map(_.split(",")).map(a => AvcData(a(0),a(1),a(2)))
    val avcData = avcDataRDD.toDF
    avcData.registerTempTable("avcData")
    val dataLoadTime = (System.currentTimeMillis()/1000) - startTime
    println("loading data time: " + dataLoadTime)
    val startTime2 = System.currentTimeMillis()/1000
    val rows = hc.sql("""
        SELECT a.*, ad.*
        FROM avcs a, avcData ad
        WHERE a.avcId = ad.avcId
    """)
    val results = rows.collect
    val queryTime = (System.currentTimeMillis()/1000) - startTime2
    println("query time: " + queryTime)
    println("number of results: " + results.length)
    println("total time taken: " + (dataLoadTime + queryTime))
  }
}<commit_msg>raise spark.storage.memoryFraction to 0.6<commit_after>package robco

import org.apache.spark.{SparkContext, SparkConf}

object SparkSql2 {
  // case class Person(firstName: String, lastName: String, gender: String)
  case class AvcInfo(avcId: String, cvcId: String)
  case class AvcData(avcId: String, octetsIn: String, octetsOut: String)

  def main(args: Array[String]) = {
    val sparkMaster = "spark://" + args(0) + ":7077"
    val hdfsRoot = "hdfs://" + args(1) + ":9000"
    val startTime = System.currentTimeMillis()/1000
    val conf =
        new SparkConf().setMaster(sparkMaster)
            .set("spark.driver.memory", "6G")
            .set("spark.storage.memoryFraction", "0.6")
            .set("spark.executor.memory", "6G")
    val dataset = args(2)
    val sc = new SparkContext(conf)
    sys.ShutdownHookThread { sc.stop() }
    val hc = new org.apache.spark.sql.hive.HiveContext(sc)
    import hc.implicits._
    val avcsRDD = sc.textFile(s"$hdfsRoot/data-schema-$dataset-avcInfo.csv").map(_.split(",")).map(a => AvcInfo(a(0),a(1)))
    val avcs = avcsRDD.toDF
    avcs.registerTempTable("avcs")
    val avcDataRDD = sc.textFile(s"$hdfsRoot/data-schema-$dataset-avcData.csv").map(_.split(",")).map(a => AvcData(a(0),a(1),a(2)))
    val avcData = avcDataRDD.toDF
    avcData.registerTempTable("avcData")
    val dataLoadTime = (System.currentTimeMillis()/1000) - startTime
    println("loading data time: " + dataLoadTime)
    val startTime2 = System.currentTimeMillis()/1000
    val rows = hc.sql("""
        SELECT a.*, ad.*
        FROM avcs a, avcData ad
        WHERE a.avcId = ad.avcId
    """)
    val results = rows.collect
    val queryTime = (System.currentTimeMillis()/1000) - startTime2
    println("query time: " + queryTime)
    println("number of results: " + results.length)
    println("total time taken: " + (dataLoadTime + queryTime))
  }
}
{'fraction_non_alphanumeric': 0.1535369774919614, 'fraction_numerical': 0.021436227224008574, 'mean_word_length': 3.396937573616019, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '804953', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1314, 'n_words': 263}