<commit_before>/* SimpleApp.scala */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf

object SimpleApp {
  def main(args: Array[String]) {
    val logFile = "/Users/pauldoo/Programs/spark-1.1.0/README.md" // Should be some file on your system
    val conf = new SparkConf().setAppName("Simple Application")
    val sc = new SparkContext(conf)
    val logData = sc.textFile(logFile, 2).cache()
    val numAs = logData.filter(line => line.contains("a")).count()
    val numBs = logData.filter(line => line.contains("b")).count()
    println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
  }
}<commit_msg>Messing with Apache spark.<commit_after>/* SimpleApp.scala */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf

object SimpleApp {
  def main(args: Array[String]) {
    val logFile = "/Users/pauldoo/Programs/spark-1.1.0/README.md" // Should be some file on your system
    val conf = new SparkConf().setMaster("spark://jam.lan:7077").setAppName("Simple Application")
    val sc = new SparkContext(conf)
    val logData = sc.textFile(logFile, 2).cache()
    val numAs = logData.filter(line => line.contains("a")).count()
    val numBs = logData.filter(line => line.contains("b")).count()
    println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))

    val wordCountsData = logData //
      .flatMap(_.split(" ")) //
      .map(w => (w, 1)) //
      .reduceByKey(_ + _)
    wordCountsData.cache()
    val wordCounts: List[(String, Int)] = wordCountsData.collect.toList
    println(s"Wordcounts: $wordCounts")

    val averageLengthData: (Int, Int) = wordCountsData //
      .map(t => ((t._1.length() * t._2), t._2)) //
      .reduce((a, b) => ((a._1 + b._1), (a._2 + b._2)))
    val averageLength: Double = averageLengthData._1.toDouble / averageLengthData._2;
    println(s"Average length: $averageLength")
  }
}
{'fraction_non_alphanumeric': 0.16216216216216217, 'fraction_numerical': 0.011218765935747067, 'mean_word_length': 3.6273584905660377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4814601', 'n_tokens_mistral': 700, 'n_tokens_neox': 662, 'n_words': 148}