<commit_before>package com.aluxian.tweeather.scripts

import org.apache.spark.Logging
import org.apache.spark.ml.ann.MultilayerPerceptron
import org.apache.spark.ml.evaluation.RegressionEvaluator
import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.sql.Row

object TwitterHoseFireTrainer extends Script with Logging {

  override def main(args: Array[String]) {
    super.main(args)
    import sqlc.implicits._

    // Prepare data sets
    logInfo("Getting datasets")
    val Array(trainingData, testData) = sqlc.read.parquet("/tw/fire/parsed/data.parquet")
      .map(row => {
        val parts = row.getString(0).split(',').map(_.toDouble)
        val outputs = Vectors.dense(parts.head)
        val inputs = Vectors.dense(parts.tail)
        (inputs, outputs)
      })
      .toDF("input", "output")
      .randomSplit(Array(0.9, 0.1))

    // Configure the perceptron
    val perceptron = new MultilayerPerceptron()
      .setLayers(Array(3, 5, 1))
      .setInputCol("input")
      .setOutputCol("output")

    // Train the perceptron
    logInfo("Training model")
    val model = perceptron.fit(trainingData)

    // Test the model precision
    logInfo("Testing model")
    val predicted = model
      .setOutputCol("predicted")
      .transform(testData)
      .select("output", "predicted")
      .map({ case Row(output: Vector, predictedOutput: Vector) =>
        (output.toArray.head, predictedOutput.toArray.head)
      })
      .toDF("label", "prediction")

    // The output layer has only 1 value, use a simple RegressionEvaluator
    val accuracy = new RegressionEvaluator().evaluate(predicted)
    logInfo(s"Accuracy: $accuracy")<commit_msg>Improves formatting in fire trainer<commit_after>package com.aluxian.tweeather.scripts

import org.apache.spark.Logging
import org.apache.spark.ml.ann.MultilayerPerceptron
import org.apache.spark.ml.evaluation.RegressionEvaluator
import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.sql.Row

object TwitterHoseFireTrainer extends Script with Logging {

  override def main(args: Array[String]) {
    super.main(args)
    import sqlc.implicits._

    // Prepare data sets
    logInfo("Getting datasets")
    val Array(trainingData, testData) = sqlc.read.parquet("/tw/fire/parsed/data.parquet")
      .map(row => {
        val parts = row.getString(0).split(',').map(_.toDouble)
        val outputs = Vectors.dense(parts.head)
        val inputs = Vectors.dense(parts.tail)
        (inputs, outputs)
      })
      .toDF("input", "output")
      .randomSplit(Array(0.9, 0.1))

    // Configure the perceptron
    val perceptron = new MultilayerPerceptron()
      .setLayers(Array(3, 5, 1))
      .setInputCol("input")
      .setOutputCol("output")

    // Train the perceptron
    logInfo("Training model")
    val model = perceptron.fit(trainingData)

    // Test the model precision
    logInfo("Testing model")
    val predicted = model
      .setOutputCol("predicted")
      .transform(testData)
      .select("output", "predicted")
      .map {
        case Row(output: Vector, predicted: Vector) =>
          (output.toArray.head, predicted.toArray.head)
      }
      .toDF("label", "prediction")

    // The output layer has only 1 value, use a simple RegressionEvaluator
    val accuracy = new RegressionEvaluator().evaluate(predicted)
    logInfo(s"Accuracy: $accuracy")
{'fraction_non_alphanumeric': 0.12393036293892004, 'fraction_numerical': 0.005311301268810859, 'mean_word_length': 3.216417910447761, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3142480', 'n_tokens_mistral': 1120, 'n_tokens_neox': 1064, 'n_words': 225}