<commit_before>package com.aluxian.tweeather.scripts

import com.aluxian.tweeather.transformers.{ColumnDropper, FeatureReducer, StringSanitizer}
import org.apache.spark.Logging
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.classification.NaiveBayes
import org.apache.spark.ml.feature.{HashingTF, StopWordsRemover, Tokenizer}
import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics
import org.apache.spark.sql.Row

import scala.io.StdIn

/**
  * This script trains a Naive Bayes classifier with the Sentiment140 dataset of tweets.
  * For training, it uses the dataset of 1.6M tweets. For testing, it uses the manually labelled dataset.
  *
  * Before running this script, the dataset must be first downloaded with [[Sentiment140Downloader]] and then parsed
  * with [[Sentiment140Parser]]. After the model is created, it can be tested with [[Sentiment140Repl]].
  *
  * The resulting model has an accuracy of 81%.
  */
object Sentiment140Trainer extends Script with Logging {

  override def main(args: Array[String]) {
    super.main(args)

    // Prepare data sets
    logInfo("Getting datasets")
    val testData = sqlc.read.parquet("/tw/sentiment/140/parsed/test.parquet")
    val trainingData = sqlc.read.parquet("/tw/sentiment/140/parsed/training.parquet")

    // Configure the pipeline
    val pipeline = new Pipeline().setStages(Array(
      new FeatureReducer().setInputCol("raw_text").setOutputCol("reduced_text"),
      new StringSanitizer().setInputCol("reduced_text").setOutputCol("text"),
      new Tokenizer().setInputCol("text").setOutputCol("raw_words"),
      new StopWordsRemover().setInputCol("raw_words").setOutputCol("words"),
      new HashingTF().setInputCol("words").setOutputCol("features"),
      new NaiveBayes().setSmoothing(0.5).setFeaturesCol("features"),
      new ColumnDropper().setDropColumns("raw_text", "reduced_text", "text", "raw_words", "words", "features")
    ))

    // Fit the pipeline
    logInfo("Training model")
    val model = pipeline.fit(trainingData)<commit_msg>Updates measured accuracy in Sentiment140Trainer docs<commit_after>package com.aluxian.tweeather.scripts

import com.aluxian.tweeather.transformers.{ColumnDropper, FeatureReducer, StringSanitizer}
import org.apache.spark.Logging
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.classification.NaiveBayes
import org.apache.spark.ml.feature.{HashingTF, StopWordsRemover, Tokenizer}
import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics
import org.apache.spark.sql.Row

import scala.io.StdIn

/**
  * This script trains a Naive Bayes classifier with the Sentiment140 dataset of tweets.
  * For training, it uses the dataset of 1.6M tweets. For testing, it uses the manually labelled dataset.
  *
  * Before running this script, the dataset must be first downloaded with [[Sentiment140Downloader]] and then parsed
  * with [[Sentiment140Parser]]. After the model is created, it can be tested with [[Sentiment140Repl]].
  *
  * The resulting model has an accuracy of 80%.
  */
object Sentiment140Trainer extends Script with Logging {

  override def main(args: Array[String]) {
    super.main(args)

    // Prepare data sets
    logInfo("Getting datasets")
    val testData = sqlc.read.parquet("/tw/sentiment/140/parsed/test.parquet")
    val trainingData = sqlc.read.parquet("/tw/sentiment/140/parsed/training.parquet")

    // Configure the pipeline
    val pipeline = new Pipeline().setStages(Array(
      new FeatureReducer().setInputCol("raw_text").setOutputCol("reduced_text"),
      new StringSanitizer().setInputCol("reduced_text").setOutputCol("text"),
      new Tokenizer().setInputCol("text").setOutputCol("raw_words"),
      new StopWordsRemover().setInputCol("raw_words").setOutputCol("words"),
      new HashingTF().setInputCol("words").setOutputCol("features"),
      new NaiveBayes().setSmoothing(0.5).setFeaturesCol("features"),
      new ColumnDropper().setDropColumns("raw_text", "reduced_text", "text", "raw_words", "words", "features")
    ))

    // Fit the pipeline
    logInfo("Training model")
    val model = pipeline.fit(trainingData)
{'fraction_non_alphanumeric': 0.11767554479418886, 'fraction_numerical': 0.013801452784503633, 'mean_word_length': 4.7534818941504176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2277951', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1217, 'n_words': 292}