<commit_before>package de.hpi.ingestion.textmining

import com.holdenkarau.spark.testing.SharedSparkContext
import org.scalatest.{FlatSpec, Matchers}
import de.hpi.ingestion.implicits.CollectionImplicits._
import org.apache.spark.mllib.classification.NaiveBayesModel

class ClassifierTrainingTest extends FlatSpec with Matchers with SharedSparkContext {

	/*"Run" should "return a Naive Bayes model" in {
		val input = List(sc.parallelize(TestData.classifierFeatureEntries())).toAnyRDD()
		val modelList = ClassifierTraining.run(input, sc).fromAnyRDD[NaiveBayesModel]().head.collect.toList
		modelList should not be empty
	}*/	// failing on jenkins?

	"Statistics" should "be calculated" in {
		val data = sc.parallelize(TestData.labeledPredictions())
		val statistics = ClassifierTraining.calculateStatistics(data, 0.5)
		val expectedStats = TestData.predictionStatistics()
		statistics shouldEqual expectedStats
	}

	they should "be formatted" in {
		val statistics = TestData.predictionStatistics()
		val formattedStats = ClassifierTraining.formatStatistics(statistics)
		val expectedString = TestData.formattedPredictionStatistics()
		formattedStats shouldEqual expectedString
	}
}<commit_msg>#179: Undo commenting out test (hoping it wont fail again)<commit_after>package de.hpi.ingestion.textmining

import com.holdenkarau.spark.testing.SharedSparkContext
import org.scalatest.{FlatSpec, Matchers}
import de.hpi.ingestion.implicits.CollectionImplicits._
import org.apache.spark.mllib.classification.NaiveBayesModel

class ClassifierTrainingTest extends FlatSpec with Matchers with SharedSparkContext {

	"Run" should "return a Naive Bayes model" in {
		val input = List(sc.parallelize(TestData.classifierFeatureEntries())).toAnyRDD()
		val modelList = ClassifierTraining.run(input, sc).fromAnyRDD[NaiveBayesModel]().head.collect.toList
		modelList should not be empty
	}

	"Statistics" should "be calculated" in {
		val data = sc.parallelize(TestData.labeledPredictions())
		val statistics = ClassifierTraining.calculateStatistics(data, 0.5)
		val expectedStats = TestData.predictionStatistics()
		statistics shouldEqual expectedStats
	}

	they should "be formatted" in {
		val statistics = TestData.predictionStatistics()
		val formattedStats = ClassifierTraining.formatStatistics(statistics)
		val expectedString = TestData.formattedPredictionStatistics()
		formattedStats shouldEqual expectedString
	}
}
{'fraction_non_alphanumeric': 0.08499170812603649, 'fraction_numerical': 0.002902155887230514, 'mean_word_length': 6.268072289156627, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4556952', 'n_tokens_mistral': 764, 'n_tokens_neox': 697, 'n_words': 160}