<commit_before>package de.hpi.ingestion.textmining

import de.hpi.ingestion.textmining.models.{DocumentFrequency, ParsedWikipediaEntry}
import org.apache.spark.{SparkConf, SparkContext}
import com.datastax.spark.connector._
import org.apache.spark.rdd.RDD

/**
  * Calculates document frequencies over all articles.
  */
object DocumentFrequencyCounter {
	val keyspace = "wikidumps"
	val inputArticlesTablename = "parsedwikipedia"
	val outputDocumentFrequenciesTablename = "wikipediadocfreq"
	val removeStopwords = true
	val stem = true
	val leastSignificantDocumentFrequency = 5

	/**
	  * Counts document frequencies while stemming and removing stopwords
	  *
	  * @param articles all parsed Wikipedia entries
	  * @return RDD of document frequencies
	  */
	def countDocumentFrequencies(
		articles: RDD[ParsedWikipediaEntry]
	): RDD[DocumentFrequency] = {
		val tokenizer = IngestionTokenizer(removeStopwords, stem)
		articles
			.flatMap(article => tokenizer.process(article.getText()).toSet)
			.map(word => (word, 1))
			.reduceByKey(_ + _)
			.map(DocumentFrequency.tupled)
	}

	/**
	  * Filters all document frequencies that do not meet the threshold
	  *
	  * @param documentFrequencies document frequencies to be filtered
	  * @param threshold           minimum amount of occurrences in documents
	  * @return filtered document frequencies
	  */
	def filterDocumentFrequencies(
		documentFrequencies: RDD[DocumentFrequency],
		threshold: Int
	): RDD[DocumentFrequency] = {
		documentFrequencies
			.filter(documentFrequency => documentFrequency.count >= threshold)
	}

	def main(args: Array[String]): Unit = {
		val conf = new SparkConf().setAppName("Document Frequency Counter")

		val sc = new SparkContext(conf)
		val allArticles = sc.cassandraTable[ParsedWikipediaEntry](keyspace, inputArticlesTablename)
		val documentFrequencies = countDocumentFrequencies(allArticles)
		filterDocumentFrequencies(documentFrequencies, leastSignificantDocumentFrequency)<commit_msg>:pencil2: Add dots to method descriptions<commit_after>package de.hpi.ingestion.textmining

import de.hpi.ingestion.textmining.models.{DocumentFrequency, ParsedWikipediaEntry}
import org.apache.spark.{SparkConf, SparkContext}
import com.datastax.spark.connector._
import org.apache.spark.rdd.RDD

/**
  * Counts document frequencies over all articles.
  */
object DocumentFrequencyCounter {
	val keyspace = "wikidumps"
	val inputArticlesTablename = "parsedwikipedia"
	val outputDocumentFrequenciesTablename = "wikipediadocfreq"
	val removeStopwords = true
	val stem = true
	val leastSignificantDocumentFrequency = 5

	/**
	  * Counts document frequencies while stemming and removing stopwords.
	  *
	  * @param articles all parsed Wikipedia entries
	  * @return RDD of document frequencies
	  */
	def countDocumentFrequencies(
		articles: RDD[ParsedWikipediaEntry]
	): RDD[DocumentFrequency] = {
		val tokenizer = IngestionTokenizer(removeStopwords, stem)
		articles
			.flatMap(article => tokenizer.process(article.getText()).toSet)
			.map(word => (word, 1))
			.reduceByKey(_ + _)
			.map(DocumentFrequency.tupled)
	}

	/**
	  * Filters all document frequencies that do not meet the threshold.
	  *
	  * @param documentFrequencies document frequencies to be filtered
	  * @param threshold           minimum amount of occurrences in documents
	  * @return filtered document frequencies
	  */
	def filterDocumentFrequencies(
		documentFrequencies: RDD[DocumentFrequency],
		threshold: Int
	): RDD[DocumentFrequency] = {
		documentFrequencies
			.filter(documentFrequency => documentFrequency.count >= threshold)
	}

	def main(args: Array[String]): Unit = {
		val conf = new SparkConf().setAppName("Document Frequency Counter")
		val sc = new SparkContext(conf)

		val allArticles = sc.cassandraTable[ParsedWikipediaEntry](keyspace, inputArticlesTablename)
		val documentFrequencies = countDocumentFrequencies(allArticles)
		filterDocumentFrequencies(documentFrequencies, leastSignificantDocumentFrequency)
{'fraction_non_alphanumeric': 0.08529854490717512, 'fraction_numerical': 0.001254390366281987, 'mean_word_length': 5.0963302752293576, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5327429', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1125, 'n_words': 278}