<commit_before>package com.aluxian.tweeather.scripts

import com.aluxian.tweeather.RichBoolean
import org.apache.spark.Logging
import org.apache.spark.sql.SaveMode

/**
  * This script parses the tweets collected by [[TwitterEmoCollector]].
  * It removes duplicates and tweets which contain both positive and negative emojis.
  * The resulting dataset is coalesced to reduce the number of partitions.
  */
object TwitterEmoParser extends Script with Logging {

  val positiveEmoticons = TwitterEmoCollector.positiveEmoticons
  val negativeEmoticons = TwitterEmoCollector.negativeEmoticons
  int count = 0

  override def main(args: Array[String]) {
    super.main(args)
    import sqlc.implicits._

    // Import data
    logInfo("Parsing text files")<commit_msg>Added counting variable<commit_after>package com.aluxian.tweeather.scripts

import com.aluxian.tweeather.RichBoolean
import org.apache.spark.Logging
import org.apache.spark.sql.SaveMode

/**
  * This script parses the tweets collected by [[TwitterEmoCollector]].
  * It removes duplicates and tweets which contain both positive and negative emojis.
  * The resulting dataset is coalesced to reduce the number of partitions.
  */
object TwitterEmoParser extends Script with Logging {

  val positiveEmoticons = TwitterEmoCollector.positiveEmoticons
  val negativeEmoticons = TwitterEmoCollector.negativeEmoticons
  var count = 0

  override def main(args: Array[String]) {
    super.main(args)
    import sqlc.implicits._

    // Import data
    logInfo("Parsing text files")
{'fraction_non_alphanumeric': 0.07022106631989597, 'fraction_numerical': 0.0013003901170351106, 'mean_word_length': 4.516129032258065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4627595', 'n_tokens_mistral': 469, 'n_tokens_neox': 438, 'n_words': 133}