<commit_before>package com.houseofmoran.spark.play.twitter

import com.houseofmoran.spark.play.twitter.LoadHelpers._
import org.apache.spark.SparkContext._
import org.apache.spark._
import org.apache.spark.sql.{Row, SQLContext}

object AnalyseTwitterEmojiApp {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("AnalyseTwitterEmojiApp").setMaster("local[*]")
    val sc = new SparkContext(conf)
    val sqlSc = new SQLContext(sc)

    val emojiCounts = sqlSc.parquetFiles("emojiusage")
    emojiCounts.printSchema()

    val mapped = emojiCounts.map((row) => {
      val wordEmojiPairStruct = row.getAs[Row](2)
      val count = row.getLong(3)

      ((wordEmojiPairStruct.getString(0), Emoji(wordEmojiPairStruct.getAs[Row](1).getString(0))), count)
    })
    val rolledUp = mapped.reduceByKey(_ + _).sortBy({ case (_, count) => count }, false)

    for(entry <- rolledUp.sortBy({ case (_, count) => count }, false).take(10)) {
      println(entry)
    }

    val wordCounts = rolledUp.map{ case ((word, emoji), count) => (word, count)}.reduceByKey(_ + _)

    for(entry <- wordCounts.sortBy({ case (_, count) => count }, false).take(10)) {
      println(entry)
    }

    val wordFirst = rolledUp.map{ case ((word, emoji), count) => (word, (emoji, count))}
    val joined = wordCounts.join(wordFirst)

    for(entry <- joined.takeSample(false, 10)) {
      println(entry)
    }

    val inContext = joined.map{ case (word, (overallCount, (emoji, emojiCount))) => {
      ((word, emoji), (emojiCount, overallCount))
    }}

    for(entry <- inContext.sortBy({ case (_, (e, o)) => e.toFloat / o.toFloat }, false).take(10)) {
      println(entry)
    }

    val withoutLowValues = inContext.filter{ case (_, (emojiCount, overallCount)) => {
      emojiCount > 10 && overallCount > 10
    }}

    for(entry <- withoutLowValues.sortBy({ case (_, (e, o)) => e.toFloat / o.toFloat }, false).take(10)) {
      println(entry)
    }

  }
}<commit_msg>also show pairs that where word does not contain emoji itself<commit_after>package com.houseofmoran.spark.play.twitter

import com.houseofmoran.spark.play.twitter.LoadHelpers._
import org.apache.spark.SparkContext._
import org.apache.spark._
import org.apache.spark.sql.{Row, SQLContext}

object AnalyseTwitterEmojiApp {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("AnalyseTwitterEmojiApp").setMaster("local[*]")
    val sc = new SparkContext(conf)
    val sqlSc = new SQLContext(sc)

    val emojiCounts = sqlSc.parquetFiles("emojiusage")
    emojiCounts.printSchema()

    val mapped = emojiCounts.map((row) => {
      val wordEmojiPairStruct = row.getAs[Row](2)
      val count = row.getLong(3)

      ((wordEmojiPairStruct.getString(0), Emoji(wordEmojiPairStruct.getAs[Row](1).getString(0))), count)
    })
    val rolledUp = mapped.reduceByKey(_ + _).sortBy({ case (_, count) => count }, false)

    for(entry <- rolledUp.sortBy({ case (_, count) => count }, false).take(10)) {
      println(entry)
    }

    val wordCounts = rolledUp.map{ case ((word, emoji), count) => (word, count)}.reduceByKey(_ + _)

    for(entry <- wordCounts.sortBy({ case (_, count) => count }, false).take(10)) {
      println(entry)
    }

    val wordFirst = rolledUp.map{ case ((word, emoji), count) => (word, (emoji, count))}
    val joined = wordCounts.join(wordFirst)

    for(entry <- joined.takeSample(false, 10)) {
      println(entry)
    }

    val inContext = joined.map{ case (word, (overallCount, (emoji, emojiCount))) => {
      ((word, emoji), (emojiCount, overallCount))
    }}

    for(entry <- inContext.sortBy({ case (_, (e, o)) => e.toFloat / o.toFloat }, false).take(10)) {
      println(entry)
    }

    val withoutLowValues = inContext.filter{ case (_, (emojiCount, overallCount)) => {
      emojiCount > 10 && overallCount > 10
    }}

    for(entry <- withoutLowValues.sortBy({ case (_, (e, o)) => e.toFloat / o.toFloat }, false).take(10)) {
      println(entry)
    }

    val withoutEmojiInWords = withoutLowValues.filter{ case ((word, _), _) => {
      Emoji.findEmoji(word).isEmpty
    }}

    for(entry <- withoutEmojiInWords.sortBy({ case (_, (e, o)) => e.toFloat / o.toFloat }, false).take(10)) {
      println(entry)
    }
  }
}
{'fraction_non_alphanumeric': 0.16138869340839784, 'fraction_numerical': 0.009383063570255689, 'mean_word_length': 3.474291710388248, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2656885', 'n_tokens_mistral': 1516, 'n_tokens_neox': 1449, 'n_words': 291}