<commit_before>package ml.wolfe.ui.statnlp

import java.io.{File, PrintStream}

import ml.wolfe.nlp.io.IO
import ml.wolfe.ui.MutableMoroNotebook
import ml.wolfe.util.Util

import scala.io.Source

/**
 * @author <NAME>
 */
/**
 * @author <NAME>
 */
object Smoothing extends MutableMoroNotebook with App {

  def mygreen = "#78AB46"

  def span(color: String)(body: String) = s"""<span style="color: $color">$body</span>"""

  section("Zipf") {

    md(
      s"""
        |## Zipf's Law
        |The frequency ${span(mygreen)("$f$")} of a word is inversely proportional to its rank ${span(mygreen)("$r$")}.
      """.stripMargin)

    latex("f(k; s, N) = \\frac{1 / k^s}{ \\sum_i i}")
  }

  section("ZipfGraph") {

    md("##Counts from ACL")

    wolfe(
      """
        |import scala.io.Source
        |val file = "/Users/sriedel/projects/stat-nlp-course-intern/data/counts.json"
        |def toPair(split:Array[String]) = (split(0),split(1))
        |val lines = Source.fromFile(file).getLines()
        |val counts = lines.map(_.split("\t")).map(toPair).toList
        |counts.take(10)
      """.stripMargin)
  }

  saveTo("Smoothing", new File(Slides.dir, "smoothing.json"))


}
<commit_msg>smoothing slides now show plot<commit_after>package ml.wolfe.ui.statnlp

import java.io.{File, PrintStream}

import ml.wolfe.nlp.io.IO
import ml.wolfe.ui.MutableMoroNotebook
import ml.wolfe.util.Util

import scala.io.Source

/**
 * @author <NAME>
 */
/**
 * @author <NAME>
 */
object Smoothing extends MutableMoroNotebook with App {

  def mygreen = "#78AB46"

  def span(color: String)(body: String) = s"""<span style="color: $color">$body</span>"""

  section("Zipf") {

    md(
      s"""
        |## Zipf's Law
        |The frequency ${span(mygreen)("$f$")} of a word is inversely proportional to its rank ${span(mygreen)("$r$")}.
      """.stripMargin)

    latex("f(k; s, N) = \\frac{1 / k^s}{ \\sum_i i}")
  }

  section("ZipfGraph") {

    md("##Counts from ACL")

    wolfe(
      """
        |import scala.io.Source
        |val file = "/Users/sriedel/projects/stat-nlp-course-intern/data/counts.json"
        |def toPair(split:Array[String]) = (split(0),split(1).toInt)
        |val lines = Source.fromFile(file).getLines()
        |val counts = lines.map(_.split("\t")).map(toPair).toList
        |counts.take(10)
      """.stripMargin)
  }

  section("Tail") {

    md("##The Long Tail")

    wolfe(
      """
        |val x = counts.take(100).indices.map(_.toDouble)
        |val y = counts.take(100).map(_._2).map(_.toDouble)
        |val chart = plot((x,(Y(y,"freq"))))
        |D3Plotter.lineplot(chart)
      """.stripMargin
    )

  }

  saveTo("Smoothing", new File(Slides.dir, "smoothing.json"))


}

{'fraction_non_alphanumeric': 0.18800588668138338, 'fraction_numerical': 0.009565857247976454, 'mean_word_length': 2.9178674351585014, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6814253', 'n_tokens_mistral': 1061, 'n_tokens_neox': 1011, 'n_words': 188}