<commit_before>package org.talkingpuffin.snippet

import scala.collection.JavaConversions._
import java.text.NumberFormat
import org.talkingpuffin.apix.PartitionedTweets
import twitter4j.User
import org.talkingpuffin.apix.RichStatus._
import org.talkingpuffin.user.UserAnalysis
import org.talkingpuffin.util.WordCounter.FreqToStringsMap
import org.talkingpuffin.snippet.LineCollector.InfoLine

object GeneralUserInfo {
  case class ScreenNames(names: List[String])
  case class Link(url: String) {
    override def toString = Link.stripFront(url)
  }
  object Link {
    def stripFront(string: String) = string.replaceAll("https?://(www\\.)?", "")
  }

  def create(user: User, screenName: String, pt: PartitionedTweets, ua: UserAnalysis): List[InfoLine] = {
    val lc = new LineCollector

    val fmt = NumberFormat.getInstance
    fmt.setMaximumFractionDigits(2)
    lc.disp("Name", user.getName + " (" + user.getScreenName + ")")
    lc.disp("Location", user.getLocation)
    lc.disp("Description", user.getDescription)
    lc.disp("Followers", fmt.format(user.getFollowersCount))
    lc.disp("Following", fmt.format(user.getFriendsCount))
    lc.disp("Tweets analyzed", fmt.format(ua.numTweets))
    lc.disp("Range", ua.range.getDays + " days")
    lc.disp( "Avg per day", fmt.format(ua.avgTweetsPerDay) + (
      if (ua.numReplies > 0)
        " (" + fmt.format(ua.avgTweetsPerDayExcludingReplies) + " excluding replies)"
      else
        ""
      ))
    if (ua.numLinks > 0)
      lc.disp("Links in tweets", ua.numLinks + " (" + ua.links.distinct.size + " unique)")
    if (ua.numUsers > 0)
      lc.disp("Users mentioned", ua.numUsers + " (" + ua.users.distinct.size + " unique)")
    lc.disp("Clients", ua.clients.map(_.name).mkString(", "))

    lc.msgs.reverse
  }

  def createScreenNameFreq(ua: UserAnalysis) = {
    val lc = new LineCollector
    dispFreq(lc, "Screen name frequencies", ua.screenNamesCounter.frequencies, (l) => ScreenNames(l), 0)
    lc.msgs.reverse
  }

  def createWordFreq(ua: UserAnalysis) = {
    val lc = new LineCollector
    dispFreq(lc, "Word frequencies", ua.tweetsWordCounter.frequencies, (l) => l.mkString(", "), 2)
    lc.msgs.reverse
  }

  def links(ua: UserAnalysis) = ua.links.map(_.link).distinct.map(link => Link(link)).sortBy(_.toString)

  private def dispFreq(lc: LineCollector, title: String, bmap: FreqToStringsMap,
  fn: (List[String]) => AnyRef, minFreq: Int): Unit =
    bmap match {
      case freqs if ! freqs.isEmpty =>
        lc.disp(title, "")
        for (freq <- freqs.keysIterator.filter(_ > minFreq).toList.sorted.reverse)
          lc.disp(freq.toString, fn(freqs.get(freq).get.sorted))
      case _ =>
    }

}<commit_msg>Remove headings from frequency displays<commit_after>package org.talkingpuffin.snippet

import scala.collection.JavaConversions._
import java.text.NumberFormat
import org.talkingpuffin.apix.PartitionedTweets
import twitter4j.User
import org.talkingpuffin.apix.RichStatus._
import org.talkingpuffin.user.UserAnalysis
import org.talkingpuffin.util.WordCounter.FreqToStringsMap
import org.talkingpuffin.snippet.LineCollector.InfoLine

object GeneralUserInfo {
  case class ScreenNames(names: List[String])
  case class Link(url: String) {
    override def toString = Link.stripFront(url)
  }
  object Link {
    def stripFront(string: String) = string.replaceAll("https?://(www\\.)?", "")
  }

  def create(user: User, screenName: String, pt: PartitionedTweets, ua: UserAnalysis): List[InfoLine] = {
    val lc = new LineCollector

    val fmt = NumberFormat.getInstance
    fmt.setMaximumFractionDigits(2)
    lc.disp("Name", user.getName + " (" + user.getScreenName + ")")
    lc.disp("Location", user.getLocation)
    lc.disp("Description", user.getDescription)
    lc.disp("Followers", fmt.format(user.getFollowersCount))
    lc.disp("Following", fmt.format(user.getFriendsCount))
    lc.disp("Tweets analyzed", fmt.format(ua.numTweets))
    lc.disp("Range", ua.range.getDays + " days")
    lc.disp( "Avg per day", fmt.format(ua.avgTweetsPerDay) + (
      if (ua.numReplies > 0)
        " (" + fmt.format(ua.avgTweetsPerDayExcludingReplies) + " excluding replies)"
      else
        ""
      ))
    if (ua.numLinks > 0)
      lc.disp("Links in tweets", ua.numLinks + " (" + ua.links.distinct.size + " unique)")
    if (ua.numUsers > 0)
      lc.disp("Users mentioned", ua.numUsers + " (" + ua.users.distinct.size + " unique)")
    lc.disp("Clients", ua.clients.map(_.name).mkString(", "))

    lc.msgs.reverse
  }

  def createScreenNameFreq(ua: UserAnalysis) = {
    val lc = new LineCollector
    dispFreq(lc, ua.screenNamesCounter.frequencies, (l) => ScreenNames(l), 0)
    lc.msgs.reverse
  }

  def createWordFreq(ua: UserAnalysis) = {
    val lc = new LineCollector
    dispFreq(lc, ua.tweetsWordCounter.frequencies, (l) => l.mkString(", "), 2)
    lc.msgs.reverse
  }

  def links(ua: UserAnalysis) = ua.links.map(_.link).distinct.map(link => Link(link)).sortBy(_.toString)

  private def dispFreq(lc: LineCollector, bmap: FreqToStringsMap,
  fn: (List[String]) => AnyRef, minFreq: Int): Unit =
    bmap match {
      case freqs if ! freqs.isEmpty =>
        for (freq <- freqs.keysIterator.filter(_ > minFreq).toList.sorted.reverse)
          lc.disp(freq.toString, fn(freqs.get(freq).get.sorted))
      case _ =>
    }

}
{'fraction_non_alphanumeric': 0.1437957574619861, 'fraction_numerical': 0.002628120893561104, 'mean_word_length': 3.736, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4556720', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1850, 'n_words': 334}