<commit_before>package asciiblog

import util.{ intersectionSize, invert }
import java.{ lang => jl }

class Similarities(_articles: Seq[Article], count: Int) {
  private[this] val arts: Array[Article] = _articles.toArray
  private[this] val tags: Map[Tag, Article] = arts.iterator.collect { case a if a.isTag => (a.asTag, a) }.toMap
  private[this] val slugMap: Map[Slug, Int] = arts.iterator.zipWithIndex.map { case (k, v) => (k.asSlug, v) }.toMap
  private[this] val tagMap: Map[Tag, Array[Int]] =
    invert(arts.map { a => (a, (a.tags.visible ++ a.tags.hidden).distinct) })
      .map { case (t, as) =>
        val idxs = as.map(a => slugMap(a.asSlug)).toArray
        java.util.Arrays.sort(idxs)
        (t, idxs)
      }

  // do not recommend tags that
  // - are used only as hidden tags
  // - are associated with only one article
  private[this] val tagsToRecommend: Seq[(Tag, Array[Int])] = {
    val visibleTags = arts.iterator.flatMap(a => a.tags.visible).toSet
    tagMap.iterator.filter { case (t, as) => visibleTags.contains(t) && as.length > 1 }.toVector
  }

  private[this] val reverseRels: Map[Slug, Seq[Slug]] =
    invert(_articles.collect { case a if a.rel.nonEmpty => (a.asSlug, a.rel.map(Slug)) })

  private def dateDiff(a: Article, b: Article): Int = {
    val ta = if (a.date == null) 0 else (a.date.getTime/(1000*3600)).toInt
    val tb = if (b.date == null) 0 else (b.date.getTime/(1000*3600)).toInt
    Math.abs(ta - tb)
  }

  private[this] val similarities: Array[Seq[Article]] = {
    val sims = arts.map {
      case a if a.isTag => similarTags(a.asTag, count)
      case a            => similarArticles(a, count, a.backlinks)
    }

    Array.tabulate(sims.length) { i =>
      arts(i) match {
        case a if !a.isTag && a.tags.isEmpty && a.rel.nonEmpty =>
          // not terribly optimized, but it's called only few times
          // TODO return sorted by frequency
          val slugs = sims(i).map(_.slug).toSet
          val extraSims = sims(i)
            .flatMap(a => sims(slugMap(a.asSlug)))
            .filter(a => !slugs.contains(a.slug))
            .distinct
            .take(count - sims(i).size)

          sims(i) ++ extraSims
        case a => sims(i)<commit_msg>Similarities - extra similarities sorted by frequency<commit_after>package asciiblog

import util.{ intersectionSize, invert }
import java.{ lang => jl }

class Similarities(_articles: Seq[Article], count: Int) {
  private[this] val arts: Array[Article] = _articles.toArray
  private[this] val tags: Map[Tag, Article] = arts.iterator.collect { case a if a.isTag => (a.asTag, a) }.toMap
  private[this] val slugMap: Map[Slug, Int] = arts.iterator.zipWithIndex.map { case (k, v) => (k.asSlug, v) }.toMap
  private[this] val tagMap: Map[Tag, Array[Int]] =
    invert(arts.map { a => (a, (a.tags.visible ++ a.tags.hidden).distinct) })
      .map { case (t, as) =>
        val idxs = as.map(a => slugMap(a.asSlug)).toArray
        java.util.Arrays.sort(idxs)
        (t, idxs)
      }

  // do not recommend tags that
  // - are used only as hidden tags
  // - are associated with only one article
  private[this] val tagsToRecommend: Seq[(Tag, Array[Int])] = {
    val visibleTags = arts.iterator.flatMap(a => a.tags.visible).toSet
    tagMap.iterator.filter { case (t, as) => visibleTags.contains(t) && as.length > 1 }.toVector
  }

  private[this] val reverseRels: Map[Slug, Seq[Slug]] =
    invert(_articles.collect { case a if a.rel.nonEmpty => (a.asSlug, a.rel.map(Slug)) })

  private def dateDiff(a: Article, b: Article): Int = {
    val ta = if (a.date == null) 0 else (a.date.getTime/(1000*3600)).toInt
    val tb = if (b.date == null) 0 else (b.date.getTime/(1000*3600)).toInt
    Math.abs(ta - tb)
  }

  private[this] val similarities: Array[Seq[Article]] = {
    val sims = arts.map {
      case a if a.isTag => similarTags(a.asTag, count)
      case a            => similarArticles(a, count, a.backlinks)
    }

    Array.tabulate(sims.length) { i =>
      arts(i) match {
        case a if !a.isTag && a.tags.isEmpty && a.rel.nonEmpty =>
          // not terribly optimized, but it's called only few times
          val slugs = sims(i).map(_.slug).toSet
          val extraSims = sims(i)
            .flatMap(a => sims(slugMap(a.asSlug)))
            .filter(a => !slugs.contains(a.slug))
            .groupBy(a => a.slug)
            .toSeq
            .map { case (_, as) => (as.head, as.size) }
            .sortBy(~_._2)
            .take(count - sims(i).size)
            .map(_._1)

          sims(i) ++ extraSims
        case a => sims(i)
{'fraction_non_alphanumeric': 0.15480349344978167, 'fraction_numerical': 0.008733624454148471, 'mean_word_length': 2.592941176470588, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024896', 'n_tokens_mistral': 1672, 'n_tokens_neox': 1611, 'n_words': 402}