<commit_before>package walfie.gbf.raidfinder

import monix.execution.{Cancelable, Scheduler}
import monix.reactive._
import scala.concurrent.duration._
import twitter4j._
import walfie.gbf.raidfinder.domain._

trait RaidFinder {
  def getRaidTweets(bossName: BossName): Observable[RaidTweet]
  def getKnownBosses(): Map[BossName, RaidBoss]
  def shutdown(): Unit
}

object RaidFinder {
  val DefaultCacheSizePerBoss = 50
  val DefaultBacklogSize = 100

  /** Stream tweets without looking up old tweets first */
  def withoutBacklog(
    twitterStream:    TwitterStream = TwitterStreamFactory.getSingleton,
    cacheSizePerBoss: Int           = DefaultCacheSizePerBoss
  )(implicit scheduler: Scheduler): DefaultRaidFinder = {
    val statuses = TwitterStreamer(twitterStream).observable
    new DefaultRaidFinder(statuses, cacheSizePerBoss)
  }

  /** Search for old tweets first before streaming new tweets */
  def withBacklog(
    twitter:          Twitter       = TwitterFactory.getSingleton,
    twitterStream:    TwitterStream = TwitterStreamFactory.getSingleton,
    backlogSize:      Int           = DefaultBacklogSize,
    cacheSizePerBoss: Int           = DefaultCacheSizePerBoss
  )(implicit scheduler: Scheduler): DefaultRaidFinder = {
    import TwitterSearcher._

    val backlog = TwitterSearcher(twitter, TwitterSearcher.ReverseChronological)
      .observable(DefaultSearchTerm, None, MaxCount)
      .flatMap(Observable.fromIterable)
      .take(backlogSize)
    val newStatuses = TwitterStreamer(twitterStream).observable
    new DefaultRaidFinder(backlog ++ newStatuses, cacheSizePerBoss) {<commit_msg>Sort backlog by earliest first before returning it<commit_after>package walfie.gbf.raidfinder

import monix.eval.Task
import monix.execution.{Cancelable, Scheduler}
import monix.reactive._
import scala.concurrent.duration._
import twitter4j._
import walfie.gbf.raidfinder.domain._

trait RaidFinder {
  def getRaidTweets(bossName: BossName): Observable[RaidTweet]
  def getKnownBosses(): Map[BossName, RaidBoss]
  def shutdown(): Unit
}

object RaidFinder {
  val DefaultCacheSizePerBoss = 50
  val DefaultBacklogSize = 100

  /** Stream tweets without looking up old tweets first */
  def withoutBacklog(
    twitterStream:    TwitterStream = TwitterStreamFactory.getSingleton,
    cacheSizePerBoss: Int           = DefaultCacheSizePerBoss
  )(implicit scheduler: Scheduler): DefaultRaidFinder = {
    val statuses = TwitterStreamer(twitterStream).observable
    new DefaultRaidFinder(statuses, cacheSizePerBoss)
  }

  /** Search for old tweets first before streaming new tweets */
  def withBacklog(
    twitter:          Twitter       = TwitterFactory.getSingleton,
    twitterStream:    TwitterStream = TwitterStreamFactory.getSingleton,
    backlogSize:      Int           = DefaultBacklogSize,
    cacheSizePerBoss: Int           = DefaultCacheSizePerBoss
  )(implicit scheduler: Scheduler): DefaultRaidFinder = {
    import TwitterSearcher._

    // Get backlog of tweets, then sort them by earliest first
    // TODO: This is getting kinda complex -- should write a test
    val backlogTask: Task[Seq[Status]] =
      TwitterSearcher(twitter, TwitterSearcher.ReverseChronological)
        .observable(DefaultSearchTerm, None, MaxCount)
        .flatMap(Observable.fromIterable)
        .take(backlogSize)
        .toListL
        .map(_.sortBy(_.getCreatedAt)) // earliest first

    // Once the backlog is populated, new tweets will stream in
    val backlogObservable = Observable.fromTask(backlogTask).flatMap(Observable.fromIterable)
    val newStatusesObservable = TwitterStreamer(twitterStream).observable

    new DefaultRaidFinder(backlogObservable ++ newStatusesObservable, cacheSizePerBoss) {
{'fraction_non_alphanumeric': 0.07445099089448312, 'fraction_numerical': 0.0032137118371719335, 'mean_word_length': 4.054127198917456, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2978811', 'n_tokens_mistral': 1111, 'n_tokens_neox': 1077, 'n_words': 257}