<commit_before>package p1bar

import java.io.PrintStream
import java.time.{Instant, ZoneId}
import java.time.format.DateTimeFormatter
import java.util.concurrent.TimeUnit

import scala.concurrent.duration.{Duration, FiniteDuration}

trait BarFormat {
  def leftBoundary: String
  def bar: String
  def empty: String
  def rightBoundary: String
}

object AsciiBarFormat extends BarFormat {
  override def leftBoundary: String = "|"
  override def bar: String = "#"
  override def empty: String = "-"
  override def rightBoundary: String = "|"
}

object UnicodeBarFormat extends BarFormat {
  override def leftBoundary: String = "|"
  override def bar: String = "\u2588"
  override def empty: String = "\u2591"
  override def rightBoundary: String = "|"
}

class BarFormatter(barFormat: BarFormat = AsciiBarFormat, unit: String = "it", unitScale: Boolean = false,
                   ncols: Int = 10) {
  private val longFmt = DateTimeFormatter.ofPattern("HH:mm:ss")
  private val shortFmt = DateTimeFormatter.ofPattern("mm:ss")

  def format(n: Int, total: Int, elapsed: Duration): String = {
    require(n <= total)
    require(n >= 0 && total >= 0)

    val leftBar = percentage(n, total)
    val nBars = Math.max(1, ncols - leftBar.length - 1)

    val elapsedFmt = formatInterval(elapsed)

    val rate = 1.0 * n / elapsed.toSeconds
    val rateFmt = f"$rate%5.2f$unit%s/s"

    val remainingFmt = formatInterval(FiniteDuration(((total - n) / rate).toLong, TimeUnit.SECONDS))

    val rightBar = s"$n/$total [$elapsedFmt<$remainingFmt, $rateFmt]"
    s"$leftBar ${progressBar(n, total, nBars)} $rightBar"
  }

  private def formatInterval(int: Duration): String = {
    val inst = Instant.ofEpochMilli(int.toMillis).atZone(ZoneId.systemDefault()).toLocalDateTime
    if (int.toHours >= 1) longFmt.format(inst) else shortFmt.format(inst)
  }

  private def percentage(n: Int, total: Int): String = {
    val v: Double = 100.0 * n / total
    f"$v%.1f%%"
  }

  private def progressBar(n: Int, total: Int, nBars: Int): String = {
    val totalLength = nBars - barFormat.leftBoundary.length - barFormat.rightBoundary.length
    val frac = 1.0 * n / total
    val done = (frac * totalLength).toInt
    val remaining = totalLength - done

    s"${barFormat.leftBoundary}${barFormat.bar * done}${barFormat.empty * remaining}${barFormat.rightBoundary}"
  }
}

trait Updater {
  def update(incr: Int): Unit
}

class ProgressBar(total: Int, barFormatter: BarFormatter) {
  def this() = this(ProgressBar.UnknownTotal, new BarFormatter())

  def this(total: Int) = this(total, new BarFormatter())

  private lazy val console = new PrintStream(System.err, true, "UTF-8")
  private var n = 0
  private var lastLen = 0
  private var startTime: Long = _

  private def now(): Long = System.nanoTime

  private def update(incr: Int): Unit = {
    val curTime  = now()

    require(incr >= 0)
    n += incr

    val elapsed = FiniteDuration(curTime - startTime, TimeUnit.NANOSECONDS)

    val barLine: String = barFormatter.format(n, total, elapsed)
    val padding: String = " " * Math.max(lastLen - barLine.length, 0)

    console.print(s"\r$barLine$padding")

    lastLen = barLine.length
  }

  def meter[A](block: Updater => A): Unit = {
    start()
    block(update)
    stop()
  }

  private def start(): Unit = {
    startTime = now()
    n = 0
    lastLen = 0
  }

  private def stop(): Unit = {
    console.println(" Done.")
  }
}

object ProgressBar {
  private val UnknownTotal: Int = -1
}

object Main extends App {
  val its = 60

  val progress = new ProgressBar(its, new BarFormatter(ncols = 30))
  progress meter { updater =>
    (1 to its).foreach { i =>
      Thread.sleep(500)
      updater.update(1)
    }
  }
}
<commit_msg>Added render rate limiter<commit_after>package p1bar

import java.io.PrintStream
import java.time.{Instant, ZoneId}
import java.time.format.DateTimeFormatter
import java.util.concurrent.TimeUnit

import scala.concurrent.duration.{Duration, FiniteDuration}

trait BarFormat {
  def leftBoundary: String
  def bar: String
  def empty: String
  def rightBoundary: String
}

object AsciiBarFormat extends BarFormat {
  override def leftBoundary: String = "|"
  override def bar: String = "#"
  override def empty: String = "-"
  override def rightBoundary: String = "|"
}

object UnicodeBarFormat extends BarFormat {
  override def leftBoundary: String = "|"
  override def bar: String = "\u2588"
  override def empty: String = "\u2591"
  override def rightBoundary: String = "|"
}

class BarFormatter(barFormat: BarFormat = AsciiBarFormat, unit: String = "it", unitScale: Boolean = false,
                   ncols: Int = 10) {
  private val longFmt = DateTimeFormatter.ofPattern("HH:mm:ss")
  private val shortFmt = DateTimeFormatter.ofPattern("mm:ss")

  def format(n: Int, total: Int, elapsed: Duration): String = {
    require(n <= total && total > 0)
    require(n >= 0)

    val leftBarStr = leftBar(n, total)
    val rightBarStr = rightBar(n, total, elapsed)

    val nBars = Math.max(1, ncols - leftBarStr.length - rightBarStr.length - 2)
    val bar = if (nBars > 6) " " + progressBar(n, total, nBars) + " " else "|"

    s"$leftBarStr$bar$rightBarStr"
  }

  def format(n: Int, elapsed: Duration): String = rightBar(n, elapsed)

  private def formatInterval(int: Duration): String = {
    val inst = Instant.ofEpochMilli(int.toMillis).atZone(ZoneId.systemDefault()).toLocalDateTime
    if (int.toHours >= 1) longFmt.format(inst) else shortFmt.format(inst)
  }

  private def leftBar(n: Int, total: Int): String = {
    val v: Double = 100d * n / total
    f"$v%.1f%%"
  }

  private def progressBar(n: Int, total: Int, nBars: Int): String = {
    val bodyLength = nBars - barFormat.leftBoundary.length - barFormat.rightBoundary.length
    val frac = n.toDouble / total
    val done = (frac * bodyLength).toInt
    val remaining = bodyLength - done

    s"${barFormat.leftBoundary}${barFormat.bar * done}${barFormat.empty * remaining}${barFormat.rightBoundary}"
  }

  private def rightBar(n: Int, total: Int, elapsed: Duration): String = {
    val elapsedFmt = formatInterval(elapsed)

    val rate = n.toDouble / elapsed.toSeconds
    val rateFmt = f"$rate%5.2f$unit%s/s"

    val remainingFmt = formatInterval(FiniteDuration(((total - n) / rate).toLong, TimeUnit.SECONDS))

    s"$n/$total [$elapsedFmt<$remainingFmt, $rateFmt]"
  }

  private def rightBar(n: Int, elapsed: Duration): String = {
    val elapsedFmt = formatInterval(elapsed)

    val rate = n.toDouble / elapsed.toSeconds
    val rateFmt = f"$rate%5.2f$unit%s/s"

    s"$n [$elapsedFmt, $rateFmt]"
  }
}

trait Updater {
  def update(incr: Int): Unit
}

class ProgressBar(total: Int, barFormatter: BarFormatter) {
  def this(barFormatter: BarFormatter) = this(ProgressBar.UnknownTotal, barFormatter)
  def this() = this(ProgressBar.UnknownTotal, new BarFormatter())

  def this(total: Int) = this(total, new BarFormatter())

  private lazy val console = new PrintStream(System.err, true, "UTF-8")
  private val renderInterval: Long = TimeUnit.MILLISECONDS.toNanos(100)

  private var startTime: Long = _
  private var n = 0
  private var lastLen = 0
  private var lastRenderTime: Long = 0

  private def now(): Long = System.nanoTime

  private def update(incr: Int): Unit = {
    require(incr >= 0)
    n += incr

    if (now() - lastRenderTime > renderInterval || n == total) {
      render()
      lastRenderTime = now()
    }
  }

  private def render(): Unit = {
    val elapsed = FiniteDuration(now() - startTime, TimeUnit.NANOSECONDS)

    val barLine: String = if (total == ProgressBar.UnknownTotal) {
      barFormatter.format(n, elapsed)
    } else {
      barFormatter.format(n, total, elapsed)
    }
    val padding: String = " " * Math.max(lastLen - barLine.length, 0)

    console.print(s"\r$barLine$padding")

    lastLen = barLine.length
  }

  def meter[A](block: Updater => A): Unit = {
    start()
    block(update)
    stop()
  }

  private def start(): Unit = {
    startTime = now()
    n = 0
    lastLen = 0
  }

  private def stop(): Unit = {
    console.println(" Done.")
  }
}

object ProgressBar {
  val UnknownTotal: Int = -1
}

object Main extends App {
  val its = 6000

  val progress = new ProgressBar(new BarFormatter(ncols = 5))
  progress meter { updater =>
    (1 to its).foreach { i =>
      Thread.sleep(1)
      updater.update(1)
    }
  }
}

{'fraction_non_alphanumeric': 0.12486583184257603, 'fraction_numerical': 0.010614192009540847, 'mean_word_length': 3.4725333333333332, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '657610', 'n_tokens_mistral': 2893, 'n_tokens_neox': 2754, 'n_words': 742}