<commit_before>package de.rmgk

import java.nio.file.Paths
import java.util.concurrent.TimeUnit

import org.openjdk.jmh.annotations._
import org.openjdk.jmh.infra.Blackhole
import viscel.Services
import viscel.server.ContentLoader
import viscel.shared.Vid
import viscel.store.{Book, ReadableContent, ScribeDataRow}


@State(Scope.Thread)
@BenchmarkMode(Array(Mode.AverageTime, Mode.SingleShotTime))
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Warmup(iterations = 0, time = 1000, timeUnit = TimeUnit.MILLISECONDS)
@Measurement(iterations = 5, time = 1000, timeUnit = TimeUnit.MILLISECONDS)
@Fork(1)
class ContentLoading {

  @Param(Array(""))
  var bookId: String = _

  @Param(Array(""))
  var dbpath: String = _

  var services: Services = _
  var vid     : Vid      = _

  var loadedContents: (String, Seq[ScribeDataRow]) = _

  var book: Book = _

  var pages: List[ReadableContent] = _

  @Setup
  def setup(): Unit = {
    services = new Services(Paths.get(dbpath), Paths.get("./blobs/"), "localhost", 2358)
    vid = Vid.from(bookId)
    loadedContents = services.rowStore.loadOld(vid)
    book = Book.fromEntries(vid, loadedContents._1, loadedContents._2)
    pages = ContentLoader.linearizedPages(book)
  }

  @Benchmark
  def loadContents(bh: Blackhole) = {
    services.rowStore.loadOld(vid)
  }

  @Benchmark
  def parseBook(bh: Blackhole) = {
    Book.fromEntries(vid, loadedContents._1, loadedContents._2)
  }

  @Benchmark
  def linearizeBook(bh: Blackhole) = {
    ContentLoader.linearizedPages(book)
  }

  @Benchmark
  def pagesToContents(bh: Blackhole) = {
    ContentLoader.pagesToContents(pages)
  }


}
<commit_msg>compile benchmarks<commit_after>package viscel.benchmarks

import java.nio.file.Paths
import java.util.concurrent.TimeUnit

import org.openjdk.jmh.annotations._
import org.openjdk.jmh.infra.Blackhole
import viscel.Services
import viscel.server.ContentLoader
import viscel.server.ContentLoader.LinearResult
import viscel.shared.Vid
import viscel.store.Book
import viscel.store.v4.{DataRow, RowStoreV4}


@State(Scope.Thread)
@BenchmarkMode(Array(Mode.AverageTime, Mode.SingleShotTime))
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Warmup(iterations = 0, time = 1000, timeUnit = TimeUnit.MILLISECONDS)
@Measurement(iterations = 5, time = 1000, timeUnit = TimeUnit.MILLISECONDS)
@Fork(1)
class ContentLoading {

  @Param(Array(""))
  var bookId: String = _

  @Param(Array(""))
  var dbpath: String = _

  var services: RowStoreV4 = _
  var vid     : Vid        = _

  var loadedContents: (String, Seq[DataRow]) = _

  var book: Book = _

  var pages: LinearResult = _

  @Setup
  def setup(): Unit = {
    services = new RowStoreV4(Paths.get(dbpath))
    vid = Vid.from(bookId)
    loadedContents = services.load(vid)
    book = Book.fromEntries(vid, loadedContents._1, loadedContents._2)
    pages = ContentLoader.linearizedPages(book)
  }

  @Benchmark
  def loadContents(bh: Blackhole) = {
    services.load(vid)
  }

  @Benchmark
  def parseBook(bh: Blackhole) = {
    Book.fromEntries(vid, loadedContents._1, loadedContents._2)
  }

  @Benchmark
  def linearizeBook(bh: Blackhole) = {
    ContentLoader.linearizedPages(book)
  }

  @Benchmark
  def pagesToContents(bh: Blackhole) = {
    ContentLoader.pagesToContents(pages)
  }


}

{'fraction_non_alphanumeric': 0.1158443150475023, 'fraction_numerical': 0.011645724793135151, 'mean_word_length': 4.315960912052117, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '531918', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1127, 'n_words': 211}