<commit_before>package viscel.store

import java.nio.charset.StandardCharsets
import java.nio.file.Path

import better.files.File
import com.github.plokhotnyuk.jsoniter_scala.core._
import viscel.narration.Narrator
import viscel.shared.Log.{Store => Log}
import viscel.shared.{DataRow, JsoniterCodecs, Vid}


class RowStoreV4(db4dir: Path) {

  val base = File(db4dir)

  def file(vid: Vid): File = {
    base / vid.str
  }

  def allVids(): List[Vid] = synchronized {
    base.list(_.isRegularFile, 1).map(f => Vid.from(f.name)).toList
  }

  def open(narrator: Narrator): RowAppender = synchronized {
    open(narrator.id, narrator.name)
  }

  def open(id: Vid, name: String): RowAppender = synchronized {
    val f = file(id)
    if (!f.exists || f.size <= 0) JsoniterStorage.store(f.path, name)(JsoniterCodecs.StringRw)
    new RowAppender(f)
  }


  def load(id: Vid): (String, List[DataRow]) = synchronized {
    val start = System.currentTimeMillis()

    val f = file(id)

    if (!f.isRegularFile || f.size == 0)
      throw new IllegalStateException(s"$f does not contain data")
    else {
      val lines = f.lineIterator(StandardCharsets.UTF_8)
      val name  = readFromString[String](lines.next())(JsoniterCodecs.StringRw)

      val dataRows = lines.zipWithIndex.map { case (line, nr) =>
        try readFromString[DataRow](line)(JsoniterCodecs.DataRowRw) catch {
          case t : Throwable  =>
            Log.error(s"Failed to decode $id:${nr + 2}: $line")
            throw t
        }
      }.toList
      Log.info(s"loading $id (${System.currentTimeMillis() - start}ms)")
      (name, dataRows)
    }
  }

  def loadBook(id: Vid): Book = {
    val (name, rows) = try {load(id)} catch {
      case e: Throwable =>
        Log.error(s"loading book $id failed: ${e.getMessage}")
        throw e
    }
    Book.fromEntries(id, name, rows)
  }


}

class RowAppender(file: File) {
  def append(row: DataRow): Unit = synchronized {
    Log.trace(s"Store $row into $file")
    val bytes = JsoniterCodecs.writeArray(row)(JsoniterCodecs.DataRowRw)
    file.appendByteArray(bytes)
    file.appendByteArray(Array('\n'.toByte))
  }
}<commit_msg>do a more atomic write … maybe?<commit_after>package viscel.store

import java.nio.charset.StandardCharsets
import java.nio.file.Path

import better.files.File
import better.files.File.OpenOptions
import com.github.plokhotnyuk.jsoniter_scala.core._
import viscel.narration.Narrator
import viscel.shared.Log.{Store => Log}
import viscel.shared.{DataRow, JsoniterCodecs, Vid}


class RowStoreV4(db4dir: Path) {

  val base = File(db4dir)

  def file(vid: Vid): File = {
    base / vid.str
  }

  def allVids(): List[Vid] = synchronized {
    base.list(_.isRegularFile, 1).map(f => Vid.from(f.name)).toList
  }

  def open(narrator: Narrator): RowAppender = synchronized {
    open(narrator.id, narrator.name)
  }

  def open(id: Vid, name: String): RowAppender = synchronized {
    val f = file(id)
    if (!f.exists || f.size <= 0) JsoniterStorage.store(f.path, name)(JsoniterCodecs.StringRw)
    new RowAppender(f)
  }


  def load(id: Vid): (String, List[DataRow]) = synchronized {
    val start = System.currentTimeMillis()

    val f = file(id)

    if (!f.isRegularFile || f.size == 0)
      throw new IllegalStateException(s"$f does not contain data")
    else {
      val lines = f.lineIterator(StandardCharsets.UTF_8)
      val name  = readFromString[String](lines.next())(JsoniterCodecs.StringRw)

      val dataRows = lines.zipWithIndex.map { case (line, nr) =>
        try readFromString[DataRow](line)(JsoniterCodecs.DataRowRw) catch {
          case t : Throwable  =>
            Log.error(s"Failed to decode $id:${nr + 2}: $line")
            throw t
        }
      }.toList
      Log.info(s"loading $id (${System.currentTimeMillis() - start}ms)")
      (name, dataRows)
    }
  }

  def loadBook(id: Vid): Book = {
    val (name, rows) = try {load(id)} catch {
      case e: Throwable =>
        Log.error(s"loading book $id failed: ${e.getMessage}")
        throw e
    }
    Book.fromEntries(id, name, rows)
  }


}

class RowAppender(file: File) {
  def append(row: DataRow): Unit = synchronized {
    Log.trace(s"Store $row into $file")
    val bytes = JsoniterCodecs.writeArray(row)(JsoniterCodecs.DataRowRw)
    file.outputStream(OpenOptions.append).foreach { os =>
      os.write(bytes)
      os.write('\n')
    }
  }
}
{'fraction_non_alphanumeric': 0.13622924721400956, 'fraction_numerical': 0.003638844666818285, 'mean_word_length': 3.1845861084681255, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '815299', 'n_tokens_mistral': 1558, 'n_tokens_neox': 1514, 'n_words': 330}