<commit_before>package woot

// # Each character has an `Id`.
// An `Id` is usually made up of a `SiteId` and a `ClockValue`, but there are two special cases
// called `Beginning` and `Ending`. This gives something for the first and last characters to point to.

sealed trait Id {
  def <(that: Id): Boolean
}

object Beginning extends Id {
  def <(that: Id) = that match {
    case Beginning => false
    case _         => true
  }

  override def toString = "Beginning"
}

object Ending extends Id {
  def <(that: Id) = false
  override def toString = "Ending"
}

case class SiteId(value: String) extends AnyVal {
  def <(that: SiteId) = this.value < that.value
}

object SiteId {
  import util.Random
  // Some arbitrary session identifier for a site:
  def random: SiteId =
    SiteId(Random.alphanumeric.take(8).mkString)
}

case class ClockValue(value: Int) extends AnyVal {
  def incr: ClockValue = ClockValue(value + 1)
  def <(that: ClockValue) = this.value < that.value
}


case class CharId(ns: SiteId, ng: ClockValue) extends Id {
  // Each new character retains the site id, but increments the logical local clock, making the `Id` unique.
  def incr = copy(ng = ng.incr)

  // The `<` comparison is defined in _Definition 7_ (p. 8) of [RR5580]<commit_msg>Removes unused code<commit_after>package woot

// # Each character has an `Id`.
// An `Id` is usually made up of a `SiteId` and a `ClockValue`, but there are two special cases
// called `Beginning` and `Ending`. This gives something for the first and last characters to point to.

sealed trait Id {
  def <(that: Id): Boolean
}

object Beginning extends Id {
  def <(that: Id) = true
}

object Ending extends Id {
  def <(that: Id) = false
}

case class SiteId(value: String) extends AnyVal {
  def <(that: SiteId) = this.value < that.value
}

object SiteId {
  import util.Random
  // Some arbitrary session identifier for a site:
  def random: SiteId =
    SiteId(Random.alphanumeric.take(8).mkString)
}

case class ClockValue(value: Int) extends AnyVal {
  def incr: ClockValue = ClockValue(value + 1)
  def <(that: ClockValue) = this.value < that.value
}


case class CharId(ns: SiteId, ng: ClockValue) extends Id {
  // The `<` comparison is defined in _Definition 7_ (p. 8) of [RR5580]
{'fraction_non_alphanumeric': 0.10712704736609119, 'fraction_numerical': 0.007082779991146525, 'mean_word_length': 3.556451612903226, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '120723', 'n_tokens_mistral': 746, 'n_tokens_neox': 720, 'n_words': 286}