<commit_before>package org.http4s.util

import java.time.{ZoneId, Instant}
import java.time.format.DateTimeFormatter
import java.util.Locale

import scodec.bits.ByteVector
import scala.annotation.tailrec
import scala.collection.immutable.BitSet


/** A type class that describes how to efficiently render a type
 * @tparam T the type which will be rendered
 */
trait Renderer[T] {

  /** Renders the object to the writer
    * @param writer [[Writer]] to render to
    * @param t object to render
    * @return the same [[Writer]] provided
    */
  def render(writer: Writer, t: T): writer.type
}

object Renderer {
  def renderString[T: Renderer](t: T): String = new StringWriter().append(t).result

  implicit val RFC7231InstantRenderer: Renderer[Instant] = new Renderer[Instant] {

    private val dateFormat =
      DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss zzz")
        .withLocale(Locale.US)
        .withZone(ZoneId.of("UTC"))

    override def render(writer: Writer, t: Instant): writer.type =
      writer << dateFormat.format(t)

  }
}

/** Mixin that makes a type writable by a [[Writer]] without needing a [[Renderer]] instance */
trait Renderable extends Any {

  /** Base method for rendering this object efficiently */
  def render(writer: Writer): writer.type

  /** Generates a String rendering of this object */
  def renderString = Renderer.renderString(this)

  override def toString: String = renderString
}

object Renderable {
  implicit def renderableInst[T <: Renderable]: Renderer[T] =
    genericInstance.asInstanceOf[Renderer[T]]

  // Cache the Renderable because GC pauses suck
  private val genericInstance = new Renderer[Renderable] {
    override def render(writer: Writer, t: Renderable): writer.type =
      t.render(writer)
  }
}

object Writer {
  val HeaderValueDQuote = BitSet("\\\"".map(_.toInt):_*)
}

/** Efficiently accumulate [[Renderable]] representations */
trait Writer {
  def append(s: String):                 this.type
  def append(ci: CaseInsensitiveString): this.type = append(ci.toString)
  def append(char: Char):                this.type = append(char.toString)
  def append(float: Float):              this.type = append(float.toString)
  def append(double: Double):            this.type = append(double.toString)
  def append(int: Int):                  this.type = append(int.toString)
  def append(long: Long):                this.type = append(long.toString)

  def append[T](r: T)(implicit R: Renderer[T]): this.type = R.render(this, r)

  def quote(s: String, escapedChars: BitSet = Writer.HeaderValueDQuote, escapeChar: Char = '\\'): this.type = {
    this << '"'

    @tailrec
    def go(i: Int): Unit = if (i < s.length) {
      val c = s.charAt(i)
      if (escapedChars.contains(c.toInt)) this << escapeChar
      this << c
      go(i+1)
    }

    go(0)
    this << '"'
  }

  def addStrings(s: Seq[String], sep: String = "", start: String = "", end: String = ""): this.type = {
    append(start)
    if (s.nonEmpty) {
      append(s.head)
      s.tail.foreach(s => append(sep).append(s))
    }
    append(end)
  }

  def addStringNel(s: NonEmptyList[String], sep: String = "", start: String = "", end: String = ""): this.type = {
    append(start)
    append(s.head)
    s.tail.foreach(s => append(sep).append(s))
    append(end)
  }

  def addSeq[T: Renderer](s: Seq[T], sep: String = "", start: String = "", end: String = ""): this.type = {
    append(start)
    if (s.nonEmpty) {
      append(s.head)
      s.tail.foreach(s => append(s).append(sep))
    }
    append(end)
  }

  final def <<(s: String):                this.type = append(s)
  final def <<#(s: String):               this.type = quote(s)
  final def <<(s: CaseInsensitiveString): this.type = append(s)
  final def <<(char: Char):               this.type = append(char)
  final def <<(float: Float):             this.type = append(float)
  final def <<(double: Double):           this.type = append(double)
  final def <<(int: Int):                 this.type = append(int)
  final def <<(long: Long):               this.type = append(long)
  final def <<[T: Renderer](r: T):        this.type = append(r)


}

/** [[Writer]] that will result in a `String`
  * @param size initial buffer size of the underlying `StringBuilder`
  */
class StringWriter(size: Int = 64) extends Writer {
  private val sb = new java.lang.StringBuilder(size)

  def append(s: String) = { sb.append(s); this }
  override def append(char: Char) = { sb.append(char); this }
  override def append(float: Float) = { sb.append(float); this }
  override def append(double: Double) = { sb.append(double); this }
  override def append(int: Int) = { sb.append(int); this }
  override def append(long: Long) = { sb.append(long); this }

  def result(): String = sb.toString
}
/** [[Writer]] that will result in a `ByteVector`
  * @param bv initial ByteVector`
  */
case class ByteVectorWriter(private var bv:ByteVector = ByteVector.empty) extends Writer {

  override def append(s: String)       = { bv = bv ++ ByteVector(s.getBytes);this}
  override def append(char: Char)      = { bv = bv ++ ByteVector(char.toByte);this}
  override def append(float: Float)    = { bv = bv ++ ByteVector(float.toByte);this}
  override def append(double: Double)  = { bv = bv ++ ByteVector(double.toByte);this}
  override def append(int: Int)        = { bv = bv ++ ByteVector(int.toByte);this}
  override def append(long: Long)      = { bv = bv ++ ByteVector(long.toByte);this}

  def toByteVector() = bv

}
<commit_msg>Give ByteStringWriter a charset for encoding.<commit_after>package org.http4s.util

import java.nio.charset.{ Charset, StandardCharsets }
import java.time.{ZoneId, Instant}
import java.time.format.DateTimeFormatter
import java.util.Locale

import scodec.bits.ByteVector
import scala.annotation.tailrec
import scala.collection.immutable.BitSet


/** A type class that describes how to efficiently render a type
 * @tparam T the type which will be rendered
 */
trait Renderer[T] {

  /** Renders the object to the writer
    * @param writer [[Writer]] to render to
    * @param t object to render
    * @return the same [[Writer]] provided
    */
  def render(writer: Writer, t: T): writer.type
}

object Renderer {
  def renderString[T: Renderer](t: T): String = new StringWriter().append(t).result

  implicit val RFC7231InstantRenderer: Renderer[Instant] = new Renderer[Instant] {

    private val dateFormat =
      DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss zzz")
        .withLocale(Locale.US)
        .withZone(ZoneId.of("UTC"))

    override def render(writer: Writer, t: Instant): writer.type =
      writer << dateFormat.format(t)

  }
}

/** Mixin that makes a type writable by a [[Writer]] without needing a [[Renderer]] instance */
trait Renderable extends Any {

  /** Base method for rendering this object efficiently */
  def render(writer: Writer): writer.type

  /** Generates a String rendering of this object */
  def renderString = Renderer.renderString(this)

  override def toString: String = renderString
}

object Renderable {
  implicit def renderableInst[T <: Renderable]: Renderer[T] =
    genericInstance.asInstanceOf[Renderer[T]]

  // Cache the Renderable because GC pauses suck
  private val genericInstance = new Renderer[Renderable] {
    override def render(writer: Writer, t: Renderable): writer.type =
      t.render(writer)
  }
}

object Writer {
  val HeaderValueDQuote = BitSet("\\\"".map(_.toInt):_*)
}

/** Efficiently accumulate [[Renderable]] representations */
trait Writer {
  def append(s: String):                 this.type
  def append(ci: CaseInsensitiveString): this.type = append(ci.toString)
  def append(char: Char):                this.type = append(char.toString)
  def append(float: Float):              this.type = append(float.toString)
  def append(double: Double):            this.type = append(double.toString)
  def append(int: Int):                  this.type = append(int.toString)
  def append(long: Long):                this.type = append(long.toString)

  def append[T](r: T)(implicit R: Renderer[T]): this.type = R.render(this, r)

  def quote(s: String, escapedChars: BitSet = Writer.HeaderValueDQuote, escapeChar: Char = '\\'): this.type = {
    this << '"'

    @tailrec
    def go(i: Int): Unit = if (i < s.length) {
      val c = s.charAt(i)
      if (escapedChars.contains(c.toInt)) this << escapeChar
      this << c
      go(i+1)
    }

    go(0)
    this << '"'
  }

  def addStrings(s: Seq[String], sep: String = "", start: String = "", end: String = ""): this.type = {
    append(start)
    if (s.nonEmpty) {
      append(s.head)
      s.tail.foreach(s => append(sep).append(s))
    }
    append(end)
  }

  def addStringNel(s: NonEmptyList[String], sep: String = "", start: String = "", end: String = ""): this.type = {
    append(start)
    append(s.head)
    s.tail.foreach(s => append(sep).append(s))
    append(end)
  }

  def addSeq[T: Renderer](s: Seq[T], sep: String = "", start: String = "", end: String = ""): this.type = {
    append(start)
    if (s.nonEmpty) {
      append(s.head)
      s.tail.foreach(s => append(s).append(sep))
    }
    append(end)
  }

  final def <<(s: String):                this.type = append(s)
  final def <<#(s: String):               this.type = quote(s)
  final def <<(s: CaseInsensitiveString): this.type = append(s)
  final def <<(char: Char):               this.type = append(char)
  final def <<(float: Float):             this.type = append(float)
  final def <<(double: Double):           this.type = append(double)
  final def <<(int: Int):                 this.type = append(int)
  final def <<(long: Long):               this.type = append(long)
  final def <<[T: Renderer](r: T):        this.type = append(r)


}

/** [[Writer]] that will result in a `String`
  * @param size initial buffer size of the underlying `StringBuilder`
  */
class StringWriter(size: Int = 64) extends Writer {
  private val sb = new java.lang.StringBuilder(size)

  def append(s: String) = { sb.append(s); this }
  override def append(char: Char) = { sb.append(char); this }
  override def append(float: Float) = { sb.append(float); this }
  override def append(double: Double) = { sb.append(double); this }
  override def append(int: Int) = { sb.append(int); this }
  override def append(long: Long) = { sb.append(long); this }

  def result(): String = sb.toString
}
/** [[Writer]] that will result in a `ByteVector`
  * @param bv initial ByteVector`
  */
case class ByteVectorWriter(private var bv: ByteVector = ByteVector.empty,
                            charset: Charset = StandardCharsets.UTF_8) extends Writer {

  override def append(s: String)       = { bv = bv ++ ByteVector(s.getBytes(charset)); this}
  override def append(char: Char)      = append(char.toString)
  override def append(float: Float)    = append(float.toString)
  override def append(double: Double)  = append(double.toString)
  override def append(int: Int)        = append(int.toString)
  override def append(long: Long)      = append(long.toString)

  def toByteVector() = bv
}

{'fraction_non_alphanumeric': 0.13877881844380405, 'fraction_numerical': 0.0017110951008645534, 'mean_word_length': 3.1282527881040894, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '974425', 'n_tokens_mistral': 3535, 'n_tokens_neox': 3467, 'n_words': 982}