<commit_before>package org.http4s

import java.io.{File, FileOutputStream, StringReader}
import javax.xml.parsers.SAXParser

import org.xml.sax.{InputSource, SAXParseException}
import java.io.{File, FileOutputStream}
import org.http4s.headers.`Content-Type`
import scodec.bits.ByteVector

import scala.annotation.unchecked.uncheckedVariance
import scala.util.control.NonFatal
import scalaz.Liskov.{<~<, refl}
import scalaz.concurrent.Task
import scalaz.stream.{io, process1}
import scalaz.syntax.monad._
import scalaz.{-\/, EitherT, \/, \/-}

import util.UrlFormCodec.{ decode => formDecode }
import util.byteVector._

/** A type that can be used to decode an [[EntityBody]]
  * EntityDecoder is used to attempt to decode an [[EntityBody]] returning the
  * entire resulting A. If an error occurs it will result in a failed Task
  * These are not streaming constructs.
  * @tparam T result type produced by the decoder
  */
sealed trait EntityDecoder[T] { self =>
  /** Attempt to decode the body of the [[Message]] */
  def decode(msg: Message): DecodeResult[T]

  /** The [[MediaRange]]s this [[EntityDecoder]] knows how to handle */
  def consumes: Set[MediaRange]

  /** Make a new [[EntityDecoder]] by mapping the output result */
  def map[T2](f: T => T2): EntityDecoder[T2] = new EntityDecoder[T2] {
    override def consumes: Set[MediaRange] = self.consumes

    override def decode(msg: Message): DecodeResult[T2] = self.decode(msg).map(f)
  }

  def flatMapR[T2](f: T => DecodeResult[T2]): EntityDecoder[T2] = new EntityDecoder[T2] {
    override def decode(msg: Message): DecodeResult[T2] = self.decode(msg).flatMap(f)

    override def consumes: Set[MediaRange] = self.consumes
  }

  /** Combine two [[EntityDecoder]]'s
    *
    * The new [[EntityDecoder]] will first attempt to determine if it can perform the decode,
    * and if not, defer to the second [[EntityDecoder]]
    * @param other backup [[EntityDecoder]]
    */
  def orElse[T2](other: EntityDecoder[T2])(implicit ev: T <~< T2): EntityDecoder[T2] =
    new EntityDecoder.OrDec(widen[T2], other)

  /** true if the [[Message]]s Content-Type header contains a [[MediaType]]
    * this [[EntityDecoder]] knows how to decode */
  def matchesMediaType(msg: Message): Boolean = {
    msg.headers.get(`Content-Type`) match {
      case Some(h) => matchesMediaType(h.mediaType)
      case None => false
    }
  }

  /** true if this [[EntityDecoder]] knows how to decode the provided [[MediaType]] */
  def matchesMediaType(mediaType: MediaType): Boolean =
    consumes.exists(_.satisfiedBy(mediaType))

  // shamelessly stolen from IList
  def widen[B](implicit ev: T <~< B): EntityDecoder[B] =
    ev.subst[({type λ[-α] = EntityDecoder[α @uncheckedVariance] <~< EntityDecoder[B]})#λ](refl)(this)
}

/** EntityDecoder is used to attempt to decode an [[EntityBody]]
  * This companion object provides a way to create `new EntityDecoder`s along
  * with some commonly used instances which can be resolved implicitly.
  */
object EntityDecoder extends EntityDecoderInstances {

  /** summon an implicit [[EntityEncoder]] */
  def apply[T](implicit ev: EntityDecoder[T]): EntityDecoder[T] = ev

  /** Create a new [[EntityDecoder]]
    *
    * The new [[EntityEncoder]] will attempt to decoder messages of type `T`
    */
  def decodeBy[T](r1: MediaRange, rs: MediaRange*)(f: Message => DecodeResult[T]): EntityDecoder[T] = new EntityDecoder[T] {
    override def decode(msg: Message): DecodeResult[T] = {
      try f(msg)
      catch {
        case NonFatal(e) => DecodeResult[T](Task.fail(e))
      }
    }

    override val consumes: Set[MediaRange] = (r1 +: rs).toSet
  }

  private class OrDec[T](a: EntityDecoder[T], b: EntityDecoder[T]) extends EntityDecoder[T] {
    override def decode(msg: Message): DecodeResult[T] = {
      if (a.matchesMediaType(msg)) a.decode(msg)
      else b.decode(msg)
    }

    override val consumes: Set[MediaRange] = a.consumes ++ b.consumes
  }

  /** Helper method which simply gathers the body into a single ByteVector */
  def collectBinary(msg: Message): DecodeResult[ByteVector] =
    DecodeResult.success(msg.body.runFoldMap(identity))

  /** Decodes a message to a String */
  def decodeString(msg: Message): Task[String] = {
    val buff = new StringBuilder
    (msg.body |> process1.fold(buff) { (b, c) => {
      b.append(new String(c.toArray, msg.charset.getOrElse(Charset.`ISO-8859-1`).nioCharset))
    }}).map(_.result()).runLastOr("")
  }
}

/** Implementations of the EntityDecoder instances */
trait EntityDecoderInstances {
  import org.http4s.EntityDecoder._

  /////////////////// Instances //////////////////////////////////////////////

  /** Provides a mechanism to fail decoding */
  def error[T](t: Throwable) = new EntityDecoder[T] {
    override def decode(msg: Message): DecodeResult[T] = {
      DecodeResult(msg.body.kill.run.flatMap(_ => Task.fail(t)))
    }
    override def consumes: Set[MediaRange] = Set.empty
  }<commit_msg>Add `defaultCharset` to the decodeString function<commit_after>package org.http4s

import java.io.{File, FileOutputStream, StringReader}
import javax.xml.parsers.SAXParser

import org.xml.sax.{InputSource, SAXParseException}
import java.io.{File, FileOutputStream}
import org.http4s.headers.`Content-Type`
import scodec.bits.ByteVector

import scala.annotation.unchecked.uncheckedVariance
import scala.util.control.NonFatal
import scalaz.Liskov.{<~<, refl}
import scalaz.concurrent.Task
import scalaz.stream.{io, process1}
import scalaz.syntax.monad._
import scalaz.{-\/, EitherT, \/, \/-}

import util.UrlFormCodec.{ decode => formDecode }
import util.byteVector._

/** A type that can be used to decode an [[EntityBody]]
  * EntityDecoder is used to attempt to decode an [[EntityBody]] returning the
  * entire resulting A. If an error occurs it will result in a failed Task
  * These are not streaming constructs.
  * @tparam T result type produced by the decoder
  */
sealed trait EntityDecoder[T] { self =>
  /** Attempt to decode the body of the [[Message]] */
  def decode(msg: Message): DecodeResult[T]

  /** The [[MediaRange]]s this [[EntityDecoder]] knows how to handle */
  def consumes: Set[MediaRange]

  /** Make a new [[EntityDecoder]] by mapping the output result */
  def map[T2](f: T => T2): EntityDecoder[T2] = new EntityDecoder[T2] {
    override def consumes: Set[MediaRange] = self.consumes

    override def decode(msg: Message): DecodeResult[T2] = self.decode(msg).map(f)
  }

  def flatMapR[T2](f: T => DecodeResult[T2]): EntityDecoder[T2] = new EntityDecoder[T2] {
    override def decode(msg: Message): DecodeResult[T2] = self.decode(msg).flatMap(f)

    override def consumes: Set[MediaRange] = self.consumes
  }

  /** Combine two [[EntityDecoder]]'s
    *
    * The new [[EntityDecoder]] will first attempt to determine if it can perform the decode,
    * and if not, defer to the second [[EntityDecoder]]
    * @param other backup [[EntityDecoder]]
    */
  def orElse[T2](other: EntityDecoder[T2])(implicit ev: T <~< T2): EntityDecoder[T2] =
    new EntityDecoder.OrDec(widen[T2], other)

  /** true if the [[Message]]s Content-Type header contains a [[MediaType]]
    * this [[EntityDecoder]] knows how to decode */
  def matchesMediaType(msg: Message): Boolean = {
    msg.headers.get(`Content-Type`) match {
      case Some(h) => matchesMediaType(h.mediaType)
      case None => false
    }
  }

  /** true if this [[EntityDecoder]] knows how to decode the provided [[MediaType]] */
  def matchesMediaType(mediaType: MediaType): Boolean =
    consumes.exists(_.satisfiedBy(mediaType))

  // shamelessly stolen from IList
  def widen[B](implicit ev: T <~< B): EntityDecoder[B] =
    ev.subst[({type λ[-α] = EntityDecoder[α @uncheckedVariance] <~< EntityDecoder[B]})#λ](refl)(this)
}

/** EntityDecoder is used to attempt to decode an [[EntityBody]]
  * This companion object provides a way to create `new EntityDecoder`s along
  * with some commonly used instances which can be resolved implicitly.
  */
object EntityDecoder extends EntityDecoderInstances {

  /** summon an implicit [[EntityEncoder]] */
  def apply[T](implicit ev: EntityDecoder[T]): EntityDecoder[T] = ev

  /** Create a new [[EntityDecoder]]
    *
    * The new [[EntityEncoder]] will attempt to decoder messages of type `T`
    */
  def decodeBy[T](r1: MediaRange, rs: MediaRange*)(f: Message => DecodeResult[T]): EntityDecoder[T] = new EntityDecoder[T] {
    override def decode(msg: Message): DecodeResult[T] = {
      try f(msg)
      catch {
        case NonFatal(e) => DecodeResult[T](Task.fail(e))
      }
    }

    override val consumes: Set[MediaRange] = (r1 +: rs).toSet
  }

  private class OrDec[T](a: EntityDecoder[T], b: EntityDecoder[T]) extends EntityDecoder[T] {
    override def decode(msg: Message): DecodeResult[T] = {
      if (a.matchesMediaType(msg)) a.decode(msg)
      else b.decode(msg)
    }

    override val consumes: Set[MediaRange] = a.consumes ++ b.consumes
  }

  /** Helper method which simply gathers the body into a single ByteVector */
  def collectBinary(msg: Message): DecodeResult[ByteVector] =
    DecodeResult.success(msg.body.runFoldMap(identity))

  /** Decodes a message to a String */
  def decodeString(msg: Message, defaultCharset: Option[Charset] = None): Task[String] = {
    val buff = new StringBuilder
    (msg.body |> process1.fold(buff) { (b, c) => {
      b.append(new String(c.toArray, (msg.charset orElse defaultCharset getOrElse(Charset.`ISO-8859-1`)).nioCharset))
    }}).map(_.result()).runLastOr("")
  }
}

/** Implementations of the EntityDecoder instances */
trait EntityDecoderInstances {
  import org.http4s.EntityDecoder._

  /////////////////// Instances //////////////////////////////////////////////

  /** Provides a mechanism to fail decoding */
  def error[T](t: Throwable) = new EntityDecoder[T] {
    override def decode(msg: Message): DecodeResult[T] = {
      DecodeResult(msg.body.kill.run.flatMap(_ => Task.fail(t)))
    }
    override def consumes: Set[MediaRange] = Set.empty
  }
{'fraction_non_alphanumeric': 0.14374003189792664, 'fraction_numerical': 0.005382775119617225, 'mean_word_length': 3.782173498570067, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4933406', 'n_tokens_mistral': 3169, 'n_tokens_neox': 3084, 'n_words': 889}