<commit_before>/*
 * Copyright 2019 http4s.org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.http4s.ember.core

import cats.data.OptionT
import cats.effect._
import cats.effect.std.Queue
import cats.syntax.all._
import fs2._
import org.http4s._
import org.http4s.headers.Expires
import org.http4s.implicits._
import org.typelevel.ci._
import scodec.bits.ByteVector

class ParsingSuite extends Http4sSuite {
  object Helpers {
    def stripLines(s: String): String = s.replace("\r\n", "\n")
    def httpifyString(s: String): String = s.replace("\n", "\r\n")

    def taking[F[_]: Concurrent, A](stream: Stream[F, A]): F[F[Option[Chunk[A]]]] =
      for {
        q <- Queue.unbounded[F, Option[Chunk[A]]]
        _ <- stream.chunks.map(Some(_)).evalMap(q.offer(_)).compile.drain
        _ <- q.offer(None)
      } yield q.take

    // Only for Use with Text Requests
    def parseRequestRig[F[_]: Concurrent](s: String): F[Request[F]] = {
      val byteStream: Stream[F, Byte] = Stream
        .emit(s)
        .map(httpifyString)
        .through(fs2.text.utf8.encode[F])

      taking(byteStream).flatMap { read =>
        Parser.Request.parser[F](Int.MaxValue)(Array.emptyByteArray, read).map(_._1)
      }
    }

    def parseResponseRig[F[_]: Concurrent](s: String): Resource[F, Response[F]] = {
      val byteStream: Stream[F, Byte] = Stream
        .emit(s)
        .map(httpifyString)
        .through(fs2.text.utf8.encode[F])

      Resource.eval(
        taking(byteStream).flatMap { read =>
          Parser.Response.parser[F](Int.MaxValue)(Array.emptyByteArray, read).map(_._1)
        }
      )
    }

    def forceScopedParsing[F[_]: Concurrent](s: String): Stream[F, Byte] = {
      val pivotPoint = s.trim().length - 1
      val firstChunk = s.substring(0, pivotPoint).replace("\n", "\r\n")
      val secondChunk = s.substring(pivotPoint, s.length).replace("\n", "\r\n")

      sealed trait StreamState
      case object FirstChunk extends StreamState
      case object SecondChunk extends StreamState
      case object Completed extends StreamState

      def unfoldStream(closed: Ref[F, Boolean]): Stream[F, Byte] = {
        val scope = Resource(((), closed.set(true)).pure[F])
        val noneChunk = OptionT.none[F, (Chunk[Byte], StreamState)].value

        Stream.resource(scope) >>
          Stream.unfoldChunkEval[F, StreamState, Byte](FirstChunk) {
            case FirstChunk =><commit_msg>Randomly rechunk requests and responses in ParserSuite<commit_after>/*
 * Copyright 2019 http4s.org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.http4s.ember.core

import cats.data.OptionT
import cats.effect._
import cats.effect.std.Queue
import cats.syntax.all._
import fs2._
import org.http4s._
import org.http4s.headers.Expires
import org.http4s.implicits._
import org.typelevel.ci._
import scodec.bits.ByteVector

class ParsingSuite extends Http4sSuite {
  object Helpers {
    def stripLines(s: String): String = s.replace("\r\n", "\n")
    def httpifyString(s: String): String = s.replace("\n", "\r\n")

    def taking[F[_]: Concurrent, A](stream: Stream[F, A]): F[F[Option[Chunk[A]]]] =
      for {
        q <- Queue.unbounded[F, Option[Chunk[A]]]
        _ <- stream.chunks.map(Some(_)).evalMap(q.offer(_)).compile.drain
        _ <- q.offer(None)
      } yield q.take

    // Only for Use with Text Requests
    def parseRequestRig[F[_]: Concurrent](s: String): F[Request[F]] = {
      val byteStream: Stream[F, Byte] = Stream
        .emit(s)
        .map(httpifyString)
        .through(fs2.text.utf8.encode[F])
        .rechunkRandomly(0.1, 0.5)

      taking(byteStream).flatMap { read =>
        Parser.Request.parser[F](Int.MaxValue)(Array.emptyByteArray, read).map(_._1)
      }
    }

    def parseResponseRig[F[_]: Concurrent](s: String): Resource[F, Response[F]] = {
      val byteStream: Stream[F, Byte] = Stream
        .emit(s)
        .map(httpifyString)
        .through(fs2.text.utf8.encode[F])
        .rechunkRandomly(0.1, 0.5)

      Resource.eval(
        taking(byteStream).flatMap { read =>
          Parser.Response.parser[F](Int.MaxValue)(Array.emptyByteArray, read).map(_._1)
        }
      )
    }

    def forceScopedParsing[F[_]: Concurrent](s: String): Stream[F, Byte] = {
      val pivotPoint = s.trim().length - 1
      val firstChunk = s.substring(0, pivotPoint).replace("\n", "\r\n")
      val secondChunk = s.substring(pivotPoint, s.length).replace("\n", "\r\n")

      sealed trait StreamState
      case object FirstChunk extends StreamState
      case object SecondChunk extends StreamState
      case object Completed extends StreamState

      def unfoldStream(closed: Ref[F, Boolean]): Stream[F, Byte] = {
        val scope = Resource(((), closed.set(true)).pure[F])
        val noneChunk = OptionT.none[F, (Chunk[Byte], StreamState)].value

        Stream.resource(scope) >>
          Stream.unfoldChunkEval[F, StreamState, Byte](FirstChunk) {
            case FirstChunk =>
{'fraction_non_alphanumeric': 0.13097817030494918, 'fraction_numerical': 0.00899850024995834, 'mean_word_length': 3.080217539089055, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6080363', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1834, 'n_words': 486}