<commit_before>package fs2

import scala.concurrent.ExecutionContext
// import cats.Functor
import cats.Id
import cats.effect.Effect

/** Generic implementations of common 2-argument pipes. */
object pipe2 {

  // NB: Pure instances

  /** Converts a pure `Pipe2` to an effectful `Pipe2` of the specified type. */
  def covary[F[_],I,I2,O](p: Pipe2[Id,I,I2,O]): Pipe2[F,I,I2,O] =
    p.asInstanceOf[Pipe2[F,I,I2,O]]

  private type ZipWithCont[F[_],I,O,R] = Either[(Segment[I,Unit], Stream[F,I]), Stream[F,I]] => Pull[F,O,Option[R]]

  private def zipWith_[F[_],I,I2,O](k1: ZipWithCont[F,I,O,Nothing], k2: ZipWithCont[F,I2,O,Nothing])(f: (I, I2) => O): Pipe2[F,I,I2,O] = {
    def go(t1: (Segment[I,Unit], Stream[F,I]), t2: (Segment[I2,Unit], Stream[F,I2])): Pull[F,O,Option[Nothing]] =
      (t1, t2) match {
        case ((hd1, tl1), (hd2, tl2)) => Pull.segment(hd1.zipWith(hd2)(f)).flatMap {
          case Left(((),extra2)) =>
            tl1.pull.receiveOption {
              case None => k2(Left((extra2, tl2)))
              case Some(tl1) => go(tl1, (extra2, tl2))
            }
          case Right(((),extra1)) =>
            tl2.pull.receiveOption {
              case None => k1(Left((extra1, tl1)))
              case Some(tl2) => go((extra1, tl1), tl2)
            }
        }
      }
    (s1, s2) => s1.pull.receiveOption {
      case Some(s1) => s2.pull.receiveOption {
        case Some(s2) => go(s1, s2)
        case None => k1(Left(s1))
      }
      case None => k2(Right(s2))
    }.close
  }

  /**
   * Determinsitically zips elements with the specified function, terminating
   * when the ends of both branches are reached naturally, padding the left
   * branch with `pad1` and padding the right branch with `pad2` as necessary.
   */
  def zipAllWith[F[_],I,I2,O](pad1: I, pad2: I2)(f: (I, I2) => O): Pipe2[F,I,I2,O] = {
    def cont1(z: Either[(Segment[I,Unit], Stream[F, I]), Stream[F, I]]): Pull[F,O,Option[Nothing]] = {
      def putLeft(c: Chunk[I]) = Pull.output(c.zipWith(Chunk.vector(Vector.fill(c.size)(pad2)))(f).voidResult)
      def contLeft(s: Stream[F,I]): Pull[F,O,Option[Nothing]] = s.pull.receive {
        (hd, tl) => putLeft(hd.toChunk) >> contLeft(tl)
      }
      z match {
        case Left((c, h)) => putLeft(c.toChunk) >> contLeft(h)
        case Right(h)     => contLeft(h)
      }
    }
    def cont2(z: Either[(Segment[I2,Unit], Stream[F, I2]), Stream[F, I2]]): Pull[F,O,Option[Nothing]] = {
      def putRight(c: Chunk[I2]) = Pull.output(Chunk.vector(Vector.fill(c.size)(pad1)).zipWith(c)(f).voidResult)
      def contRight(s: Stream[F,I2]): Pull[F,O,Option[Nothing]] = s.pull.receive {
        (hd, tl) => putRight(hd.toChunk) >> contRight(tl)
      }
      z match {
        case Left((c, h)) => putRight(c.toChunk) >> contRight(h)
        case Right(h)     => contRight(h)
      }<commit_msg>Added a few more doctests<commit_after>package fs2

import scala.concurrent.ExecutionContext
// import cats.Functor
import cats.Id
import cats.effect.Effect

/** Generic implementations of common 2-argument pipes. */
object pipe2 {

  // NB: Pure instances

  /** Converts a pure `Pipe2` to an effectful `Pipe2` of the specified type. */
  def covary[F[_],I,I2,O](p: Pipe2[Id,I,I2,O]): Pipe2[F,I,I2,O] =
    p.asInstanceOf[Pipe2[F,I,I2,O]]

  private type ZipWithCont[F[_],I,O,R] = Either[(Segment[I,Unit], Stream[F,I]), Stream[F,I]] => Pull[F,O,Option[R]]

  private def zipWith_[F[_],I,I2,O](k1: ZipWithCont[F,I,O,Nothing], k2: ZipWithCont[F,I2,O,Nothing])(f: (I, I2) => O): Pipe2[F,I,I2,O] = {
    def go(t1: (Segment[I,Unit], Stream[F,I]), t2: (Segment[I2,Unit], Stream[F,I2])): Pull[F,O,Option[Nothing]] =
      (t1, t2) match {
        case ((hd1, tl1), (hd2, tl2)) => Pull.segment(hd1.zipWith(hd2)(f)).flatMap {
          case Left(((),extra2)) =>
            tl1.pull.receiveOption {
              case None => k2(Left((extra2, tl2)))
              case Some(tl1) => go(tl1, (extra2, tl2))
            }
          case Right(((),extra1)) =>
            tl2.pull.receiveOption {
              case None => k1(Left((extra1, tl1)))
              case Some(tl2) => go((extra1, tl1), tl2)
            }
        }
      }
    (s1, s2) => s1.pull.receiveOption {
      case Some(s1) => s2.pull.receiveOption {
        case Some(s2) => go(s1, s2)
        case None => k1(Left(s1))
      }
      case None => k2(Right(s2))
    }.close
  }

  /**
   * Determinsitically zips elements with the specified function, terminating
   * when the ends of both branches are reached naturally, padding the left
   * branch with `pad1` and padding the right branch with `pad2` as necessary.
   *
   * @example {{{
   * scala> pipe2.zipAllWith(0, 0)(_ + _)(Stream(1, 2, 3), Stream(4, 5, 6, 7)).toList
   * res0: List[Int] = List(5, 7, 9, 7)
   * }}}
   */
  def zipAllWith[F[_],I,I2,O](pad1: I, pad2: I2)(f: (I, I2) => O): Pipe2[F,I,I2,O] = {
    def cont1(z: Either[(Segment[I,Unit], Stream[F, I]), Stream[F, I]]): Pull[F,O,Option[Nothing]] = {
      def putLeft(c: Chunk[I]) = Pull.output(c.zipWith(Chunk.vector(Vector.fill(c.size)(pad2)))(f).voidResult)
      def contLeft(s: Stream[F,I]): Pull[F,O,Option[Nothing]] = s.pull.receive {
        (hd, tl) => putLeft(hd.toChunk) >> contLeft(tl)
      }
      z match {
        case Left((c, h)) => putLeft(c.toChunk) >> contLeft(h)
        case Right(h)     => contLeft(h)
      }
    }
    def cont2(z: Either[(Segment[I2,Unit], Stream[F, I2]), Stream[F, I2]]): Pull[F,O,Option[Nothing]] = {
      def putRight(c: Chunk[I2]) = Pull.output(Chunk.vector(Vector.fill(c.size)(pad1)).zipWith(c)(f).voidResult)
      def contRight(s: Stream[F,I2]): Pull[F,O,Option[Nothing]] = s.pull.receive {
        (hd, tl) => putRight(hd.toChunk) >> contRight(tl)
      }
      z match {
        case Left((c, h)) => putRight(c.toChunk) >> contRight(h)
        case Right(h)     => contRight(h)
      }
{'fraction_non_alphanumeric': 0.19148211243611585, 'fraction_numerical': 0.029471890971039184, 'mean_word_length': 3.021232876712329, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7308944', 'n_tokens_mistral': 2339, 'n_tokens_neox': 2275, 'n_words': 453}