<commit_before>package com.wavesplatform.lang.v1

import java.io.ByteArrayOutputStream
import java.nio.ByteBuffer

import com.wavesplatform.common.state.ByteStr
import cats.implicits._
import com.wavesplatform.lang.v1.compiler.Terms._
import com.wavesplatform.lang.utils.Serialize._
import monix.eval.Coeval

import scala.util.Try

object Serde {
  val E_LONG: Byte    = 0
  val E_BYTES: Byte   = 1
  val E_STRING: Byte  = 2
  val E_IF: Byte      = 3
  val E_BLOCK: Byte   = 4
  val E_REF: Byte     = 5
  val E_TRUE: Byte    = 6
  val E_FALSE: Byte   = 7
  val E_GETTER: Byte  = 8
  val E_FUNCALL: Byte = 9

  val FH_NATIVE: Byte = 0
  val FH_USER: Byte   = 1

  val E_BLOCK_V2 = 10

  val DEC_LET  = 0
  val DEC_FUNC = 1

  def serializeDeclaration(out: ByteArrayOutputStream, dec: DECLARATION, aux: EXPR => Coeval[Unit]): Coeval[Unit] = {
    dec match {
      case LET(name, value) =>
        Coeval.now {
          out.write(DEC_LET)
          out.writeString(name)
        } *> aux(value)
      case FUNC(name, args, body) =>
        Coeval.now {
          out.write(DEC_FUNC)
          out.writeString(name)
          out.writeInt(args.size)
          args.foreach(out.writeString)
        } *> aux(body)
    }
  }

  def deserializeDeclaration(bb: ByteBuffer, aux: => Coeval[EXPR], decType: Byte): Coeval[DECLARATION] = {
    decType match {
      case DEC_LET =>
        for {
          name <- Coeval.now(bb.getString)
          body <- aux
        } yield LET(name, body)
      case DEC_FUNC =>
        for {
          name <- Coeval.now(bb.getString)
          args <- ({
            val argsCnt = bb.getInt
            if (argsCnt <= (bb.limit() - bb.position())/2) {
              Coeval.now(for (_ <- 1 to argsCnt) yield bb.getString)
            } else {
              Coeval.raiseError(new Exception(s"At position ${bb.position()} array of arguments names too big."))
            }
          })
          body <- aux
        } yield FUNC(name, args.toList, body)
    }
  }

  def desAux(bb: ByteBuffer, acc: Coeval[Unit] = Coeval.now(())): Coeval[EXPR] = acc.flatMap { _ =>
    bb.get() match {
      case E_LONG   => Coeval.now(CONST_LONG(bb.getLong))
      case E_BYTES  => Coeval.now(CONST_BYTESTR(ByteStr(bb.getBytes)))
      case E_STRING => Coeval.now(CONST_STRING(bb.getString))
      case E_IF     => (desAux(bb), desAux(bb), desAux(bb)).mapN(IF)
      case E_BLOCK =>
        for {
          name     <- Coeval.now(bb.getString)
          letValue <- desAux(bb)
          body     <- desAux(bb)
        } yield
          LET_BLOCK(
            let = LET(name, letValue),
            body = body
          )
      case E_BLOCK_V2 =>
        for {
          decType <- Coeval.now(bb.get())
          dec     <- deserializeDeclaration(bb, desAux(bb), decType)
          body    <- desAux(bb)
        } yield BLOCK(dec, body)
      case E_REF    => Coeval.now(REF(bb.getString))
      case E_TRUE   => Coeval.now(TRUE)
      case E_FALSE  => Coeval.now(FALSE)
      case E_GETTER => desAux(bb).map(GETTER(_, field = bb.getString))
      case E_FUNCALL =>
        Coeval
          .now((bb.getFunctionHeader, bb.getInt))
          .flatMap {
            case (header, argc) =>
              if (argc <= (bb.limit() - bb.position())/2) {
                val args: List[Coeval[EXPR]] = (1 to argc).map(_ => desAux(bb))(collection.breakOut)
                args.sequence[Coeval, EXPR].map(FUNCTION_CALL(header, _))
              } else {
                Coeval.raiseError(new Exception(s"At position ${bb.position()} array of arguments too big."))
              }
          }
    }
  }

  def deserialize(bytes: Array[Byte], all: Boolean = true): Either[String, (EXPR, Int)] = {
    val bb = ByteBuffer.wrap(bytes)
    val res = Try(desAux(bb).value).toEither.left
      .map(_.getMessage)<commit_msg>SC-112-limit-loops-in-deserializer Fix<commit_after>package com.wavesplatform.lang.v1

import java.io.ByteArrayOutputStream
import java.nio.ByteBuffer

import com.wavesplatform.common.state.ByteStr
import cats.implicits._
import com.wavesplatform.lang.v1.compiler.Terms._
import com.wavesplatform.lang.utils.Serialize._
import monix.eval.Coeval

import scala.util.Try

object Serde {
  val E_LONG: Byte    = 0
  val E_BYTES: Byte   = 1
  val E_STRING: Byte  = 2
  val E_IF: Byte      = 3
  val E_BLOCK: Byte   = 4
  val E_REF: Byte     = 5
  val E_TRUE: Byte    = 6
  val E_FALSE: Byte   = 7
  val E_GETTER: Byte  = 8
  val E_FUNCALL: Byte = 9

  val FH_NATIVE: Byte = 0
  val FH_USER: Byte   = 1

  val E_BLOCK_V2 = 10

  val DEC_LET  = 0
  val DEC_FUNC = 1

  def serializeDeclaration(out: ByteArrayOutputStream, dec: DECLARATION, aux: EXPR => Coeval[Unit]): Coeval[Unit] = {
    dec match {
      case LET(name, value) =>
        Coeval.now {
          out.write(DEC_LET)
          out.writeString(name)
        } *> aux(value)
      case FUNC(name, args, body) =>
        Coeval.now {
          out.write(DEC_FUNC)
          out.writeString(name)
          out.writeInt(args.size)
          args.foreach(out.writeString)
        } *> aux(body)
    }
  }

  def deserializeDeclaration(bb: ByteBuffer, aux: => Coeval[EXPR], decType: Byte): Coeval[DECLARATION] = {
    decType match {
      case DEC_LET =>
        for {
          name <- Coeval.now(bb.getString)
          body <- aux
        } yield LET(name, body)
      case DEC_FUNC =>
        for {
          name <- Coeval.now(bb.getString)
          args <- ({
            val argsCnt = bb.getInt
            if (argsCnt <= (bb.limit() - bb.position())/2) {
              Coeval.now(for (_ <- 1 to argsCnt) yield bb.getString)
            } else {
              Coeval.raiseError(new Exception(s"At position ${bb.position()} array of arguments names too big."))
            }
          })
          body <- aux
        } yield FUNC(name, args.toList, body)
    }
  }

  def desAux(bb: ByteBuffer, acc: Coeval[Unit] = Coeval.now(())): Coeval[EXPR] = acc.flatMap { _ =>
    bb.get() match {
      case E_LONG   => Coeval.now(CONST_LONG(bb.getLong))
      case E_BYTES  => Coeval.now(CONST_BYTESTR(ByteStr(bb.getBytes)))
      case E_STRING => Coeval.now(CONST_STRING(bb.getString))
      case E_IF     => (desAux(bb), desAux(bb), desAux(bb)).mapN(IF)
      case E_BLOCK =>
        for {
          name     <- Coeval.now(bb.getString)
          letValue <- desAux(bb)
          body     <- desAux(bb)
        } yield
          LET_BLOCK(
            let = LET(name, letValue),
            body = body
          )
      case E_BLOCK_V2 =>
        for {
          decType <- Coeval.now(bb.get())
          dec     <- deserializeDeclaration(bb, desAux(bb), decType)
          body    <- desAux(bb)
        } yield BLOCK(dec, body)
      case E_REF    => Coeval.now(REF(bb.getString))
      case E_TRUE   => Coeval.now(TRUE)
      case E_FALSE  => Coeval.now(FALSE)
      case E_GETTER => desAux(bb).map(GETTER(_, field = bb.getString))
      case E_FUNCALL =>
        Coeval
          .now((bb.getFunctionHeader, bb.getInt))
          .flatMap {
            case (header, argc) =>
              if (argc <= (bb.limit() - bb.position())) {
                val args: List[Coeval[EXPR]] = (1 to argc).map(_ => desAux(bb))(collection.breakOut)
                args.sequence[Coeval, EXPR].map(FUNCTION_CALL(header, _))
              } else {
                Coeval.raiseError(new Exception(s"At position ${bb.position()} array of arguments too big."))
              }
          }
    }
  }

  def deserialize(bytes: Array[Byte], all: Boolean = true): Either[String, (EXPR, Int)] = {
    val bb = ByteBuffer.wrap(bytes)
    val res = Try(desAux(bb).value).toEither.left
      .map(_.getMessage)
{'fraction_non_alphanumeric': 0.12690951821386603, 'fraction_numerical': 0.006528267397832615, 'mean_word_length': 2.0252764612954186, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '677914', 'n_tokens_mistral': 2676, 'n_tokens_neox': 2556, 'n_words': 554}