<commit_before>package bifrost.state

import java.io.File
import java.time.Instant
import bifrost.forging.ForgingSettings

import bifrost.bfr.BFR
import bifrost.history.BifrostHistory
import bifrost.blocks.BifrostBlock
import bifrost.program.Program
import bifrost.exceptions.TransactionValidationException
import bifrost.scorexMod.{GenericBoxMinimalState, GenericStateChanges}
import bifrost.transaction._
import bifrost.transaction.box._
import bifrost.transaction.box.proposition.MofNProposition
import bifrost.transaction.proof.MultiSignature25519
import com.google.common.primitives.Longs
import io.circe.Json
import io.iohk.iodb.{ByteArrayWrapper, LSMStore}
import bifrost.crypto.hash.FastCryptographicHash
import bifrost.forging.ForgingSettings
import bifrost.settings.Settings
import bifrost.srb.{SBR, StateBoxRegistry}
import bifrost.transaction.bifrostTransaction.{AssetRedemption, _}
import bifrost.transaction.box.proposition.{ProofOfKnowledgeProposition, PublicKey25519Proposition}
import bifrost.transaction.state.MinimalState.VersionTag
import bifrost.transaction.state.PrivateKey25519
import bifrost.utils.ScorexLogging
import scorex.crypto.encode.Base58

import scala.util.{Failure, Success, Try}

case class BifrostTransactionChanges(toRemove: Set[BifrostBox], toAppend: Set[BifrostBox], minerReward: Long)

case class BifrostStateChanges(override val boxIdsToRemove: Set[Array[Byte]],
                               override val toAppend: Set[BifrostBox], timestamp: Long)
  extends GenericStateChanges[Any, ProofOfKnowledgeProposition[PrivateKey25519], BifrostBox](boxIdsToRemove, toAppend)

/**
  * BifrostState is a data structure which deterministically defines whether an arbitrary transaction is valid and so
  * applicable to it or not. Also has methods to get a closed box, to apply a persistent modifier, and to roll back
  * to a previous version.
  *
  * @param storage           singleton Iodb storage instance
  * @param version           blockId used to identify each block. Also used for rollback
  * @param timestamp         timestamp of the block that results in this state
  * @param history           Main box storage
  * @param stateBoxRegistry  Separate box set for program StateBoxes
  */
case class BifrostState(storage: LSMStore, override val version: VersionTag, timestamp: Long, history: BifrostHistory, sbr: SBR = null, bfr: BFR = null, nodeKeys: Set[ByteArrayWrapper] = null)
  extends GenericBoxMinimalState[Any, ProofOfKnowledgeProposition[PrivateKey25519],
    BifrostBox, BifrostTransaction, BifrostBlock, BifrostState] with ScorexLogging {

  override type NVCT = BifrostState
  type P = BifrostState.P
  type T = BifrostState.T
  type TX = BifrostState.TX
  type BX = BifrostState.BX
  type BPMOD = BifrostState.BPMOD
  type GSC = BifrostState.GSC
  type BSC = BifrostState.BSC



  override def semanticValidity(tx: BifrostTransaction): Try[Unit] = BifrostState.semanticValidity(tx)

  private def lastVersionString = storage.lastVersionID.map(v => Base58.encode(v.data)).getOrElse("None")

  private def getProfileBox(prop: PublicKey25519Proposition, field: String): Try[ProfileBox] = {
    storage.get(ByteArrayWrapper(ProfileBox.idFromBox(prop, field))) match {
      case Some(bytes) => ProfileBoxSerializer.parseBytes(bytes.data)
      case None => Failure(new Exception(s"Couldn't find profile box for ${
        Base58.encode(prop
          .pubKeyBytes)
      } with field <$field>"))
    }
  }

  override def closedBox(boxId: Array[Byte]): Option[BX] =
    storage.get(ByteArrayWrapper(boxId))
      .map(_.data)
      .map(BifrostBoxSerializer.parseBytes)
      .flatMap(_.toOption)

  //TODO rollback SBR and BFR here as well
//  override def rollbackTo(version: VersionTag): Try[NVCT] = Try {
//    if (storage.lastVersionID.exists(_.data sameElements version)) {
//      this
//    } else {
//      log.debug(s"Rollback BifrostState to ${Base58.encode(version)} from version $lastVersionString")
//      storage.rollback(ByteArrayWrapper(version))
//      val timestamp: Long = Longs.fromByteArray(storage.get(ByteArrayWrapper(FastCryptographicHash("timestamp"
//        .getBytes))).get
//        .data)
//      BifrostState(storage, version, timestamp, history, stateBoxRegistry, bfr)
//    }
//  }

  override def rollbackTo(version: VersionTag): Try[NVCT] = Try {
    if (storage.lastVersionID.exists(_.data sameElements version)) {
      this
    } else {
      log.debug(s"Rollback BifrostState to ${Base58.encode(version)} from version $lastVersionString")
      storage.rollback(ByteArrayWrapper(version))
      bfr.rollbackTo(version, storage)
      sbr.rollbackTo(version, storage)
      val timestamp: Long = Longs.fromByteArray(storage.get(ByteArrayWrapper(FastCryptographicHash("timestamp"
        .getBytes))).get
        .data)
      BifrostState(storage, version, timestamp, history, sbr, bfr)
    }
  }

  override def changes(mod: BPMOD): Try[GSC] = BifrostState.changes(mod)

//  override def applyChanges(changes: GSC, newVersion: VersionTag): Try[NVCT] = Try {
//
//    val boxesToAdd = changes.toAppend.map(b => ByteArrayWrapper(b.id) -> ByteArrayWrapper(b.bytes))
//
//    /* This seeks to avoid the scenario where there is remove and then update of the same keys */
//    val boxIdsToRemove = (changes.boxIdsToRemove -- boxesToAdd.map(_._1.data)).map(ByteArrayWrapper.apply)
//
//    log.debug(s"Update BifrostState from version $lastVersionString to version ${Base58.encode(newVersion)}. " +
//      s"Removing boxes with ids ${boxIdsToRemove.map(b => Base58.encode(b.data))}, " +
//      s"adding boxes ${boxesToAdd.map(b => Base58.encode(b._1.data))}")
//
//    val timestamp: Long = changes.asInstanceOf[BifrostStateChanges].timestamp
//
//    if (storage.lastVersionID.isDefined) boxIdsToRemove.foreach(i => require(closedBox(i.data).isDefined))
//
//    //BFR must be updated before state since it uses the boxes from state that are being removed in the update
//    if(bfr != null) bfr.updateFromState(newVersion, changes)
//    if(sbr != null) sbr.updateFromState(newVersion, changes)
//
//
//    storage.update(
//      ByteArrayWrapper(newVersion),
//      boxIdsToRemove,
//      boxesToAdd + (ByteArrayWrapper(FastCryptographicHash("timestamp".getBytes)) -> ByteArrayWrapper(Longs.toByteArray(
//        timestamp)))
//    )
//
//    val newSt = BifrostState(storage, newVersion, timestamp, history, sbr, bfr)
//
//    boxIdsToRemove.foreach(box => require(newSt.closedBox(box.data).isEmpty, s"Box $box is still in state"))
//    newSt
//
//  }

  override def applyChanges(changes: GSC, newVersion: VersionTag): Try[NVCT] = Try {

//    val boxesToAdd = changes.toAppend.map(b => ByteArrayWrapper(b.id) -> ByteArrayWrapper(b.bytes))

    //Filtering boexs pertaining to public keys specified in settings file
    val filteredBoxesToAdd =
      if(nodeKeys != null)
        changes.toAppend
        .filter(b => (b.isInstanceOf[BifrostPublic25519NoncedBox] && nodeKeys.contains(ByteArrayWrapper(b.asInstanceOf[BifrostPublic25519NoncedBox].proposition.pubKeyBytes))) ||
          (b.isInstanceOf[BifrostProgramBox] && nodeKeys.contains(ByteArrayWrapper(b.asInstanceOf[BifrostProgramBox].proposition.pubKeyBytes))))
      else
        changes.toAppend

    val filteredBoxIdsToRemove =
      if(nodeKeys != null)
        changes.boxIdsToRemove
        .flatMap(closedBox(_))
        .filter(b => (b.isInstanceOf[BifrostPublic25519NoncedBox] && nodeKeys.contains(ByteArrayWrapper(b.asInstanceOf[BifrostPublic25519NoncedBox].proposition.pubKeyBytes)))  ||
          (b.isInstanceOf[BifrostProgramBox]&& nodeKeys.contains(ByteArrayWrapper(b.asInstanceOf[BifrostProgramBox].proposition.pubKeyBytes))))
        .map(b => b.id)
      else
        changes.boxIdsToRemove

    val boxesToAdd = filteredBoxesToAdd
      .map(b => ByteArrayWrapper(b.id) -> ByteArrayWrapper(b.bytes))

    /* This seeks to avoid the scenario where there is remove and then update of the same keys */
    val boxIdsToRemove = (filteredBoxIdsToRemove -- boxesToAdd.map(_._1.data)).map(ByteArrayWrapper.apply)

    log.debug(s"Update BifrostState from version $lastVersionString to version ${Base58.encode(newVersion)}. " +
      s"Removing boxes with ids ${boxIdsToRemove.map(b => Base58.encode(b.data))}, " +
      s"adding boxes ${boxesToAdd.map(b => Base58.encode(b._1.data))}")

    val timestamp: Long = changes.asInstanceOf[BifrostStateChanges].timestamp

    if (storage.lastVersionID.isDefined) boxIdsToRemove.foreach(i => require(closedBox(i.data).isDefined))

    //BFR must be updated before state since it uses the boxes from state that are being removed in the update
    if(bfr != null) bfr.updateFromState(newVersion, filteredBoxIdsToRemove, filteredBoxesToAdd)
    if(sbr != null) sbr.updateFromState(newVersion, filteredBoxIdsToRemove, filteredBoxesToAdd)


    storage.update(
      ByteArrayWrapper(newVersion),<commit_msg>Change to filtering process for boxes to accommodate all box types<commit_after>package bifrost.state

import java.io.File
import java.time.Instant
import bifrost.forging.ForgingSettings

import bifrost.bfr.BFR
import bifrost.history.BifrostHistory
import bifrost.blocks.BifrostBlock
import bifrost.program.Program
import bifrost.exceptions.TransactionValidationException
import bifrost.scorexMod.{GenericBoxMinimalState, GenericStateChanges}
import bifrost.transaction._
import bifrost.transaction.box._
import bifrost.transaction.box.proposition.MofNProposition
import bifrost.transaction.proof.MultiSignature25519
import com.google.common.primitives.Longs
import io.circe.Json
import io.iohk.iodb.{ByteArrayWrapper, LSMStore}
import bifrost.crypto.hash.FastCryptographicHash
import bifrost.forging.ForgingSettings
import bifrost.settings.Settings
import bifrost.srb.{SBR, StateBoxRegistry}
import bifrost.transaction.bifrostTransaction.{AssetRedemption, _}
import bifrost.transaction.box.proposition.{ProofOfKnowledgeProposition, PublicKey25519Proposition}
import bifrost.transaction.state.MinimalState.VersionTag
import bifrost.transaction.state.PrivateKey25519
import bifrost.utils.ScorexLogging
import scorex.crypto.encode.Base58

import scala.util.{Failure, Success, Try}

case class BifrostTransactionChanges(toRemove: Set[BifrostBox], toAppend: Set[BifrostBox], minerReward: Long)

case class BifrostStateChanges(override val boxIdsToRemove: Set[Array[Byte]],
                               override val toAppend: Set[BifrostBox], timestamp: Long)
  extends GenericStateChanges[Any, ProofOfKnowledgeProposition[PrivateKey25519], BifrostBox](boxIdsToRemove, toAppend)

/**
  * BifrostState is a data structure which deterministically defines whether an arbitrary transaction is valid and so
  * applicable to it or not. Also has methods to get a closed box, to apply a persistent modifier, and to roll back
  * to a previous version.
  *
  * @param storage           singleton Iodb storage instance
  * @param version           blockId used to identify each block. Also used for rollback
  * @param timestamp         timestamp of the block that results in this state
  * @param history           Main box storage
  * @param stateBoxRegistry  Separate box set for program StateBoxes
  */
case class BifrostState(storage: LSMStore, override val version: VersionTag, timestamp: Long, history: BifrostHistory, sbr: SBR = null, bfr: BFR = null, nodeKeys: Set[ByteArrayWrapper] = null)
  extends GenericBoxMinimalState[Any, ProofOfKnowledgeProposition[PrivateKey25519],
    BifrostBox, BifrostTransaction, BifrostBlock, BifrostState] with ScorexLogging {

  override type NVCT = BifrostState
  type P = BifrostState.P
  type T = BifrostState.T
  type TX = BifrostState.TX
  type BX = BifrostState.BX
  type BPMOD = BifrostState.BPMOD
  type GSC = BifrostState.GSC
  type BSC = BifrostState.BSC



  override def semanticValidity(tx: BifrostTransaction): Try[Unit] = BifrostState.semanticValidity(tx)

  private def lastVersionString = storage.lastVersionID.map(v => Base58.encode(v.data)).getOrElse("None")

  private def getProfileBox(prop: PublicKey25519Proposition, field: String): Try[ProfileBox] = {
    storage.get(ByteArrayWrapper(ProfileBox.idFromBox(prop, field))) match {
      case Some(bytes) => ProfileBoxSerializer.parseBytes(bytes.data)
      case None => Failure(new Exception(s"Couldn't find profile box for ${
        Base58.encode(prop
          .pubKeyBytes)
      } with field <$field>"))
    }
  }

  override def closedBox(boxId: Array[Byte]): Option[BX] =
    storage.get(ByteArrayWrapper(boxId))
      .map(_.data)
      .map(BifrostBoxSerializer.parseBytes)
      .flatMap(_.toOption)

  //TODO rollback SBR and BFR here as well
//  override def rollbackTo(version: VersionTag): Try[NVCT] = Try {
//    if (storage.lastVersionID.exists(_.data sameElements version)) {
//      this
//    } else {
//      log.debug(s"Rollback BifrostState to ${Base58.encode(version)} from version $lastVersionString")
//      storage.rollback(ByteArrayWrapper(version))
//      val timestamp: Long = Longs.fromByteArray(storage.get(ByteArrayWrapper(FastCryptographicHash("timestamp"
//        .getBytes))).get
//        .data)
//      BifrostState(storage, version, timestamp, history, stateBoxRegistry, bfr)
//    }
//  }

  override def rollbackTo(version: VersionTag): Try[NVCT] = Try {
    if (storage.lastVersionID.exists(_.data sameElements version)) {
      this
    } else {
      log.debug(s"Rollback BifrostState to ${Base58.encode(version)} from version $lastVersionString")
      storage.rollback(ByteArrayWrapper(version))
      bfr.rollbackTo(version, storage)
      sbr.rollbackTo(version, storage)
      val timestamp: Long = Longs.fromByteArray(storage.get(ByteArrayWrapper(FastCryptographicHash("timestamp"
        .getBytes))).get
        .data)
      BifrostState(storage, version, timestamp, history, sbr, bfr)
    }
  }

  override def changes(mod: BPMOD): Try[GSC] = BifrostState.changes(mod)

//  override def applyChanges(changes: GSC, newVersion: VersionTag): Try[NVCT] = Try {
//
//    val boxesToAdd = changes.toAppend.map(b => ByteArrayWrapper(b.id) -> ByteArrayWrapper(b.bytes))
//
//    /* This seeks to avoid the scenario where there is remove and then update of the same keys */
//    val boxIdsToRemove = (changes.boxIdsToRemove -- boxesToAdd.map(_._1.data)).map(ByteArrayWrapper.apply)
//
//    log.debug(s"Update BifrostState from version $lastVersionString to version ${Base58.encode(newVersion)}. " +
//      s"Removing boxes with ids ${boxIdsToRemove.map(b => Base58.encode(b.data))}, " +
//      s"adding boxes ${boxesToAdd.map(b => Base58.encode(b._1.data))}")
//
//    val timestamp: Long = changes.asInstanceOf[BifrostStateChanges].timestamp
//
//    if (storage.lastVersionID.isDefined) boxIdsToRemove.foreach(i => require(closedBox(i.data).isDefined))
//
//    //BFR must be updated before state since it uses the boxes from state that are being removed in the update
//    if(bfr != null) bfr.updateFromState(newVersion, changes)
//    if(sbr != null) sbr.updateFromState(newVersion, changes)
//
//
//    storage.update(
//      ByteArrayWrapper(newVersion),
//      boxIdsToRemove,
//      boxesToAdd + (ByteArrayWrapper(FastCryptographicHash("timestamp".getBytes)) -> ByteArrayWrapper(Longs.toByteArray(
//        timestamp)))
//    )
//
//    val newSt = BifrostState(storage, newVersion, timestamp, history, sbr, bfr)
//
//    boxIdsToRemove.foreach(box => require(newSt.closedBox(box.data).isEmpty, s"Box $box is still in state"))
//    newSt
//
//  }

  override def applyChanges(changes: GSC, newVersion: VersionTag): Try[NVCT] = Try {

//    val boxesToAdd = changes.toAppend.map(b => ByteArrayWrapper(b.id) -> ByteArrayWrapper(b.bytes))

    //Filtering boexs pertaining to public keys specified in settings file
    val filteredBoxesToAdd =
      if(nodeKeys != null)
        changes.toAppend
//        .filter(b => (b.isInstanceOf[BifrostPublic25519NoncedBox] && nodeKeys.contains(ByteArrayWrapper(b.asInstanceOf[BifrostPublic25519NoncedBox].proposition.pubKeyBytes))) ||
//          (b.isInstanceOf[BifrostProgramBox] && nodeKeys.contains(ByteArrayWrapper(b.asInstanceOf[BifrostProgramBox].proposition.pubKeyBytes))))
        .filter(b => nodeKeys.contains(ByteArrayWrapper(b.proposition.bytes)))
      else
        changes.toAppend

    val filteredBoxIdsToRemove =
      if(nodeKeys != null)
        changes.boxIdsToRemove
        .flatMap(closedBox(_))
//        .filter(b => (b.isInstanceOf[BifrostPublic25519NoncedBox] && nodeKeys.contains(ByteArrayWrapper(b.asInstanceOf[BifrostPublic25519NoncedBox].proposition.pubKeyBytes)))  ||
//          (b.isInstanceOf[BifrostProgramBox]&& nodeKeys.contains(ByteArrayWrapper(b.asInstanceOf[BifrostProgramBox].proposition.pubKeyBytes))))
          .filter(b => nodeKeys.contains(ByteArrayWrapper(b.proposition.bytes)))
          .map(b => b.id)
      else
        changes.boxIdsToRemove

    val boxesToAdd = filteredBoxesToAdd
      .map(b => ByteArrayWrapper(b.id) -> ByteArrayWrapper(b.bytes))

    /* This seeks to avoid the scenario where there is remove and then update of the same keys */
    val boxIdsToRemove = (filteredBoxIdsToRemove -- boxesToAdd.map(_._1.data)).map(ByteArrayWrapper.apply)

    log.debug(s"Update BifrostState from version $lastVersionString to version ${Base58.encode(newVersion)}. " +
      s"Removing boxes with ids ${boxIdsToRemove.map(b => Base58.encode(b.data))}, " +
      s"adding boxes ${boxesToAdd.map(b => Base58.encode(b._1.data))}")

    val timestamp: Long = changes.asInstanceOf[BifrostStateChanges].timestamp

    if (storage.lastVersionID.isDefined) boxIdsToRemove.foreach(i => require(closedBox(i.data).isDefined))

    //BFR must be updated before state since it uses the boxes from state that are being removed in the update
    if(bfr != null) bfr.updateFromState(newVersion, filteredBoxIdsToRemove, filteredBoxesToAdd)
    if(sbr != null) sbr.updateFromState(newVersion, filteredBoxIdsToRemove, filteredBoxesToAdd)


    storage.update(
      ByteArrayWrapper(newVersion),
{'fraction_non_alphanumeric': 0.10845842796211157, 'fraction_numerical': 0.008419653243228272, 'mean_word_length': 4.207383905393712, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '246594', 'n_tokens_mistral': 5640, 'n_tokens_neox': 5430, 'n_words': 1215}