<commit_before>package hmda.api.http.institutions.submissions

import akka.actor.{ ActorRef, ActorSystem }
import akka.event.LoggingAdapter
import akka.http.scaladsl.marshalling.ToResponseMarshallable
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.server.Directives._
import akka.pattern.ask
import akka.stream.ActorMaterializer
import akka.util.Timeout
import hmda.api.http.{ HmdaCustomDirectives, ValidationErrorConverter }
import hmda.api.model._
import hmda.api.protocol.processing.{ ApiErrorProtocol, EditResultsProtocol, InstitutionProtocol }
import hmda.model.fi.{ Submission, SubmissionId, SubmissionStatus }
import hmda.model.validation.{ Macro, Quality, ValidationErrorType }
import hmda.persistence.messages.CommonMessages.GetState
import hmda.persistence.HmdaSupervisor.{ FindProcessingActor, FindSubmissions }
import hmda.persistence.institutions.SubmissionPersistence
import hmda.persistence.institutions.SubmissionPersistence.GetSubmissionById
import hmda.persistence.processing.{ HmdaFileValidator, SubmissionManager }
import hmda.persistence.processing.HmdaFileValidator._
import scala.concurrent.{ ExecutionContext, Future }
import scala.util.matching.Regex
import scala.util.{ Failure, Success }

trait SubmissionEditPaths
    extends InstitutionProtocol
    with ApiErrorProtocol
    with EditResultsProtocol
    with HmdaCustomDirectives
    with RequestVerificationUtils
    with ValidationErrorConverter {

  implicit val system: ActorSystem
  implicit val materializer: ActorMaterializer
  val log: LoggingAdapter

  implicit val timeout: Timeout

  // institutions/<institutionId>/filings/<period>/submissions/<seqNr>/edits
  def submissionEditsPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits") { (period, seqNr) =>
      timedGet { uri =>
        completeVerified(supervisor, institutionId, period, seqNr, uri) {
          val fState = getStatusAndValidationState(supervisor, SubmissionId(institutionId, period, seqNr))
          onComplete(fState) {
            case Success((vs, status)) =>
              val s = EditCollection(editInfos(vs.syntacticalErrors))
              val v = EditCollection(editInfos(vs.validityErrors))
              val q = VerifiableEditCollection(vs.qualityVerified, editInfos(vs.qualityErrors))
              val m = VerifiableEditCollection(vs.macroVerified, editInfos(vs.larMacro))
              complete(ToResponseMarshallable(SummaryEditResults(s, v, q, m, status)))
            case Failure(error) => completeWithInternalError(uri, error)
          }
        }
      }
    }

  // institutions/<institutionId>/filings/<period>/submissions/<seqNr>/edits/csv
  def submissionEditCsvPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits" / "csv") { (period, seqNr) =>
      timedGet { uri =>
        completeVerified(supervisor, institutionId, period, seqNr, uri) {
          val fValidationState = getValidationState(supervisor, institutionId, period, seqNr)
          onComplete(fValidationState) {
            case Success(validationState) =>
              val csv: String = validationErrorsToCsvResults(validationState)
              complete(ToResponseMarshallable(csv))
            case Failure(error) => completeWithInternalError(uri, error)
          }
        }
      }
    }

  // institutions/<institutionId>/filings/<period>/submissions/<seqNr>/edits/<editType>
  private val svqmRegex = new Regex("syntactical|validity|quality|macro")
  def submissionSingleEditPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits" / svqmRegex) { (period, seqNr, editType) =>
      timedGet { uri =>
        completeVerified(supervisor, institutionId, period, seqNr, uri) {
          val fState = getStatusAndValidationState(supervisor, SubmissionId(institutionId, period, seqNr))
          onComplete(fState) {
            case Success((vs, status)) =>
              val edits = editInfos(editsOfType(editType, vs))
              complete(ToResponseMarshallable(SingleTypeEditResults(edits, status)))
            case Failure(error) => completeWithInternalError(uri, error)
          }
        }
      }
    }

  // /institutions/<institution>/filings/<period>/submissions/<submissionId>/edits/<edit>
  private val editNameRegex: Regex = new Regex("""[SVQ]\d\d\d""")

  def editFailureDetailsPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits" / editNameRegex) { (period, seqNr, editName) =>
      timedGet { uri =>
        completeVerified(supervisor, institutionId, period, seqNr, uri) {
          parameters('page.as[Int] ? 1) { (page: Int) =>
            val fValidator: Future[ActorRef] = fHmdaFileValidator(supervisor, SubmissionId(institutionId, period, seqNr))
            val fPaginatedErrors: Future[(PaginatedErrors, HmdaFileValidationState)] = for {
              va <- fValidator
              vs <- (va ? GetState).mapTo[HmdaFileValidationState]
              p <- (va ? GetNamedErrorResultsPaginated(editName, page)).mapTo[PaginatedErrors]
            } yield (p, vs)

            onComplete(fPaginatedErrors) {
              case Success((errorCollection, vs)) =>
                val rows: Seq[EditResultRow] = errorCollection.errors.map(validationErrorToResultRow(_, vs))
                val result = EditResult(editName, rows, uri.path.toString, page, errorCollection.totalErrors)
                complete(ToResponseMarshallable(result))
              case Failure(error) => completeWithInternalError(uri, error)
            }
          }
        }
      }
    }

  // institutions/<institutionId>/filings/<period>/submissions/<seqNr>/edits/quality|macro
  private val editTypeRegex = new Regex("quality|macro")
  def verifyEditsPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits" / editTypeRegex) { (period, seqNr, verificationType) =>
      timedPost { uri =>
        entity(as[EditsVerification]) { verification =>
          completeVerified(supervisor, institutionId, period, seqNr, uri) {
            val verified = verification.verified
            val fSubmissionManager = (supervisor ? FindProcessingActor(SubmissionManager.name, SubmissionId(institutionId, period, seqNr))).mapTo[ActorRef]
            val subId = SubmissionId(institutionId, period, seqNr)
            val fValidator = fHmdaFileValidator(supervisor, subId)
            val editType: ValidationErrorType = if (verificationType == "quality") Quality else Macro

            val fVerification = for {
              replyTo <- fSubmissionManager
              va <- fValidator
              v <- (va ? VerifyEdits(editType, verified, replyTo)).mapTo[SubmissionStatus]
            } yield v

            onComplete(fVerification) {
              case Success(status) =>
                complete(ToResponseMarshallable(EditsVerifiedResponse(verified, status)))
              case Failure(error) => completeWithInternalError(uri, error)
            }
          }
        }
      }
    }

  /////// Helper Methods ///////
  private def fHmdaFileValidator(supervisor: ActorRef, submissionId: SubmissionId): Future[ActorRef] =
    (supervisor ? FindProcessingActor(HmdaFileValidator.name, submissionId)).mapTo[ActorRef]

  private def getStatusAndValidationState(supervisor: ActorRef, submissionId: SubmissionId)(implicit ec: ExecutionContext): Future[(HmdaFileValidationState, SubmissionStatus)] = {
    val fValidator = fHmdaFileValidator(supervisor, submissionId)
    val fSubmissionsActor = (supervisor ? FindSubmissions(SubmissionPersistence.name, submissionId.institutionId, submissionId.period)).mapTo[ActorRef]

    for {
      sa <- fSubmissionsActor
      s <- (sa ? GetSubmissionById(submissionId)).mapTo[Submission]
      va <- fValidator
      vs <- (va ? GetState).mapTo[HmdaFileValidationState]
    } yield (vs, s.status)

  }

  private def getValidationState(supervisor: ActorRef, institutionId: String, period: String, seqNr: Int)(implicit ec: ExecutionContext): Future[HmdaFileValidationState] = {
    val fValidator = fHmdaFileValidator(supervisor, SubmissionId(institutionId, period, seqNr))
    for {
      s <- fValidator
      xs <- (s ? GetState).mapTo[HmdaFileValidationState]
    } yield xs
  }

}
<commit_msg>Use event stream methods for validation error API<commit_after>package hmda.api.http.institutions.submissions

import akka.NotUsed
import akka.actor.{ ActorRef, ActorSystem }
import akka.event.LoggingAdapter
import akka.http.scaladsl.marshalling.ToResponseMarshallable
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.server.Directives._
import akka.pattern.ask
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Source
import akka.util.Timeout
import hmda.api.http.{ HmdaCustomDirectives, ValidationErrorConverter }
import hmda.api.model._
import hmda.api.protocol.processing.{ ApiErrorProtocol, EditResultsProtocol, InstitutionProtocol }
import hmda.model.fi.{ Submission, SubmissionId, SubmissionStatus }
import hmda.model.validation.{ Macro, Quality, ValidationErrorType }
import hmda.persistence.messages.CommonMessages.{ Event, GetState }
import hmda.persistence.processing.HmdaQuery._
import hmda.persistence.HmdaSupervisor.{ FindProcessingActor, FindSubmissions }
import hmda.persistence.institutions.SubmissionPersistence
import hmda.persistence.institutions.SubmissionPersistence.GetSubmissionById
import hmda.persistence.processing.{ HmdaFileValidator, SubmissionManager }
import hmda.persistence.processing.HmdaFileValidator._

import scala.concurrent.{ ExecutionContext, Future }
import scala.util.matching.Regex
import scala.util.{ Failure, Success }

trait SubmissionEditPaths
    extends InstitutionProtocol
    with ApiErrorProtocol
    with EditResultsProtocol
    with HmdaCustomDirectives
    with RequestVerificationUtils
    with ValidationErrorConverter {

  implicit val system: ActorSystem
  implicit val materializer: ActorMaterializer
  val log: LoggingAdapter

  implicit val timeout: Timeout

  // institutions/<institutionId>/filings/<period>/submissions/<seqNr>/edits
  def submissionEditsPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits") { (period, seqNr) =>
      timedGet { uri =>
        completeVerified(supervisor, institutionId, period, seqNr, uri) {
          val submissionId = SubmissionId(institutionId, period, seqNr)
          val eventStream = validationEventStream(submissionId)

          val fValidator = fHmdaFileValidator(supervisor, submissionId)
          val fSubmissionsActor = (supervisor ? FindSubmissions(SubmissionPersistence.name, submissionId.institutionId, submissionId.period)).mapTo[ActorRef]

          val fState = for {
            sa <- fSubmissionsActor
            submission <- (sa ? GetSubmissionById(submissionId)).mapTo[Submission]
            va <- fValidator
            vs <- (va ? GetState).mapTo[HmdaFileValidationState]
            s <- editInfosF("syntactical", eventStream)
            v <- editInfosF("validity", eventStream)
            q <- editInfosF("quality", eventStream)
            m <- editInfosF("macro", eventStream)
          } yield (vs, submission.status, List(s, v, q, m))

          onComplete(fState) {
            case Success((vs, status, infoList)) =>
              val s = EditCollection(infoList(0))
              val v = EditCollection(infoList(1))
              val q = VerifiableEditCollection(vs.qualityVerified, infoList(2))
              val m = VerifiableEditCollection(vs.macroVerified, infoList(3))
              complete(ToResponseMarshallable(SummaryEditResults(s, v, q, m, status)))
            case Failure(error) => completeWithInternalError(uri, error)
          }
        }
      }
    }

  // institutions/<institutionId>/filings/<period>/submissions/<seqNr>/edits/csv
  def submissionEditCsvPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits" / "csv") { (period, seqNr) =>
      timedGet { uri =>
        completeVerified(supervisor, institutionId, period, seqNr, uri) {
          val fValidationState = getValidationState(supervisor, institutionId, period, seqNr)
          onComplete(fValidationState) {
            case Success(validationState) =>
              val csv: String = validationErrorsToCsvResults(validationState)
              complete(ToResponseMarshallable(csv))
            case Failure(error) => completeWithInternalError(uri, error)
          }
        }
      }
    }

  // institutions/<institutionId>/filings/<period>/submissions/<seqNr>/edits/<editType>
  private val svqmRegex = new Regex("syntactical|validity|quality|macro")
  def submissionSingleEditPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits" / svqmRegex) { (period, seqNr, editType) =>
      timedGet { uri =>
        completeVerified(supervisor, institutionId, period, seqNr, uri) {
          val fState = getStatusAndValidationState(supervisor, SubmissionId(institutionId, period, seqNr))
          onComplete(fState) {
            case Success((vs, status)) =>
              val edits = editInfos(editsOfType(editType, vs))
              complete(ToResponseMarshallable(SingleTypeEditResults(edits, status)))
            case Failure(error) => completeWithInternalError(uri, error)
          }
        }
      }
    }

  // /institutions/<institution>/filings/<period>/submissions/<submissionId>/edits/<edit>
  private val editNameRegex: Regex = new Regex("""[SVQ]\d\d\d""")

  def editFailureDetailsPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits" / editNameRegex) { (period, seqNr, editName) =>
      timedGet { uri =>
        completeVerified(supervisor, institutionId, period, seqNr, uri) {
          parameters('page.as[Int] ? 1) { (page: Int) =>
            val fValidator: Future[ActorRef] = fHmdaFileValidator(supervisor, SubmissionId(institutionId, period, seqNr))
            val fPaginatedErrors: Future[(PaginatedErrors, HmdaFileValidationState)] = for {
              va <- fValidator
              vs <- (va ? GetState).mapTo[HmdaFileValidationState]
              p <- (va ? GetNamedErrorResultsPaginated(editName, page)).mapTo[PaginatedErrors]
            } yield (p, vs)

            onComplete(fPaginatedErrors) {
              case Success((errorCollection, vs)) =>
                val rows: Seq[EditResultRow] = errorCollection.errors.map(validationErrorToResultRow(_, vs))
                val result = EditResult(editName, rows, uri.path.toString, page, errorCollection.totalErrors)
                complete(ToResponseMarshallable(result))
              case Failure(error) => completeWithInternalError(uri, error)
            }
          }
        }
      }
    }

  // institutions/<institutionId>/filings/<period>/submissions/<seqNr>/edits/quality|macro
  private val editTypeRegex = new Regex("quality|macro")
  def verifyEditsPath(supervisor: ActorRef, querySupervisor: ActorRef, institutionId: String)(implicit ec: ExecutionContext) =
    path("filings" / Segment / "submissions" / IntNumber / "edits" / editTypeRegex) { (period, seqNr, verificationType) =>
      timedPost { uri =>
        entity(as[EditsVerification]) { verification =>
          completeVerified(supervisor, institutionId, period, seqNr, uri) {
            val verified = verification.verified
            val fSubmissionManager = (supervisor ? FindProcessingActor(SubmissionManager.name, SubmissionId(institutionId, period, seqNr))).mapTo[ActorRef]
            val subId = SubmissionId(institutionId, period, seqNr)
            val fValidator = fHmdaFileValidator(supervisor, subId)
            val editType: ValidationErrorType = if (verificationType == "quality") Quality else Macro

            val fVerification = for {
              replyTo <- fSubmissionManager
              va <- fValidator
              v <- (va ? VerifyEdits(editType, verified, replyTo)).mapTo[SubmissionStatus]
            } yield v

            onComplete(fVerification) {
              case Success(status) =>
                complete(ToResponseMarshallable(EditsVerifiedResponse(verified, status)))
              case Failure(error) => completeWithInternalError(uri, error)
            }
          }
        }
      }
    }

  /////// Helper Methods ///////
  private def fHmdaFileValidator(supervisor: ActorRef, submissionId: SubmissionId): Future[ActorRef] =
    (supervisor ? FindProcessingActor(HmdaFileValidator.name, submissionId)).mapTo[ActorRef]

  private def getStatusAndValidationState(supervisor: ActorRef, submissionId: SubmissionId)(implicit ec: ExecutionContext): Future[(HmdaFileValidationState, SubmissionStatus)] = {
    val fValidator = fHmdaFileValidator(supervisor, submissionId)
    val fSubmissionsActor = (supervisor ? FindSubmissions(SubmissionPersistence.name, submissionId.institutionId, submissionId.period)).mapTo[ActorRef]

    for {
      sa <- fSubmissionsActor
      s <- (sa ? GetSubmissionById(submissionId)).mapTo[Submission]
      va <- fValidator
      vs <- (va ? GetState).mapTo[HmdaFileValidationState]
    } yield (vs, s.status)

  }

  private def getValidationState(supervisor: ActorRef, institutionId: String, period: String, seqNr: Int)(implicit ec: ExecutionContext): Future[HmdaFileValidationState] = {
    val fValidator = fHmdaFileValidator(supervisor, SubmissionId(institutionId, period, seqNr))
    for {
      s <- fValidator
      xs <- (s ? GetState).mapTo[HmdaFileValidationState]
    } yield xs
  }

  private def validationEventStream(submissionId: SubmissionId): Source[Event, NotUsed] = {
    val persistenceId = s"${HmdaFileValidator.name}-$submissionId"
    events(persistenceId)
  }

}

{'fraction_non_alphanumeric': 0.10187031746887296, 'fraction_numerical': 0.0003233978332345173, 'mean_word_length': 3.557602554654876, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5632199', 'n_tokens_mistral': 5443, 'n_tokens_neox': 5286, 'n_words': 1040}