<commit_before>package hmda.api.http.institutions.submissions

import akka.actor.ActorRef
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.model.StatusCodes
import akka.http.scaladsl.server.Route
import akka.pattern.ask
import hmda.api.http.InstitutionHttpApiSpec
import hmda.api.model.{ EditResult, _ }
import hmda.model.fi._
import hmda.model.fi.lar.LarGenerators
import hmda.model.fi.ts.{ TransmittalSheet, TsGenerators }
import hmda.persistence.HmdaSupervisor.FindProcessingActor
import hmda.persistence.messages.CommonMessages.GetState
import hmda.persistence.processing.HmdaFileValidator
import hmda.persistence.processing.HmdaFileValidator.HmdaFileValidationState
import hmda.validation.engine._
import spray.json.{ JsNumber, JsObject }

import scala.concurrent.{ Await, Future }
import scala.concurrent.duration._

class SubmissionEditPathsSpec extends InstitutionHttpApiSpec with LarGenerators with TsGenerators {

  override def beforeAll(): Unit = {
    super.beforeAll()
    loadValidationErrors()
  }

  val s020Description = "Agency code must = 1, 2, 3, 5, 7, 9. The agency that submits the data must be the same as the reported agency code."
  val s010Description = "The first record identifier in the file must = 1 (TS). The second and all subsequent record identifiers must = 2 (LAR)."
  val v280Description = "MSA/MD must = a valid Metropolitan Statistical Area or Metropolitan Division (if appropriate) code for period being processed or NA."
  val v285Description = "State must = a valid FIPS code or (NA where MSA/MD = NA)."
  val q007Description = "If action taken type = 2, then the total number of these loans should be ≤ 15% of the total number of loan applications."

  val s020info = EditInfo("S020", s020Description)
  val s010info = EditInfo("S010", s010Description)
  val v280info = EditInfo("V280", v280Description)
  val v285info = EditInfo("V285", v285Description)
  val q007info = EditInfo("Q007", q007Description)

  "return summary of validation errors" in {
    getWithCfpbHeaders(s"/institutions/0/filings/2017/submissions/1/edits") ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      val r = responseAs[SummaryEditResults]
      r.syntactical mustBe EditCollection(Seq(s020info, s010info))
      r.validity mustBe EditCollection(Seq(v285info, v280info))
      r.quality mustBe VerifiableEditCollection(verified = false, Seq())
      r.`macro` mustBe VerifiableEditCollection(verified = false, Seq(q007info))
    }
  }

  "return a list of validation errors for a single type" in {
    getWithCfpbHeaders(s"/institutions/0/filings/2017/submissions/1/edits/validity") ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[SingleTypeEditResults].edits mustBe Seq(v285info, v280info)
    }

    getWithCfpbHeaders(s"/institutions/0/filings/2017/submissions/1/edits/macro") ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[SingleTypeEditResults].edits mustBe Seq(q007info)
    }
  }

  val s010 = Seq(EditResultRow(RowId("loan1"), JsObject(("Record Identifier", JsNumber(111)))))
  "return row details for an edit" in {
    val path = "/institutions/0/filings/2017/submissions/1/edits/S010"
    val expected = EditResult("S010", s010, path, 1, 1)

    getWithCfpbHeaders(path) ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[EditResult] mustBe expected
    }
  }

  ///// CSV /////

  "Return summary edits in CSV format" in {
    getWithCfpbHeaders(s"/institutions/0/filings/2017/submissions/1/edits/csv") ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[String] must include("editType, editId, loanId")
      responseAs[String] must include("Syntactical, S020, Transmittal Sheet")
      responseAs[String] must include("Validity, V285, loan2")
    }
  }

  ///// Verification /////

  "Set up: get submission to ValidatedWithErrors state" in {
    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/2", file) ~> institutionsRoutes ~> check {
      Thread.sleep(5000) // wait for the submission to complete validation
      status mustBe StatusCodes.Accepted
    }
  }

  "Verify Macro edits endpoint: Responds with correct json and updates validation state" in {
    val verification = EditsVerification(true)
    val currentStatus = ValidatedWithErrors

    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/2/edits/macro", verification) ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK

      // test that it responds correctly
      responseAs[EditsVerifiedResponse] mustBe EditsVerifiedResponse(true, currentStatus)

      // test that it updates validation state
      val state: HmdaFileValidationState = Await.result(fValidationState, 5.seconds)
      state.macroVerified mustBe true
      state.qualityVerified mustBe false
    }
  }

  "Verify Quality edits endpoint: Responds with correct json, updates validation state, updates submission state" in {
    val verification = EditsVerification(true)
    val currentStatus = Validated

    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/2/edits/quality", verification) ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK

      // test that it responds correctly
      responseAs[EditsVerifiedResponse] mustBe EditsVerifiedResponse(true, currentStatus)

      // test that it updates validation state
      val state: HmdaFileValidationState = Await.result(fValidationState, 5.seconds)
      state.qualityVerified mustBe true
    }
  }

  ///// 405 (Method Not Allowed) Responses /////

  "Edit Type endpoint: return 405 when posting verification to syntactical endpoint" in {
    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/0/edits/syntactical") ~> Route.seal(institutionsRoutes) ~> check {
      status mustBe StatusCodes.MethodNotAllowed
    }
  }

  "Edit Type endpoint: return 405 when posting verification to validity endpoint" in {
    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/0/edits/validity") ~> Route.seal(institutionsRoutes) ~> check {
      status mustBe StatusCodes.MethodNotAllowed
    }
  }

  ///// 404 (Not Found) Responses /////
<commit_msg>Add test for un-verifying edits<commit_after>package hmda.api.http.institutions.submissions

import akka.actor.ActorRef
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.model.StatusCodes
import akka.http.scaladsl.server.Route
import akka.pattern.ask
import hmda.api.http.InstitutionHttpApiSpec
import hmda.api.model.{ EditResult, _ }
import hmda.model.fi._
import hmda.model.fi.lar.LarGenerators
import hmda.model.fi.ts.{ TransmittalSheet, TsGenerators }
import hmda.persistence.HmdaSupervisor.FindProcessingActor
import hmda.persistence.messages.CommonMessages.GetState
import hmda.persistence.processing.HmdaFileValidator
import hmda.persistence.processing.HmdaFileValidator.HmdaFileValidationState
import hmda.validation.engine._
import spray.json.{ JsNumber, JsObject }

import scala.concurrent.{ Await, Future }
import scala.concurrent.duration._

class SubmissionEditPathsSpec extends InstitutionHttpApiSpec with LarGenerators with TsGenerators {

  override def beforeAll(): Unit = {
    super.beforeAll()
    loadValidationErrors()
  }

  val s020Description = "Agency code must = 1, 2, 3, 5, 7, 9. The agency that submits the data must be the same as the reported agency code."
  val s010Description = "The first record identifier in the file must = 1 (TS). The second and all subsequent record identifiers must = 2 (LAR)."
  val v280Description = "MSA/MD must = a valid Metropolitan Statistical Area or Metropolitan Division (if appropriate) code for period being processed or NA."
  val v285Description = "State must = a valid FIPS code or (NA where MSA/MD = NA)."
  val q007Description = "If action taken type = 2, then the total number of these loans should be ≤ 15% of the total number of loan applications."

  val s020info = EditInfo("S020", s020Description)
  val s010info = EditInfo("S010", s010Description)
  val v280info = EditInfo("V280", v280Description)
  val v285info = EditInfo("V285", v285Description)
  val q007info = EditInfo("Q007", q007Description)

  "return summary of validation errors" in {
    getWithCfpbHeaders(s"/institutions/0/filings/2017/submissions/1/edits") ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      val r = responseAs[SummaryEditResults]
      r.syntactical mustBe EditCollection(Seq(s020info, s010info))
      r.validity mustBe EditCollection(Seq(v285info, v280info))
      r.quality mustBe VerifiableEditCollection(verified = false, Seq())
      r.`macro` mustBe VerifiableEditCollection(verified = false, Seq(q007info))
    }
  }

  "return a list of validation errors for a single type" in {
    getWithCfpbHeaders(s"/institutions/0/filings/2017/submissions/1/edits/validity") ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[SingleTypeEditResults].edits mustBe Seq(v285info, v280info)
    }

    getWithCfpbHeaders(s"/institutions/0/filings/2017/submissions/1/edits/macro") ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[SingleTypeEditResults].edits mustBe Seq(q007info)
    }
  }

  val s010 = Seq(EditResultRow(RowId("loan1"), JsObject(("Record Identifier", JsNumber(111)))))
  "return row details for an edit" in {
    val path = "/institutions/0/filings/2017/submissions/1/edits/S010"
    val expected = EditResult("S010", s010, path, 1, 1)

    getWithCfpbHeaders(path) ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[EditResult] mustBe expected
    }
  }

  ///// CSV /////

  "Return summary edits in CSV format" in {
    getWithCfpbHeaders(s"/institutions/0/filings/2017/submissions/1/edits/csv") ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[String] must include("editType, editId, loanId")
      responseAs[String] must include("Syntactical, S020, Transmittal Sheet")
      responseAs[String] must include("Validity, V285, loan2")
    }
  }

  ///// Verification /////

  "Set up: get submission to ValidatedWithErrors state" in {
    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/2", file) ~> institutionsRoutes ~> check {
      Thread.sleep(5000) // wait for the submission to complete validation
      status mustBe StatusCodes.Accepted
    }
  }

  "Verify Macro edits endpoint: Responds with correct json and updates validation state" in {
    val verification = EditsVerification(true)
    val currentStatus = ValidatedWithErrors

    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/2/edits/macro", verification) ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK

      // test that it responds correctly
      responseAs[EditsVerifiedResponse] mustBe EditsVerifiedResponse(true, currentStatus)

      // test that it updates validation state
      val state: HmdaFileValidationState = Await.result(fValidationState, 5.seconds)
      state.macroVerified mustBe true
      state.qualityVerified mustBe false
    }
  }

  "Verify Quality edits endpoint: Responds with correct json, updates validation state, updates submission state" in {
    val verification = EditsVerification(true)
    val currentStatus = Validated

    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/2/edits/quality", verification) ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK

      // test that it responds correctly
      responseAs[EditsVerifiedResponse] mustBe EditsVerifiedResponse(true, currentStatus)

      // test that it updates validation state
      val state: HmdaFileValidationState = Await.result(fValidationState, 5.seconds)
      state.qualityVerified mustBe true
    }
  }

  "Un-verify quality edits: updates submission state correctly back to `ValidatedWithErrors`" in {
    val verification = EditsVerification(false)
    val currentStatus = ValidatedWithErrors

    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/2/edits/quality", verification) ~> institutionsRoutes ~> check {
      status mustBe StatusCodes.OK
      responseAs[EditsVerifiedResponse] mustBe EditsVerifiedResponse(false, currentStatus)
    }
  }

  ///// 405 (Method Not Allowed) Responses /////

  "Edit Type endpoint: return 405 when posting verification to syntactical endpoint" in {
    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/0/edits/syntactical") ~> Route.seal(institutionsRoutes) ~> check {
      status mustBe StatusCodes.MethodNotAllowed
    }
  }

  "Edit Type endpoint: return 405 when posting verification to validity endpoint" in {
    postWithCfpbHeaders("/institutions/0/filings/2017/submissions/0/edits/validity") ~> Route.seal(institutionsRoutes) ~> check {
      status mustBe StatusCodes.MethodNotAllowed
    }
  }

  ///// 404 (Not Found) Responses /////

{'fraction_non_alphanumeric': 0.09430195054523115, 'fraction_numerical': 0.030870833973275995, 'mean_word_length': 4.135252365930599, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1698966', 'n_tokens_mistral': 4266, 'n_tokens_neox': 3906, 'n_words': 998}