<commit_before>package hmda.publication.reports.disclosure

import akka.NotUsed
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Source
import hmda.model.fi.lar.{ LarGenerators, LoanApplicationRegister }
import hmda.model.publication.reports.ActionTakenTypeEnum._
import hmda.model.publication.reports.ApplicantIncomeEnum.LessThan50PercentOfMSAMedian
import hmda.model.publication.reports.{ EthnicityBorrowerCharacteristic, MSAReport, MinorityStatusBorrowerCharacteristic, RaceBorrowerCharacteristic }
import hmda.query.model.filing.LoanApplicationRegisterQuery
import hmda.query.repository.filing.LarConverter._
import org.scalacheck.Gen
import org.scalatest.{ MustMatchers, WordSpec }

import scala.concurrent.{ Await, Future }
import scala.concurrent.duration._

class D51Spec extends WordSpec with MustMatchers with LarGenerators {

  implicit val system = ActorSystem()
  implicit val ec = system.dispatcher
  implicit val materializer = ActorMaterializer()

  val respId = "98765"
  val fips = 18700 //Corvallis, OR
  def loanType = Gen.oneOf(2, 3, 4).sample.get
  def propType = Gen.oneOf(1, 2).sample.get

  val lars = lar100ListGen.sample.get.map { lar: LoanApplicationRegister =>
    val geo = lar.geography.copy(msa = fips.toString)
    val loan = lar.loan.copy(loanType = loanType, propertyType = propType, purpose = 1)
    lar.copy(respondentId = respId, geography = geo, loan = loan)
  }

  val source: Source[LoanApplicationRegisterQuery, NotUsed] = Source
    .fromIterator(() => lars.toIterator)
    .map(lar => toLoanApplicationRegisterQuery(lar))

  val expectedDispositions = List(ApplicationReceived, LoansOriginated, ApprovedButNotAccepted, ApplicationsDenied, ApplicationsWithdrawn, ClosedForIncompleteness)

  "Generate a Disclosure 5-1 report" in {
    val result = Await.result(D51.generate(source, fips, respId, Future("Corvallis Test Bank")), 5.seconds)

    result.msa mustBe MSAReport("18700", "Corvallis, OR", "OR", "Oregon")
    result.table mustBe "5-1"
    result.respondentId mustBe "98765"
    result.institutionName mustBe "Corvallis Test Bank"
    result.applicantIncomes.size mustBe 5

    val lowestIncome = result.applicantIncomes.head
    lowestIncome.applicantIncome mustBe LessThan50PercentOfMSAMedian

    val races = lowestIncome.borrowerCharacteristics.head.asInstanceOf[RaceBorrowerCharacteristic].races
    races.size mustBe 8

    val ethnicities = lowestIncome.borrowerCharacteristics(1).asInstanceOf[EthnicityBorrowerCharacteristic].ethnicities
    ethnicities.size mustBe 4

    val minorityStatuses = lowestIncome.borrowerCharacteristics(2).asInstanceOf[MinorityStatusBorrowerCharacteristic].minoritystatus
    minorityStatuses.size mustBe 2

    races.head.dispositions.map(_.disposition) mustBe expectedDispositions
  }

}
<commit_msg>Use AsyncWordSpec to test report generation<commit_after>package hmda.publication.reports.disclosure

import akka.NotUsed
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Source
import hmda.model.fi.lar.{ LarGenerators, LoanApplicationRegister }
import hmda.model.publication.reports.ActionTakenTypeEnum._
import hmda.model.publication.reports.ApplicantIncomeEnum.LessThan50PercentOfMSAMedian
import hmda.model.publication.reports.{ EthnicityBorrowerCharacteristic, MSAReport, MinorityStatusBorrowerCharacteristic, RaceBorrowerCharacteristic }
import hmda.query.model.filing.LoanApplicationRegisterQuery
import hmda.query.repository.filing.LarConverter._
import org.scalacheck.Gen
import org.scalatest.{ AsyncWordSpec, MustMatchers }

import scala.concurrent.Future

class D51Spec extends AsyncWordSpec with MustMatchers with LarGenerators {

  implicit val system = ActorSystem()
  implicit val ec = system.dispatcher
  implicit val materializer = ActorMaterializer()

  val respId = "98765"
  val fips = 18700 //Corvallis, OR
  def loanType = Gen.oneOf(2, 3, 4).sample.get
  def propType = Gen.oneOf(1, 2).sample.get

  val lars = lar100ListGen.sample.get.map { lar: LoanApplicationRegister =>
    val geo = lar.geography.copy(msa = fips.toString)
    val loan = lar.loan.copy(loanType = loanType, propertyType = propType, purpose = 1)
    lar.copy(respondentId = respId, geography = geo, loan = loan)
  }

  val source: Source[LoanApplicationRegisterQuery, NotUsed] = Source
    .fromIterator(() => lars.toIterator)
    .map(lar => toLoanApplicationRegisterQuery(lar))

  val expectedDispositions = List(ApplicationReceived, LoansOriginated, ApprovedButNotAccepted, ApplicationsDenied, ApplicationsWithdrawn, ClosedForIncompleteness)

  "Generate a Disclosure 5-1 report" in {
    D51.generate(source, fips, respId, Future("Corvallis Test Bank")).map { result =>

      result.msa mustBe MSAReport("18700", "Corvallis, OR", "OR", "Oregon")
      result.table mustBe "5-1"
      result.respondentId mustBe "98765"
      result.institutionName mustBe "Corvallis Test Bank"
      result.applicantIncomes.size mustBe 5

      val lowestIncome = result.applicantIncomes.head
      lowestIncome.applicantIncome mustBe LessThan50PercentOfMSAMedian

      val races = lowestIncome.borrowerCharacteristics.head.asInstanceOf[RaceBorrowerCharacteristic].races
      races.size mustBe 8

      val ethnicities = lowestIncome.borrowerCharacteristics(1).asInstanceOf[EthnicityBorrowerCharacteristic].ethnicities
      ethnicities.size mustBe 4

      val minorityStatuses = lowestIncome.borrowerCharacteristics(2).asInstanceOf[MinorityStatusBorrowerCharacteristic].minoritystatus
      minorityStatuses.size mustBe 2

      races.head.dispositions.map(_.disposition) mustBe expectedDispositions
    }
  }

}

{'fraction_non_alphanumeric': 0.08397887323943662, 'fraction_numerical': 0.016725352112676055, 'mean_word_length': 5.277348066298343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1795645', 'n_tokens_mistral': 1884, 'n_tokens_neox': 1740, 'n_words': 359}