<commit_before>package hmda.parser.fi.ts

import hmda.model.fi.ts.{ Contact, Parent, Respondent, TransmittalSheet }
import scala.collection.immutable.ListMap
import scalaz._
import scalaz.Scalaz._
import scala.util.{ Failure, Success, Try }

object TsCsvParser {
  def apply(s: String): Either[List[String], TransmittalSheet] = {
    val values = s.split('|').map(_.trim).toList
    val parsedInts = checkTs(values)
    parsedInts match {
      case scalaz.Success(parsedValues) => {

        val id = parsedValues(0).asInstanceOf[Int]
        val respId = values(1)
        val code = parsedValues(1).asInstanceOf[Int]
        val timestamp = parsedValues(4).asInstanceOf[Long]
        val activityYear = parsedValues(2).asInstanceOf[Int]
        val taxId = values(5)
        val totalLines = parsedValues(3).asInstanceOf[Int]
        val respName = values(7)
        val respAddress = values(8)
        val respCity = values(9)
        val respState = values(10)
        val respZip = values(11)
        val parentName = values(12)
        val parentAddress = values(13)
        val parentCity = values(14)
        val parentState = values(15)
        val parentZip = values(16)
        val contactPerson = values(17)
        val contactPhone = values(18)
        val contactFax = values(19)
        val contactEmail = values(20)

        val respondent = Respondent(respId, respName, respAddress, respCity, respState, respZip)
        val parent = Parent(parentName, parentAddress, parentCity, parentState, parentZip)
        val contact = Contact(contactPerson, contactPhone, contactFax, contactEmail)
        Right(
          TransmittalSheet(
            id,
            code,
            timestamp,
            activityYear,
            taxId,
            totalLines,
            respondent,
            parent,
            contact
          )
        )
      }
      case scalaz.Failure(errors) => {
        Left(
          errors.list.toList
        )
      }
    }
  }

  def checkTs(fields: List[String]): ValidationNel[String, List[AnyVal]] = {

    if (fields.length != 21) {
      ("Incorrect number of fields. found: " + fields.length + ", expected: 21").failure.toValidationNel
    } else {
      val numericFields = ListMap(
        "Record Identifier" -> fields(0),
        "Agency Code" -> fields(2),
        "Activity Year" -> fields(4),
        "Total Lines Entries" -> fields(6)
      )

      val validationListInt = numericFields.map { case (key, value) => toIntOrFail(value, key) }
      val validationListLong = toLongOrFail(fields(3), "Timestamp")
      val validationListIntReduce = validationListInt.reduce(_ +++ _)
      validationListIntReduce +++ validationListLong
    }
  }

  def toIntOrFail(value: String, fieldName: String): ValidationNel[String, List[Int]] = {
    Try(value.toInt) match {
      case Failure(result) => s"$fieldName is not an Integer".failure.toValidationNel
      case Success(result) => List(result).success
    }
  }

  def toLongOrFail(value: String, fieldName: String): ValidationNel[String, List[Long]] = {
    Try(value.toLong) match {
      case Failure(result) => s"$fieldName is not a Long".failure.toValidationNel
      case Success(result) => List(result).success
    }
  }

}<commit_msg>refactor: use validation directly, without lists<commit_after>package hmda.parser.fi.ts

import hmda.model.fi.ts.{ Contact, Parent, Respondent, TransmittalSheet }

import scalaz._
import scalaz.Scalaz._
import scala.util.{ Failure, Success, Try }

object TsCsvParser {
  def apply(s: String): Either[List[String], TransmittalSheet] = {
    val values = s.split('|').map(_.trim).toList
    if (values.length != 21) {
      return Left(List("Incorrect number of fields. found: " + values.length + ", expected: 21"))
    }

    val id = toIntOrFail(values(0), "Record Identifier")
    val respId = values(1)
    val code = toIntOrFail(values(2), "Agency Code")
    val timestamp = toLongOrFail(values(3), "Timestamp")
    val activityYear = toIntOrFail(values(4), "Activity Year")
    val taxId = values(5)
    val totalLines = toIntOrFail(values(6), "Total Lines")
    val respName = values(7)
    val respAddress = values(8)
    val respCity = values(9)
    val respState = values(10)
    val respZip = values(11)
    val parentName = values(12)
    val parentAddress = values(13)
    val parentCity = values(14)
    val parentState = values(15)
    val parentZip = values(16)
    val contactPerson = values(17)
    val contactPhone = values(18)
    val contactFax = values(19)
    val contactEmail = values(20)

    val respondent = Respondent(respId, respName, respAddress, respCity, respState, respZip)
    val parent = Parent(parentName, parentAddress, parentCity, parentState, parentZip)
    val contact = Contact(contactPerson, contactPhone, contactFax, contactEmail)

    val maybeTS = (
      id
      |@| code
      |@| timestamp
      |@| activityYear
      |@| totalLines
    ) {
        TransmittalSheet(
          _,
          _,
          _,
          _,
          taxId,
          _,
          respondent,
          parent,
          contact
        )
      }

    maybeTS.disjunction.toEither.bimap(_.toList, identity(_))
  }

  def toIntOrFail(value: String, fieldName: String): ValidationNel[String, Int] = {
    Try(value.toInt) match {
      case Failure(result) => s"$fieldName is not an Integer".failure.toValidationNel
      case Success(result) => result.success
    }
  }

  def toLongOrFail(value: String, fieldName: String): ValidationNel[String, Long] = {
    Try(value.toLong) match {
      case Failure(result) => s"$fieldName is not a Long".failure.toValidationNel
      case Success(result) => result.success
    }
  }

}
{'fraction_non_alphanumeric': 0.11142256482130343, 'fraction_numerical': 0.013489838822704976, 'mean_word_length': 2.741153342070773, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2139428', 'n_tokens_mistral': 1791, 'n_tokens_neox': 1711, 'n_words': 412}