<commit_before>package uk.gov.tna.dri.validator

import uk.gov.tna.dri.schema.{Rule, ColumnDefinition, RegexRule, Schema}
import au.com.bytecode.opencsv.CSVReader
import java.io.Reader
import scala.collection.JavaConversions._
import scalaz._
import Scalaz._
import util.matching.Regex

trait MetaDataValidator {

  def validate(csv: Reader, schema: Schema) = {
    val rows = new CSVReader(csv).readAll() toList

    val totalCols = totalColumns(rows, schema)
    val reg = true.successNel[String]

    (totalCols |@| reg) tupled
  }

  def totalColumns(rows: List[Array[String]], schema: Schema) = {
    rows.zipWithIndex.find(r => r._1.length != schema.totalColumns) match {
      case Some((row, rowIndex)) => s"Expected @TotalColumns of ${schema.totalColumns} and found ${row.length} on line ${rowIndex + 1}".failNel[Boolean]
      case _ => true.successNel[String]
    }
  }

  def regexForValue(value: String, rule: RegexRule) : Validation[String, Boolean] = {
    val regex = rule.regex.pattern.pattern
    if (value matches regex) true.success[String] else s"Value: ${value} does not match regex: ${regex}".fail[Boolean]
  }

  def regexForRow(row: List[String], rule: RegexRule) = {
    row match {
      case first :: t => regexForValue(first, rule)
      case _ => "ColumnDefinition vale missing".fail[Boolean]
    }
  }

  def regex(rows: List[List[String]], rule: RegexRule) = {
    val validations = rows map (row => regexForRow(row, rule) liftFailNel)
    validations.sequence[({type x[a] = ValidationNEL[String, a]})#x, Boolean]
  }

  def regexForRowWithColumnDef(row: List[String], columnDefinitions: List[ColumnDefinition]) = {
    val valueWithColumnDefinition = row.zip(columnDefinitions)
    val validations =
      for { (value, columnDef) <- valueWithColumnDefinition
             rule <- columnDef.rules
        } yield rule.execute(value)

    validations.sequence[({type x[a] = ValidationNEL[String, a]})#x, Boolean]
   }
}
<commit_msg>[JC, DA] [Feature] Cleaned up imports<commit_after>package uk.gov.tna.dri.validator

import uk.gov.tna.dri.schema.{ColumnDefinition, RegexRule, Schema}
import au.com.bytecode.opencsv.CSVReader
import java.io.Reader
import scala.collection.JavaConversions._
import scalaz._
import Scalaz._

trait MetaDataValidator {

  def validate(csv: Reader, schema: Schema) = {
    val rows = new CSVReader(csv).readAll() toList

    val totalCols = totalColumns(rows, schema)
    val reg = true.successNel[String]

    (totalCols |@| reg) tupled
  }

  def totalColumns(rows: List[Array[String]], schema: Schema) = {
    rows.zipWithIndex.find(r => r._1.length != schema.totalColumns) match {
      case Some((row, rowIndex)) => s"Expected @TotalColumns of ${schema.totalColumns} and found ${row.length} on line ${rowIndex + 1}".failNel[Boolean]
      case _ => true.successNel[String]
    }
  }

  def regexForValue(value: String, rule: RegexRule) : Validation[String, Boolean] = {
    val regex = rule.regex.pattern.pattern
    if (value matches regex) true.success[String] else s"Value: ${value} does not match regex: ${regex}".fail[Boolean]
  }

  def regexForRow(row: List[String], rule: RegexRule) = {
    row match {
      case first :: t => regexForValue(first, rule)
      case _ => "ColumnDefinition vale missing".fail[Boolean]
    }
  }

  def regex(rows: List[List[String]], rule: RegexRule) = {
    val validations = rows map (row => regexForRow(row, rule) liftFailNel)
    validations.sequence[({type x[a] = ValidationNEL[String, a]})#x, Boolean]
  }

  def regexForRowWithColumnDef(row: List[String], columnDefinitions: List[ColumnDefinition]) = {
    val valueWithColumnDefinition = row.zip(columnDefinitions)
    val validations =
      for { (value, columnDef) <- valueWithColumnDefinition
             rule <- columnDef.rules
        } yield rule.execute(value)

    validations.sequence[({type x[a] = ValidationNEL[String, a]})#x, Boolean]
   }
}

{'fraction_non_alphanumeric': 0.13013523858127074, 'fraction_numerical': 0.0010206685378923194, 'mean_word_length': 4.03209242618742, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '20783', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1241, 'n_words': 294}