<commit_before>package uk.gov.tna.dri.validator

import org.specs2.mutable.Specification
import uk.gov.tna.dri.schema.{ColumnDefinition, RegexRule, Schema}
import java.io.StringReader
import scalaz._

class MetaDataValidatorSpec extends Specification {

  object TestMetaDataValidator extends MetaDataValidator

  import TestMetaDataValidator._

  "Validation" should {

    "succeed for correct total columns for multiple lines" in {
      val metaData =
        """col1, col2
           col1, col2"""

      validate(new StringReader(metaData), Schema(2)) must beLike { case Success(_) => ok }
    }

    "fail for incorrect number of total columns for multiple lines" in {
      val metaData =
        """col1, col2, col3
           col1, col2
           col1, col2, col3"""

      validate(new StringReader(metaData), Schema(3)) must beLike { case Failure(messages) => messages.head mustEqual "Expected @TotalColumns of 3 and found 2 on line 2" }
    }

    "fail if first column doesnt pass regex rule" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second"))
      validateRow(List("99", "xxx"), columnDefs) must beLike { case Failure(msgs) => msgs.head mustEqual "Value: 99 does not match regex: [3-8]*" }
    }

    "pass if first and second columns pass regex rule" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second",List(RegexRule("[a-c]*".r))))
      validateRow(List("88", "abc"), columnDefs) must beLike { case Success(_) => ok }
    }

    "fail if first column doesnt pass regex rule and second does" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second",List(RegexRule("[a-c]*".r))))
      validateRow(List("99", "abc"), columnDefs) must beLike { case Failure(msgs) => msgs.head mustEqual "Value: 99 does not match regex: [3-8]*" }
    }

    "fail if first column does pass regex rule and second does not" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second",List(RegexRule("[a-c]*".r))))
      validateRow(List("88", "xxx"), columnDefs) must beLike { case Failure(msgs) => msgs.head mustEqual "Value: xxx does not match regex: [a-c]*" }
    }

    "fail if first and second columns do not pass" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second",List(RegexRule("[a-c]*".r))))

      validateRow(List("99", "xxx"), columnDefs) must beLike {
        case Failure(msgs) => msgs.list must contain ("Value: 99 does not match regex: [3-8]*", "Value: xxx does not match regex: [a-c]*").only
      }
    }

    "succeed if validates multiple rows" in {

      val rows = List(List("a", "1"), List("b", "2"))
      val schema = Schema(2, List(ColumnDefinition("column1", List(RegexRule("[a-c]".r))), ColumnDefinition("column2", List(RegexRule("[0-9]".r)))))
      validateRows(rows, schema) must beLike { case Success(_) => ok }
    }

  }
}
<commit_msg>[BP] Test for validator using the main interface<commit_after>package uk.gov.tna.dri.validator

import org.specs2.mutable.Specification
import uk.gov.tna.dri.schema.{ColumnDefinition, RegexRule, Schema}
import java.io.StringReader
import scalaz._

class MetaDataValidatorSpec extends Specification {

  object TestMetaDataValidator extends MetaDataValidator

  import TestMetaDataValidator._

  "Validation" should {

    "succeed for correct total columns for multiple lines" in {
      val metaData =
        """col1, col2
           col1, col2"""

      validate(new StringReader(metaData), Schema(2)) must beLike { case Success(_) => ok }
    }

    "fail for incorrect number of total columns for multiple lines" in {
      val metaData =
        """col1, col2, col3
           col1, col2
           col1, col2, col3"""

      validate(new StringReader(metaData), Schema(3)) must beLike { case Failure(messages) => messages.head mustEqual "Expected @TotalColumns of 3 and found 2 on line 2" }
    }

    "fail if first column doesnt pass regex rule" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second"))
      validateRow(List("99", "xxx"), columnDefs) must beLike { case Failure(msgs) => msgs.head mustEqual "Value: 99 does not match regex: [3-8]*" }
    }

    "pass if first and second columns pass regex rule" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second",List(RegexRule("[a-c]*".r))))
      validateRow(List("88", "abc"), columnDefs) must beLike { case Success(_) => ok }
    }

    "fail if first column doesnt pass regex rule and second does" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second",List(RegexRule("[a-c]*".r))))
      validateRow(List("99", "abc"), columnDefs) must beLike { case Failure(msgs) => msgs.head mustEqual "Value: 99 does not match regex: [3-8]*" }
    }

    "fail if first column does pass regex rule and second does not" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second",List(RegexRule("[a-c]*".r))))
      validateRow(List("88", "xxx"), columnDefs) must beLike { case Failure(msgs) => msgs.head mustEqual "Value: xxx does not match regex: [a-c]*" }
    }

    "fail if first and second columns do not pass" in {

      val columnDefs = List(ColumnDefinition("first", List(RegexRule("[3-8]*".r))), ColumnDefinition("second",List(RegexRule("[a-c]*".r))))

      validateRow(List("99", "xxx"), columnDefs) must beLike {
        case Failure(msgs) => msgs.list must contain ("Value: 99 does not match regex: [3-8]*", "Value: xxx does not match regex: [a-c]*").only
      }
    }

    "succeed if validates multiple rows" in {

      val rows = List(List("a", "1"), List("b", "2"))
      val schema = Schema(2, List(ColumnDefinition("column1", List(RegexRule("[a-c]".r))), ColumnDefinition("column2", List(RegexRule("[0-9]".r)))))
      validateRows(rows, schema) must beLike { case Success(_) => ok }
    }

    "succeed for valid schema and meta-data" in {

      val schema = Schema(2, List(ColumnDefinition("col1"), ColumnDefinition("col2WithRule", List(RegexRule("[0-9]*".r)))))
      val metaData =
        """someData,345
           someMore,12"""

      validate(new StringReader(metaData), schema) must beLike { case Success(_) => ok }
    }

  }
}

{'fraction_non_alphanumeric': 0.15269277845777235, 'fraction_numerical': 0.018971848225214197, 'mean_word_length': 3.8101545253863134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2501694', 'n_tokens_mistral': 2064, 'n_tokens_neox': 1994, 'n_words': 576}