<commit_before>package org.globalnames.parser

import org.json4s._
import org.json4s.jackson.JsonMethods._
import org.specs2.mutable.Specification

import scala.io.Source

class ScientificNameParserSpec extends Specification {
  case class ExpectedName(verbatim: String, json: String, simple: String)

  "ScientificNameParser specification".p

  val scientificNameParser = new ScientificNameParser {
    val version = "test_version"
  }

  def expectedNames(filePath: String): Vector[ExpectedName] = {
    Source.fromURL(getClass.getResource(filePath), "UTF-8")
          .getLines
          .takeWhile { _.trim != "__END__" }
          .withFilter { line => !(line.isEmpty || ("#\r\n\f\t" contains line.charAt(0))) }
          .sliding(3, 3)
          .map { ls => ExpectedName(ls(0), ls(1), ls(2)) }
          .toVector
  }

  expectedNames("/test_data.txt").foreach { expectedName =>
      val json = parse(expectedName.json)
      val jsonParsed = scientificNameParser.fromString(expectedName.verbatim).json()
                         .removeField { case (_, v) => v == JNothing }

      val jsonDiff = {
        val Diff(changed, added, deleted) = jsonParsed.diff(json)
        s"""line:
           |${expectedName.verbatim}
           |parsed:
           |${pretty(jsonParsed)}
           |test_data:
           |${pretty(json)}
           |changed:
           |${pretty(changed)}
           |added:
           |${pretty(added)}
           |deleted:
           |${pretty(deleted)}""".stripMargin
      }

      s"parse correctly: '${expectedName.verbatim}'" in {
        s"original json must match expected one:\n $jsonDiff" ==> {
          json === jsonParsed
        }
      }

      val Array(uuid, verbatim, canonical, canonicalExtended,
                authorship, year, quality) = expectedName.simple.split('|')

      s"parse correctly delimited string: '${expectedName.verbatim}'" in {
        val pr = scientificNameParser.fromString(expectedName.verbatim)

        uuid              === pr.input.id.toString
        verbatim          === pr.input.verbatim
        canonical         === pr.canonized().getOrElse("")
        canonicalExtended === pr.canonized(showRanks = true).getOrElse("")
        authorship        === pr.authorshipDelimited
        year              === pr.yearDelimited
        quality.toInt     === pr.scientificName.quality
      }

      s"contain no orphans in warnings" in {
        def hasRefNode(sourceNode: AstNode, targetNode: AstNode): Boolean = targetNode match {
          case tn if tn == sourceNode => true
          case sn: ScientificName => sn.namesGroup.exists { ng => hasRefNode(sourceNode, ng) }
          case ng: NamesGroup => ng.name.exists { n => hasRefNode(sourceNode, n) }
          case n: Name => hasRefNode(sourceNode, n.uninomial) ||
                          n.subgenus.exists { sg => hasRefNode(sourceNode, sg) } ||
                          n.species.exists { sp => hasRefNode(sourceNode, sp) } ||
                          n.infraspecies.exists { is => hasRefNode(sourceNode, is) } ||
                          n.comparison.exists { cmp => hasRefNode(sourceNode, cmp) } ||
                          n.approximation.exists { aprx => hasRefNode(sourceNode, aprx) }
          case sg: SubGenus => hasRefNode(sourceNode, sg.subgenus)
          case u: Uninomial => u.authorship.exists { auth => hasRefNode(sourceNode, auth)} ||
                               u.rank.exists { rk => hasRefNode(sourceNode, rk) } ||
                               u.parent.exists { par => hasRefNode(sourceNode, par) }
          case ig: InfraspeciesGroup => ig.group.exists { is => hasRefNode(sourceNode, is) }
          case is: Infraspecies => hasRefNode(sourceNode, is.word) ||
                                   is.rank.exists { rk => hasRefNode(sourceNode, rk) } ||
                                   is.authorship.exists { a => hasRefNode(sourceNode, a) }
          case sp: Species => hasRefNode(sourceNode, sp.word) ||
                              sp.authorship.exists { a => hasRefNode(sourceNode, a) }
          case auth: Authorship => hasRefNode(sourceNode, auth.authors) ||
                                   auth.combination.exists { cmb => hasRefNode(sourceNode, cmb) }
          case ag: AuthorsGroup => hasRefNode(sourceNode, ag.authors) ||
                                   ag.authorsEx.exists { auEx => hasRefNode(sourceNode, auEx) } ||
                                   ag.year.exists { y => hasRefNode(sourceNode, y) }
          case at: AuthorsTeam => at.authors.exists { a => hasRefNode(sourceNode, a) }
          case au: Author => au.words.exists { aw => hasRefNode(sourceNode, aw) } ||
                             au.filius.exists { f => hasRefNode(sourceNode, f) }
          case _ => false
        }

        val pr = scientificNameParser.fromString(expectedName.verbatim)

        pr.warnings.filterNot {
          warning => hasRefNode(warning.node, pr.scientificName)
        } should beEmpty
      }
  }
}
<commit_msg>= Add non-duplicate in warnings test<commit_after>package org.globalnames.parser

import org.json4s._
import org.json4s.jackson.JsonMethods._
import org.specs2.mutable.Specification

import scala.io.Source

class ScientificNameParserSpec extends Specification {
  case class ExpectedName(verbatim: String, json: String, simple: String)

  "ScientificNameParser specification".p

  val scientificNameParser = new ScientificNameParser {
    val version = "test_version"
  }

  def expectedNames(filePath: String): Vector[ExpectedName] = {
    Source.fromURL(getClass.getResource(filePath), "UTF-8")
          .getLines
          .takeWhile { _.trim != "__END__" }
          .withFilter { line => !(line.isEmpty || ("#\r\n\f\t" contains line.charAt(0))) }
          .sliding(3, 3)
          .map { ls => ExpectedName(ls(0), ls(1), ls(2)) }
          .toVector
  }

  expectedNames("/test_data.txt").foreach { expectedName =>
      val json = parse(expectedName.json)
      val jsonParsed = scientificNameParser.fromString(expectedName.verbatim).json()
                         .removeField { case (_, v) => v == JNothing }

      val jsonDiff = {
        val Diff(changed, added, deleted) = jsonParsed.diff(json)
        s"""line:
           |${expectedName.verbatim}
           |parsed:
           |${pretty(jsonParsed)}
           |test_data:
           |${pretty(json)}
           |changed:
           |${pretty(changed)}
           |added:
           |${pretty(added)}
           |deleted:
           |${pretty(deleted)}""".stripMargin
      }

      s"parse correctly: '${expectedName.verbatim}'" in {
        s"original json must match expected one:\n $jsonDiff" ==> {
          json === jsonParsed
        }
      }

      val Array(uuid, verbatim, canonical, canonicalExtended,
                authorship, year, quality) = expectedName.simple.split('|')

      s"parse correctly delimited string: '${expectedName.verbatim}'" in {
        val pr = scientificNameParser.fromString(expectedName.verbatim)

        uuid              === pr.input.id.toString
        verbatim          === pr.input.verbatim
        canonical         === pr.canonized().getOrElse("")
        canonicalExtended === pr.canonized(showRanks = true).getOrElse("")
        authorship        === pr.authorshipDelimited
        year              === pr.yearDelimited
        quality.toInt     === pr.scientificName.quality
      }

      s"contain no duplicates in warnings" in {
        val pr = scientificNameParser.fromString(expectedName.verbatim)
        Set(pr.warnings: _*).size === pr.warnings.size
      }

      s"contain no orphans in warnings" in {
        def hasRefNode(sourceNode: AstNode, targetNode: AstNode): Boolean = targetNode match {
          case tn if tn == sourceNode => true
          case sn: ScientificName => sn.namesGroup.exists { ng => hasRefNode(sourceNode, ng) }
          case ng: NamesGroup => ng.name.exists { n => hasRefNode(sourceNode, n) }
          case n: Name => hasRefNode(sourceNode, n.uninomial) ||
                          n.subgenus.exists { sg => hasRefNode(sourceNode, sg) } ||
                          n.species.exists { sp => hasRefNode(sourceNode, sp) } ||
                          n.infraspecies.exists { is => hasRefNode(sourceNode, is) } ||
                          n.comparison.exists { cmp => hasRefNode(sourceNode, cmp) } ||
                          n.approximation.exists { aprx => hasRefNode(sourceNode, aprx) }
          case sg: SubGenus => hasRefNode(sourceNode, sg.subgenus)
          case u: Uninomial => u.authorship.exists { auth => hasRefNode(sourceNode, auth)} ||
                               u.rank.exists { rk => hasRefNode(sourceNode, rk) } ||
                               u.parent.exists { par => hasRefNode(sourceNode, par) }
          case ig: InfraspeciesGroup => ig.group.exists { is => hasRefNode(sourceNode, is) }
          case is: Infraspecies => hasRefNode(sourceNode, is.word) ||
                                   is.rank.exists { rk => hasRefNode(sourceNode, rk) } ||
                                   is.authorship.exists { a => hasRefNode(sourceNode, a) }
          case sp: Species => hasRefNode(sourceNode, sp.word) ||
                              sp.authorship.exists { a => hasRefNode(sourceNode, a) }
          case auth: Authorship => hasRefNode(sourceNode, auth.authors) ||
                                   auth.combination.exists { cmb => hasRefNode(sourceNode, cmb) }
          case ag: AuthorsGroup => hasRefNode(sourceNode, ag.authors) ||
                                   ag.authorsEx.exists { auEx => hasRefNode(sourceNode, auEx) } ||
                                   ag.year.exists { y => hasRefNode(sourceNode, y) }
          case at: AuthorsTeam => at.authors.exists { a => hasRefNode(sourceNode, a) }
          case au: Author => au.words.exists { aw => hasRefNode(sourceNode, aw) } ||
                             au.filius.exists { f => hasRefNode(sourceNode, f) }
          case _ => false
        }

        val pr = scientificNameParser.fromString(expectedName.verbatim)

        pr.warnings.filterNot {
          warning => hasRefNode(warning.node, pr.scientificName)
        } should beEmpty
      }
  }
}

{'fraction_non_alphanumeric': 0.1214614555784112, 'fraction_numerical': 0.0019590557351356645, 'mean_word_length': 2.026081802015412, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2017840', 'n_tokens_mistral': 3005, 'n_tokens_neox': 2888, 'n_words': 601}