<commit_before>package dpla.ingestion3.mappers

import dpla.ingestion3.enrichments.normalizations.StringNormalizationUtils._
import dpla.ingestion3.mappers.utils._
import dpla.ingestion3.messages.{IngestMessage, IngestMessageTemplates, MessageCollector}
import dpla.ingestion3.model.DplaMapData.{ExactlyOne, ZeroToMany, ZeroToOne}
import dpla.ingestion3.model._
import dpla.ingestion3.utils.Utils._
import org.json4s.DefaultFormats
import org.json4s.JsonAST._

import scala.util.{Failure, Success, Try}
import scala.xml.NodeSeq

trait Mapper[T, +E] extends IngestMessageTemplates {
  implicit val formats: DefaultFormats.type = DefaultFormats

  def map(document: Document[T], mapping: Mapping[T]): OreAggregation



  /**
    * Normalizes edmRights URIs and logs specific messages for each transformation (logged as warnings)
    *   1. HTTPS > HTTP
    *   2. Drop `www`
    *   3. Change `/page/` to `/vocab/`
    *   4. Drop query parameters (ex. ?lang=en)
    *   5. Add missing `/` to end of URI
    *   6. Remove trailing punctuation (ex. http://rightsstaments.org/vocab/Inc/;)
    *   7. Leading and trailing whitespace also remove but no messages are logged for this transformation
    *
    * If original value is not a valid java.net.URI the original value is returned
    *
    * @param values Seq[URI]              edmRights URIs to be normalized
    * @param providerId String            Provider supplied record identifier
    * @param collector MessageCollector   Ingest message collector
    * @return Seq[URI]                    URIs
    */
  def normalizeEdmRights(values: ZeroToMany[URI], providerId: String)
                        (implicit collector: MessageCollector[IngestMessage]): ZeroToMany[URI] = {
    values.map (value => {
      val normalized = Try { new java.net.URI(value.toString.trim) } match {
        case Success(uri) => {
          // does scheme (http/https) require normalization
          if (uri.toString.startsWith("https")) {
            collector.add(normalizedEdmRightsHttpsMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
          }
          // `www` to be removed?
          if (uri.toString.contains("www")) {
            collector.add(normalizedEdmRightsWWWMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
          }
          // change /page/ to /vocab/
          val path = if (uri.getPath.contains("/page/")) {
            collector.add(normalizedEdmRightsRsPageMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
            uri.getPath.replaceFirst("/page/", "/vocab/")
          } else {
            uri.getPath
          }
          // Strip `?` and all following
          if (uri.getQuery != null) {
            collector.add(normalizedEdmRightsRsPageMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
          }
          // trailing `/` on path
          if (!uri.getPath.endsWith("/")) {
            collector.add(normalizedEdmRightsTrailingSlashMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
          }
          // trailing punctuation
          if (!uri.getPath.equalsIgnoreCase(uri.getPath.cleanupEndingPunctuation)) {
            collector.add(normalizedEdmRightsTrailingPunctuationMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
          }

          val uriString = s"http://${uri.getHost}${path.cleanupEndingPunctuation}/" // force http, drop parameters and trailing punctuation

          // normalize() drops duplicates //
          Try { new java.net.URI(uriString).normalize.toString} match {<commit_msg>Trap all errors during edmRights normalization (#421)<commit_after>package dpla.ingestion3.mappers

import dpla.ingestion3.enrichments.normalizations.StringNormalizationUtils._
import dpla.ingestion3.mappers.utils._
import dpla.ingestion3.messages.{IngestMessage, IngestMessageTemplates, MessageCollector}
import dpla.ingestion3.model.DplaMapData.{ExactlyOne, ZeroToMany, ZeroToOne}
import dpla.ingestion3.model._
import dpla.ingestion3.utils.Utils._
import org.json4s.DefaultFormats
import org.json4s.JsonAST._

import scala.util.{Failure, Success, Try}
import scala.xml.NodeSeq

trait Mapper[T, +E] extends IngestMessageTemplates {
  implicit val formats: DefaultFormats.type = DefaultFormats

  def map(document: Document[T], mapping: Mapping[T]): OreAggregation



  /**
    * Normalizes edmRights URIs and logs specific messages for each transformation (logged as warnings)
    *   1. HTTPS > HTTP
    *   2. Drop `www`
    *   3. Change `/page/` to `/vocab/`
    *   4. Drop query parameters (ex. ?lang=en)
    *   5. Add missing `/` to end of URI
    *   6. Remove trailing punctuation (ex. http://rightsstaments.org/vocab/Inc/;)
    *   7. Leading and trailing whitespace also remove but no messages are logged for this transformation
    *
    * If original value is not a valid java.net.URI the original value is returned
    *
    * @param values Seq[URI]              edmRights URIs to be normalized
    * @param providerId String            Provider supplied record identifier
    * @param collector MessageCollector   Ingest message collector
    * @return Seq[URI]                    URIs
    */
  def normalizeEdmRights(values: ZeroToMany[URI], providerId: String)
                        (implicit collector: MessageCollector[IngestMessage]): ZeroToMany[URI] = {
    values.map (value => {
      val normalized = Try { new java.net.URI(value.toString.trim) } match {
        case Success(uri) => {
          Try {

            // does scheme (http/https) require normalization
            if (uri.toString.startsWith("https")) {
              collector.add(normalizedEdmRightsHttpsMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
            }
            // `www` to be removed?
            if (uri.toString.contains("www")) {
              collector.add(normalizedEdmRightsWWWMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
            }
            // change /page/ to /vocab/
            val path = if (uri.getPath.contains("/page/")) {
              collector.add(normalizedEdmRightsRsPageMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
              uri.getPath.replaceFirst("/page/", "/vocab/")
            } else {
              uri.getPath
            }
            // Strip `?` and all following
            if (uri.getQuery != null) {
              collector.add(normalizedEdmRightsRsPageMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
            }
            // trailing `/` on path
            if (!uri.getPath.endsWith("/")) {
              collector.add(normalizedEdmRightsTrailingSlashMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
            }
            // trailing punctuation
            if (!uri.getPath.equalsIgnoreCase(uri.getPath.cleanupEndingPunctuation)) {
              collector.add(normalizedEdmRightsTrailingPunctuationMsg(providerId, "edmRights", value.toString, msg = None, enforce = false))
            }

            val uriString = s"http://${uri.getHost}${path.cleanupEndingPunctuation}/" // force http, drop parameters and trailing punctuation

            // normalize() drops duplicates //
           new java.net.URI(uriString).normalize.toString

          } match {
{'fraction_non_alphanumeric': 0.10764451647757968, 'fraction_numerical': 0.004727174500270125, 'mean_word_length': 2.6859133897461422, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5102143', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2165, 'n_words': 522}