<commit_before>package dpla.ingestion3.enrichments

import dpla.ingestion3.model._
import dpla.ingestion3.enrichments.StringUtils._
import dpla.ingestion3.model.DplaMapData.LiteralOrUri

/**
  * Universal String enrichments.
  *
  */
class StringEnrichments {

  /**
    * Main entry point.
    *
    * @param record OreAggregation
    * @return enriched OreAggregation
    */
  def enrich(record: OreAggregation): OreAggregation = {
    record.copy(
      sourceResource = enrichSourceResource(record.sourceResource),
      dataProvider = enrichEdmAgent(record.dataProvider),
      hasView = record.hasView.map(enrichEdmWebResource(_)),
      intermediateProvider = record.intermediateProvider.map(enrichEdmAgent(_)),
      isShownAt = enrichEdmWebResource(record.isShownAt),
      `object` = record.`object`.map(enrichEdmWebResource(_)),
      preview = record.preview.map(enrichEdmWebResource(_)),
      provider = enrichEdmAgent(record.provider)
    )
  }

  def enrichSourceResource(sourceResource: DplaSourceResource): DplaSourceResource =
    sourceResource.copy(
      alternateTitle = sourceResource.alternateTitle.map(_.stripHTML.reduceWhitespace),
      collection = sourceResource.collection.map(enrichDcmiTypeCollection(_)),
      contributor = sourceResource.contributor.map(enrichEdmAgent(_)),
      creator = sourceResource.creator.map(enrichEdmAgent(_)),
      date = sourceResource.date.map(enrichEdmTimeSpan(_)),
      description = sourceResource.description.map(_.stripHTML.reduceWhitespace),
      extent = sourceResource.extent.map(_.stripHTML.reduceWhitespace),
      format = sourceResource.format.map(_.stripHTML.reduceWhitespace),
      genre = sourceResource.genre.map(enrichSkosConcept(_)),
      identifier = sourceResource.identifier.map(_.stripHTML.reduceWhitespace),
      language = sourceResource.language.map(enrichSkosConcept(_)),
      place = sourceResource.place.map(enrichDplaPlace(_)),
      publisher = sourceResource.publisher.map(enrichEdmAgent(_)),
      relation = sourceResource.relation.map(enrichRelation(_)),
      replacedBy = sourceResource.replacedBy.map(_.stripHTML.reduceWhitespace),
      replaces = sourceResource.replaces.map(_.stripHTML.reduceWhitespace),
      rights = sourceResource.rights.map(_.stripHTML.reduceWhitespace),
      rightsHolder = sourceResource.rightsHolder.map(enrichEdmAgent(_)),
      subject = sourceResource.subject.map(enrichSkosConcept(_)),
      temporal = sourceResource.temporal.map(enrichEdmTimeSpan(_)),
      title = sourceResource.title.map(_.stripHTML.reduceWhitespace),
      `type` = sourceResource.`type`.map(_.stripHTML.reduceWhitespace)
    )

  def enrichEdmAgent(edmAgent: EdmAgent): EdmAgent =
    edmAgent.copy(
      name = edmAgent.name.map(_.stripHTML.reduceWhitespace)
    )

  def enrichEdmWebResource(edmWebResource: EdmWebResource): EdmWebResource =
    edmWebResource.copy(
      fileFormat = edmWebResource.fileFormat.map(_.stripHTML.reduceWhitespace),
      dcRights = edmWebResource.dcRights.map(_.stripHTML.reduceWhitespace),
      edmRights = edmWebResource.edmRights.map(_.stripHTML.reduceWhitespace)
    )

  def enrichSkosConcept(skosConcept: SkosConcept): SkosConcept =
    skosConcept.copy(
      concept = skosConcept.concept.map(_.stripHTML.reduceWhitespace)
    )
<commit_msg>Add providedLabel field to enrichSkosConcept (#133)<commit_after>package dpla.ingestion3.enrichments

import dpla.ingestion3.model._
import dpla.ingestion3.enrichments.StringUtils._
import dpla.ingestion3.model.DplaMapData.LiteralOrUri

/**
  * Universal String enrichments.
  *
  */
class StringEnrichments {

  /**
    * Main entry point.
    *
    * @param record OreAggregation
    * @return enriched OreAggregation
    */
  def enrich(record: OreAggregation): OreAggregation = {
    record.copy(
      sourceResource = enrichSourceResource(record.sourceResource),
      dataProvider = enrichEdmAgent(record.dataProvider),
      hasView = record.hasView.map(enrichEdmWebResource(_)),
      intermediateProvider = record.intermediateProvider.map(enrichEdmAgent(_)),
      isShownAt = enrichEdmWebResource(record.isShownAt),
      `object` = record.`object`.map(enrichEdmWebResource(_)),
      preview = record.preview.map(enrichEdmWebResource(_)),
      provider = enrichEdmAgent(record.provider)
    )
  }

  def enrichSourceResource(sourceResource: DplaSourceResource): DplaSourceResource =
    sourceResource.copy(
      alternateTitle = sourceResource.alternateTitle.map(_.stripHTML.reduceWhitespace),
      collection = sourceResource.collection.map(enrichDcmiTypeCollection(_)),
      contributor = sourceResource.contributor.map(enrichEdmAgent(_)),
      creator = sourceResource.creator.map(enrichEdmAgent(_)),
      date = sourceResource.date.map(enrichEdmTimeSpan(_)),
      description = sourceResource.description.map(_.stripHTML.reduceWhitespace),
      extent = sourceResource.extent.map(_.stripHTML.reduceWhitespace),
      format = sourceResource.format.map(_.stripHTML.reduceWhitespace),
      genre = sourceResource.genre.map(enrichSkosConcept(_)),
      identifier = sourceResource.identifier.map(_.stripHTML.reduceWhitespace),
      language = sourceResource.language.map(enrichSkosConcept(_)),
      place = sourceResource.place.map(enrichDplaPlace(_)),
      publisher = sourceResource.publisher.map(enrichEdmAgent(_)),
      relation = sourceResource.relation.map(enrichRelation(_)),
      replacedBy = sourceResource.replacedBy.map(_.stripHTML.reduceWhitespace),
      replaces = sourceResource.replaces.map(_.stripHTML.reduceWhitespace),
      rights = sourceResource.rights.map(_.stripHTML.reduceWhitespace),
      rightsHolder = sourceResource.rightsHolder.map(enrichEdmAgent(_)),
      subject = sourceResource.subject.map(enrichSkosConcept(_)),
      temporal = sourceResource.temporal.map(enrichEdmTimeSpan(_)),
      title = sourceResource.title.map(_.stripHTML.reduceWhitespace),
      `type` = sourceResource.`type`.map(_.stripHTML.reduceWhitespace)
    )

  def enrichEdmAgent(edmAgent: EdmAgent): EdmAgent =
    edmAgent.copy(
      name = edmAgent.name.map(_.stripHTML.reduceWhitespace)
    )

  def enrichEdmWebResource(edmWebResource: EdmWebResource): EdmWebResource =
    edmWebResource.copy(
      fileFormat = edmWebResource.fileFormat.map(_.stripHTML.reduceWhitespace),
      dcRights = edmWebResource.dcRights.map(_.stripHTML.reduceWhitespace),
      edmRights = edmWebResource.edmRights.map(_.stripHTML.reduceWhitespace)
    )

  def enrichSkosConcept(skosConcept: SkosConcept): SkosConcept =
    skosConcept.copy(
      concept = skosConcept.concept.map(_.stripHTML.reduceWhitespace),
      providedLabel = skosConcept.providedLabel.map(_.stripHTML.reduceWhitespace)
    )

{'fraction_non_alphanumeric': 0.10578782919208451, 'fraction_numerical': 0.0016366612111292963, 'mean_word_length': 4.587697423108894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6000353', 'n_tokens_mistral': 2100, 'n_tokens_neox': 2051, 'n_words': 241}