<commit_before>package io.mediachain.translation.tate

import io.mediachain.translation._
import cats.data.Xor
import io.mediachain.core.TranslationError
import io.mediachain.core.TranslationError.InvalidFormat
import io.mediachain.Types._

import org.json4s._

object TateTranslator extends Translator {
  val name = "TateCreativeCommons"
  val version = 1

  private case class Contributor(fc: String, role: String)
  private case class Artwork(title: String,
                             medium: Option[String],
                             dateText: Option[String],
                             contributors: List[Contributor],
                             id: Int,
                             acno: String)

  def translate(json: JObject): Xor[TranslationError, ImageBlob] = {
    implicit val formats = org.json4s.DefaultFormats
    val artwork = json.extractOpt[Artwork]
    val result = artwork.map { a =>

      val artists = for {
        c <- a.contributors
        if c.role == "artist"
      } yield Person(None, c.fc)

      ImageBlob(None,
        a.title,
        a.medium.getOrElse(""),
        a.dateText.getOrElse(""),
        artists.headOption,
        external_ids = Map("tate:id" -> a.id.toString, "tate:acno" -> a.acno)
      )
    }

    Xor.fromOption(result, InvalidFormat())
  }
}

class TateLoader(val path: String, implicit val translator: TateTranslator.type = TateTranslator) extends DirectoryWalkerLoader[TateTranslator.type]
<commit_msg>Extract Tate author id<commit_after>package io.mediachain.translation.tate

import io.mediachain.translation._
import cats.data.Xor
import io.mediachain.core.TranslationError
import io.mediachain.core.TranslationError.InvalidFormat
import io.mediachain.Types._

import org.json4s._

object TateTranslator extends Translator {
  val name = "TateCreativeCommons"
  val version = 1

  private case class Contributor(fc: String, role: String, id: String)
  private case class Artwork(title: String,
                             medium: Option[String],
                             dateText: Option[String],
                             contributors: List[Contributor],
                             id: Int,
                             acno: String)

  def translate(json: JObject): Xor[TranslationError, ImageBlob] = {
    implicit val formats = org.json4s.DefaultFormats
    val artwork = json.extractOpt[Artwork]
    val result = artwork.map { a =>

      val artists = for {
        c <- a.contributors
        if c.role == "artist"
      } yield Person(None, c.fc, external_ids = Map("tate:id" -> c.id.toString))

      ImageBlob(None,
        a.title,
        a.medium.getOrElse(""),
        a.dateText.getOrElse(""),
        artists.headOption,
        external_ids = Map("tate:id" -> a.id.toString, "tate:acno" -> a.acno)
      )
    }

    Xor.fromOption(result, InvalidFormat())
  }
}

class TateLoader(val path: String, implicit val translator: TateTranslator.type = TateTranslator) extends DirectoryWalkerLoader[TateTranslator.type]

{'fraction_non_alphanumeric': 0.10056422170594093, 'fraction_numerical': 0.0019913707268503153, 'mean_word_length': 2.4723502304147464, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1698796', 'n_tokens_mistral': 915, 'n_tokens_neox': 882, 'n_words': 200}