<commit_before>package core.rendering

import core.Constants._
import core.HubId
import core.search.{DelvingIdType, BriefDocItem, DocItemReference, SolrQueryService}
import models.{Role, RecordDefinition, MetadataCache, DomainConfiguration}
import java.net.{URLDecoder, URLEncoder}
import play.api.Logger
import xml._
import play.api.i18n.Lang
import eu.delving.schema.SchemaVersion

/**
 * Renders a single record, with or without related items. Makes use of the search engine to retrieve related records and IDs.
 *
 * TODO simplify and consolidate this mess when we've got some time.
 *
 * @author <NAME> <<EMAIL>>
 */

object RecordRenderer {

  val log = Logger("CultureHub")

  /**
   * Gets the full view of a record, for access via the search API.
   * This is complex because we have to cater for legacy deployments with special idTypes
   */
  def getRenderedFullView(id: String,
                          idType: DelvingIdType,
                          lang: Lang,
                          schema: Option[String] = None,
                          renderRelatedItems: Boolean,
                          relatedItemsCount: Int)(implicit configuration: DomainConfiguration): Either[String, RenderedView] = {


    SolrQueryService.getSolrItemReference(URLEncoder.encode(id, "utf-8"), idType, renderRelatedItems, relatedItemsCount) match {
      case Some(DocItemReference(hubId, defaultSchema, publicSchemas, relatedItems, item)) =>
        val prefix = if(schema.isDefined && publicSchemas.contains(schema.get)) {
          schema.get
        } else if(schema.isDefined && !publicSchemas.contains(schema.get)) {
          val m = "Schema '%s' not available for hubId '%s'".format(schema.get, hubId)
          Logger("Search").info(m)
          return Left(m)
        } else {
          defaultSchema
        }

        idType match {
          case DelvingIdType.ITIN =>
            // TODO legacy support, to be removed on 01.06.2013
            renderItinItem(item, relatedItems)
          case DelvingIdType.INDEX_ITEM =>
            renderIndexItem(id)
          case _ =>
            renderMetadataRecord(prefix, URLDecoder.decode(hubId, "utf-8"), ViewType.API, lang, renderRelatedItems, relatedItems)
        }
      case None =>
        Left("Could not resolve identifier for hubId '%s' and idType '%s'".format(id, idType.idType))
    }
  }

  private def renderItinItem(item: Option[BriefDocItem], relatedItems: Seq[BriefDocItem]) = {

    val document = <result xmlns:delving="http://www.delving.eu/schemas/"
                           xmlns:icn="http://www.icn.nl/"
                           xmlns:dc="http://purl.org/dc/elements/1.1/"
                           xmlns:custom="http://www.delving.eu/schemas/"
                           xmlns:dcterms="http://purl.org/dc/termes/"
                           xmlns:itin="http://www.itin.nl/namespace"
                           xmlns:drup="http://www.itin.nl/drupal"
                           xmlns:europeana="http://www.europeana.eu/schemas/ese/">

      { item.map { i => {i.toXml()} }.getOrElse(<item/>) }
      <relatedItems>{ relatedItems.map { ri => {ri.toXml()} }}</relatedItems>
    </result>

    Right(new RenderedView {
      def toXml: NodeSeq = document
      def toViewTree: RenderNode = null
      def toXmlString: String = document.toString()
      def toJson: String = "JSON rendering is unsupported"
    })

  }

  private def renderIndexItem(id: String): Either[String, RenderedView] = {
    if(id.split("_").length < 3) {
      Left("Invalid hubId")
    } else {
      val hubId = HubId(id)
      val cache = MetadataCache.get(hubId.orgId, "indexApiItems", hubId.spec)
      val indexItem = cache.findOne(hubId.localId).getOrElse(return Left("Could not find IndexItem with id '%s".format(id)))
      Right(new RenderedView {
        def toXmlString: String = indexItem.xml("raw")
        def toJson: String = "JSON rendering not supported"
        def toXml: NodeSeq = scala.xml.XML.loadString(indexItem.xml("raw"))
        def toViewTree: RenderNode = null
      })
    }
  }

  private def renderMetadataRecord(prefix: String, hubId: String, viewType: ViewType, lang: Lang, renderRelatedItems: Boolean, relatedItems: Seq[BriefDocItem])(implicit configuration: DomainConfiguration): Either[String, RenderedView] = {
    if(hubId.split("_").length < 3) return Left("Invalid hubId " + hubId)
    val id = HubId(hubId)
    val cache = MetadataCache.get(id.orgId, id.spec, ITEM_TYPE_MDR)
    val record = cache.findOne(hubId)
    val rawRecord: Option[String] = record.flatMap(_.xml.get(prefix))
    if (rawRecord.isEmpty) {
      Logger("Search").info("Could not find cached record in mongo with format %s for hubId %s".format(prefix, hubId))
      Left("Could not find full record with hubId '%s' for format '%s'".format(hubId, prefix))
    } else {

      // handle legacy formats
      val legacyApiFormats = List("tib", "abm", "ese", "abc")
      val legacyHtmlFormats = List("abm", "ese", "abc")
      val viewDefinitionFormatName = if (viewType == ViewType.API) {
        if(legacyApiFormats.contains(prefix)) "legacy" else prefix
      } else {
        if(legacyHtmlFormats.contains(prefix)) "legacy" else prefix
      }

      val schemaVersion = record.get.schemaVersions(prefix)

      renderMetadataRecord(hubId, rawRecord.get, new SchemaVersion(prefix, schemaVersion), viewDefinitionFormatName, viewType, lang, renderRelatedItems, relatedItems.map(_.toXml()))
    }
  }

  /**
   * Renders a metadata record
   *
   * @param hubId the hubId
   * @param recordXml the raw record (XML string)
   * @param schema the schema being rendered
   * @param viewDefinitionFormatName prefix of the view rendering schema. in principle the same as the one of the record, but there may be exceptions (e.g. "legacy")
   * @param viewType viewType (API / HTML)
   * @param lang rendering languages
   * @param renderRelatedItems whether to render related items
   * @param relatedItems the related items
   * @param configuration DomainConfiguration
   * @return a rendered view if successful, or an error message
   */
  def renderMetadataRecord(hubId: String,
                           recordXml: String,
                           schema: SchemaVersion,
                           viewDefinitionFormatName: String,
                           viewType: ViewType,
                           lang: Lang,
                           renderRelatedItems: Boolean = false,
                           relatedItems: Seq[NodeSeq] = Seq.empty,
                           roles: Seq[Role] = Seq.empty,
                           parameters: Map[String, String] = Map.empty)(implicit configuration: DomainConfiguration): Either[String, RenderedView]  = {

      // let's do some rendering
      RecordDefinition.getRecordDefinition(schema) match {
        case Some(definition) =>
          val viewRenderer = ViewRenderer.fromDefinition(viewDefinitionFormatName, viewType)
          if (viewRenderer.isEmpty) {
            log.warn("Tried rendering full record with id '%s' for non-existing view type '%s'".format(hubId, viewType.name))
            Left("Could not render full record with hubId '%s' for view type '%s': view type does not exist".format(hubId, viewType.name))
          } else {
            try {

              val defaultNamespaces = Seq(
                "delving" -> "http://schemas.delving.eu/",
                "xml" -> "http://www.w3.org/XML/1998/namespace",
                "xsi" -> "http://www.w3.org/2001/XMLSchema-instance"
              )

              val relatedItemsNamespaces: Seq[(String, String)] = {
                val namespaces = relatedItems.flatMap { item =>
                  item.map { field =>
                    field.prefix
                  }.distinct.map { prefix =>
                    (prefix -> RecordDefinition.getNamespaceURI(prefix))
                  }
                }

                val (resolved, missing) = namespaces.partition(_._2.isDefined)

                log.warn("While rendering full view for item %s: following prefixes for related items are unknown: %s".format(
                  hubId, missing.map(_._1).mkString(", ")
                ))

                resolved.map(r => (r._1 -> r._2.get))
              }


              val cleanRawRecord = {
                val record = scala.xml.XML.loadString(recordXml)
                var mutableRecord: Elem = if (renderRelatedItems) {
                  // mix the related items to the cached record coming from mongo
                  val relatedItemsXml = <relatedItems>{relatedItems}</relatedItems>
                  addChild(record, relatedItemsXml).get // we know what we're doing here
                } else {
                  record
                }

                val additionalNamespaces: Map[String, String] = definition.getNamespaces ++ relatedItemsNamespaces ++ defaultNamespaces<commit_msg>Fixing detection of related items<commit_after>package core.rendering

import core.Constants._
import core.HubId
import core.search.{DelvingIdType, BriefDocItem, DocItemReference, SolrQueryService}
import models.{Role, RecordDefinition, MetadataCache, DomainConfiguration}
import java.net.{URLDecoder, URLEncoder}
import play.api.Logger
import xml._
import play.api.i18n.Lang
import eu.delving.schema.SchemaVersion

/**
 * Renders a single record, with or without related items. Makes use of the search engine to retrieve related records and IDs.
 *
 * TODO simplify and consolidate this mess when we've got some time.
 *
 * @author <NAME> <<EMAIL>>
 */

object RecordRenderer {

  val log = Logger("CultureHub")

  /**
   * Gets the full view of a record, for access via the search API.
   * This is complex because we have to cater for legacy deployments with special idTypes
   */
  def getRenderedFullView(id: String,
                          idType: DelvingIdType,
                          lang: Lang,
                          schema: Option[String] = None,
                          renderRelatedItems: Boolean,
                          relatedItemsCount: Int)(implicit configuration: DomainConfiguration): Either[String, RenderedView] = {


    SolrQueryService.getSolrItemReference(URLEncoder.encode(id, "utf-8"), idType, renderRelatedItems, relatedItemsCount) match {
      case Some(DocItemReference(hubId, defaultSchema, publicSchemas, relatedItems, item)) =>
        val prefix = if(schema.isDefined && publicSchemas.contains(schema.get)) {
          schema.get
        } else if(schema.isDefined && !publicSchemas.contains(schema.get)) {
          val m = "Schema '%s' not available for hubId '%s'".format(schema.get, hubId)
          Logger("Search").info(m)
          return Left(m)
        } else {
          defaultSchema
        }

        idType match {
          case DelvingIdType.ITIN =>
            // TODO legacy support, to be removed on 01.06.2013
            renderItinItem(item, relatedItems)
          case DelvingIdType.INDEX_ITEM =>
            renderIndexItem(id)
          case _ =>
            renderMetadataRecord(prefix, URLDecoder.decode(hubId, "utf-8"), ViewType.API, lang, renderRelatedItems, relatedItems)
        }
      case None =>
        Left("Could not resolve identifier for hubId '%s' and idType '%s'".format(id, idType.idType))
    }
  }

  private def renderItinItem(item: Option[BriefDocItem], relatedItems: Seq[BriefDocItem]) = {

    val document = <result xmlns:delving="http://www.delving.eu/schemas/"
                           xmlns:icn="http://www.icn.nl/"
                           xmlns:dc="http://purl.org/dc/elements/1.1/"
                           xmlns:custom="http://www.delving.eu/schemas/"
                           xmlns:dcterms="http://purl.org/dc/termes/"
                           xmlns:itin="http://www.itin.nl/namespace"
                           xmlns:drup="http://www.itin.nl/drupal"
                           xmlns:europeana="http://www.europeana.eu/schemas/ese/">

      { item.map { i => {i.toXml()} }.getOrElse(<item/>) }
      <relatedItems>{ relatedItems.map { ri => {ri.toXml()} }}</relatedItems>
    </result>

    Right(new RenderedView {
      def toXml: NodeSeq = document
      def toViewTree: RenderNode = null
      def toXmlString: String = document.toString()
      def toJson: String = "JSON rendering is unsupported"
    })

  }

  private def renderIndexItem(id: String): Either[String, RenderedView] = {
    if(id.split("_").length < 3) {
      Left("Invalid hubId")
    } else {
      val hubId = HubId(id)
      val cache = MetadataCache.get(hubId.orgId, "indexApiItems", hubId.spec)
      val indexItem = cache.findOne(hubId.localId).getOrElse(return Left("Could not find IndexItem with id '%s".format(id)))
      Right(new RenderedView {
        def toXmlString: String = indexItem.xml("raw")
        def toJson: String = "JSON rendering not supported"
        def toXml: NodeSeq = scala.xml.XML.loadString(indexItem.xml("raw"))
        def toViewTree: RenderNode = null
      })
    }
  }

  private def renderMetadataRecord(prefix: String, hubId: String, viewType: ViewType, lang: Lang, renderRelatedItems: Boolean, relatedItems: Seq[BriefDocItem])(implicit configuration: DomainConfiguration): Either[String, RenderedView] = {
    if(hubId.split("_").length < 3) return Left("Invalid hubId " + hubId)
    val id = HubId(hubId)
    val cache = MetadataCache.get(id.orgId, id.spec, ITEM_TYPE_MDR)
    val record = cache.findOne(hubId)
    val rawRecord: Option[String] = record.flatMap(_.xml.get(prefix))
    if (rawRecord.isEmpty) {
      Logger("Search").info("Could not find cached record in mongo with format %s for hubId %s".format(prefix, hubId))
      Left("Could not find full record with hubId '%s' for format '%s'".format(hubId, prefix))
    } else {

      // handle legacy formats
      val legacyApiFormats = List("tib", "abm", "ese", "abc")
      val legacyHtmlFormats = List("abm", "ese", "abc")
      val viewDefinitionFormatName = if (viewType == ViewType.API) {
        if(legacyApiFormats.contains(prefix)) "legacy" else prefix
      } else {
        if(legacyHtmlFormats.contains(prefix)) "legacy" else prefix
      }

      val schemaVersion = record.get.schemaVersions(prefix)

      renderMetadataRecord(hubId, rawRecord.get, new SchemaVersion(prefix, schemaVersion), viewDefinitionFormatName, viewType, lang, renderRelatedItems, relatedItems.map(_.toXml()))
    }
  }

  /**
   * Renders a metadata record
   *
   * @param hubId the hubId
   * @param recordXml the raw record (XML string)
   * @param schema the schema being rendered
   * @param viewDefinitionFormatName prefix of the view rendering schema. in principle the same as the one of the record, but there may be exceptions (e.g. "legacy")
   * @param viewType viewType (API / HTML)
   * @param lang rendering languages
   * @param renderRelatedItems whether to render related items
   * @param relatedItems the related items
   * @param configuration DomainConfiguration
   * @return a rendered view if successful, or an error message
   */
  def renderMetadataRecord(hubId: String,
                           recordXml: String,
                           schema: SchemaVersion,
                           viewDefinitionFormatName: String,
                           viewType: ViewType,
                           lang: Lang,
                           renderRelatedItems: Boolean = false,
                           relatedItems: Seq[NodeSeq] = Seq.empty,
                           roles: Seq[Role] = Seq.empty,
                           parameters: Map[String, String] = Map.empty)(implicit configuration: DomainConfiguration): Either[String, RenderedView]  = {

      // let's do some rendering
      RecordDefinition.getRecordDefinition(schema) match {
        case Some(definition) =>
          val viewRenderer = ViewRenderer.fromDefinition(viewDefinitionFormatName, viewType)
          if (viewRenderer.isEmpty) {
            log.warn("Tried rendering full record with id '%s' for non-existing view type '%s'".format(hubId, viewType.name))
            Left("Could not render full record with hubId '%s' for view type '%s': view type does not exist".format(hubId, viewType.name))
          } else {
            try {

              val defaultNamespaces = Seq(
                "delving" -> "http://schemas.delving.eu/",
                "xml" -> "http://www.w3.org/XML/1998/namespace",
                "xsi" -> "http://www.w3.org/2001/XMLSchema-instance"
              )

              val relatedItemsNamespaces: Seq[(String, String)] = {
                val namespaces = relatedItems.flatMap { item =>
                  (item \\ "fields").flatMap { field =>
                    field.child.map { field =>
                      field.prefix
                    }
                  }.distinct.map { prefix =>
                    (prefix -> RecordDefinition.getNamespaceURI(prefix))
                  }
                }

                val (resolved, missing) = namespaces.partition(_._2.isDefined)

                log.warn("While rendering full view for item %s: following prefixes for related items are unknown: %s".format(
                  hubId, missing.map(_._1).mkString(", ")
                ))

                resolved.map(r => (r._1 -> r._2.get))
              }


              val cleanRawRecord = {
                val record = scala.xml.XML.loadString(recordXml)
                var mutableRecord: Elem = if (renderRelatedItems) {
                  // mix the related items to the cached record coming from mongo
                  val relatedItemsXml = <relatedItems>{relatedItems}</relatedItems>
                  addChild(record, relatedItemsXml).get // we know what we're doing here
                } else {
                  record
                }

                val additionalNamespaces: Map[String, String] = definition.getNamespaces ++ relatedItemsNamespaces ++ defaultNamespaces
{'fraction_non_alphanumeric': 0.10687363895248199, 'fraction_numerical': 0.003350270813557429, 'mean_word_length': 2.292279411764706, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 16, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3864000', 'n_tokens_mistral': 5040, 'n_tokens_neox': 4835, 'n_words': 1282}