<commit_before>/**
 * Copyright (C) 2015-2016 DANS - Data Archiving and Networked Services (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package nl.knaw.dans.easy.stage.dataset

import java.io.File
import java.net.URI

import nl.knaw.dans.easy.stage.dataset.Util.loadBagXML
import nl.knaw.dans.easy.stage.lib.Constants
import nl.knaw.dans.easy.stage.{RejectedDepositException, Settings}
import org.apache.commons.io.FileUtils
import org.joda.time.DateTime

import scala.util.{Success, Try}
import scala.xml.{Node, NodeSeq}

object AdditionalLicense {
  type MimeType = String
  private val ddmFileName = "metadata/dataset.xml"

  def createOptionally(sdo: File)(implicit s: Settings): Try[Option[MimeType]] =
   if((loadBagXML(ddmFileName) \\ "DDM" \ "dcmiMetadata" \ "license").isEmpty) Success(None)
   else create(sdo).map(m => Some(m))

  def create(sdo: File)(implicit s: Settings): Try[MimeType] =
    for {
      ddm <- Try{loadBagXML(ddmFileName)}
      (template, mime) <- getAdditionalLicenseTemplate(ddm)
      rightsHolder <- getRightsHolder(ddm)
      year <- getYear(ddm)
      _ <- copyAdditionalLicense(template, rightsHolder, year, sdo)
    } yield mime

  def getAdditionalLicenseTemplate(ddm: NodeSeq)(implicit s: Settings): Try[(String, MimeType)] = Try {
    val licenses = ddm \\ "DDM" \ "dcmiMetadata" \ "license"
    licenses match {
      case Seq(license) =>
          if(hasXsiType(license, "http://purl.org/dc/terms/", "URI")) {
            val licenseTemplateFile = getMatchingLicense(new URI(license.text), s.licenses).getOrElse(throw RejectedDepositException(s"Not a valid license URI: ${license.text}"))
            (FileUtils.readFileToString(licenseTemplateFile, "UTF-8"), getLicenseMimeType(licenseTemplateFile.getName))
          }
          else (license.text, "text/plain")
      case lics => throw RejectedDepositException(s"Found ${lics.size} dcterms:license elements. There should be exactly one")
    }
  }

  /**
   * Retrieves a matching license, while being liberal in what it accepts:
   *
   * - both http and https accepted
   * - a trailing slash is ignored
   *
   * @param uri the URI of the license
   * @param licenses the map from license URI-string to license File
   * @return the license File if found, otherwise None
   */
  def getMatchingLicense(uri: URI, licenses: Map[String, File]): Option[File] = {
    def withScheme(uri: URI, scheme: String) = new URI(scheme, uri.getUserInfo, uri.getHost, uri.getPort, uri.getPath, uri.getQuery, uri.getFragment)

    val httpUri =
      if (uri.getScheme == "https") withScheme(uri, "https")
      else uri

    licenses.get(httpUri.toASCIIString).orElse {
      if (httpUri.toASCIIString.last == '/') licenses.get(httpUri.toASCIIString.init)
      else None
    }
  }

  def hasXsiType(e: Node, attributeNamespace: String, attributeValue: String): Boolean =
   e.head.attribute("http://www.w3.org/2001/XMLSchema-instance", "type") match {
     case Some(Seq(n)) => n.text.split("\\:") match {
       case Array(pref, label) => e.head.getNamespace(pref) == attributeNamespace && label == attributeValue
       case _ => false
     }
     case _ => false
   }

  def getLicenseMimeType(licenseFileName: String): MimeType =
    licenseFileName.split("\\.").last match {
      case "txt" => "text/plain"
      case "html" => "text/html"
      case ext => throw RejectedDepositException(s"Unknown extension for license: .$ext")
    }

  def copyAdditionalLicense(template: String, rightsHolder: String, year: String, sdo: File): Try[File] = Try {<commit_msg>Normalization of http:// and https:// licence URLs was broken by 2435e14eba1611b51a1fb748ada4c7f66a6d2da7<commit_after>/**
 * Copyright (C) 2015-2016 DANS - Data Archiving and Networked Services (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package nl.knaw.dans.easy.stage.dataset

import java.io.File
import java.net.URI

import nl.knaw.dans.easy.stage.dataset.Util.loadBagXML
import nl.knaw.dans.easy.stage.lib.Constants
import nl.knaw.dans.easy.stage.{RejectedDepositException, Settings}
import org.apache.commons.io.FileUtils
import org.joda.time.DateTime

import scala.util.{Success, Try}
import scala.xml.{Node, NodeSeq}

object AdditionalLicense {
  type MimeType = String
  private val ddmFileName = "metadata/dataset.xml"

  def createOptionally(sdo: File)(implicit s: Settings): Try[Option[MimeType]] =
   if((loadBagXML(ddmFileName) \\ "DDM" \ "dcmiMetadata" \ "license").isEmpty) Success(None)
   else create(sdo).map(m => Some(m))

  def create(sdo: File)(implicit s: Settings): Try[MimeType] =
    for {
      ddm <- Try{loadBagXML(ddmFileName)}
      (template, mime) <- getAdditionalLicenseTemplate(ddm)
      rightsHolder <- getRightsHolder(ddm)
      year <- getYear(ddm)
      _ <- copyAdditionalLicense(template, rightsHolder, year, sdo)
    } yield mime

  def getAdditionalLicenseTemplate(ddm: NodeSeq)(implicit s: Settings): Try[(String, MimeType)] = Try {
    val licenses = ddm \\ "DDM" \ "dcmiMetadata" \ "license"
    licenses match {
      case Seq(license) =>
          if(hasXsiType(license, "http://purl.org/dc/terms/", "URI")) {
            val licenseTemplateFile = getMatchingLicense(new URI(license.text), s.licenses).getOrElse(throw RejectedDepositException(s"Not a valid license URI: ${license.text}"))
            (FileUtils.readFileToString(licenseTemplateFile, "UTF-8"), getLicenseMimeType(licenseTemplateFile.getName))
          }
          else (license.text, "text/plain")
      case lics => throw RejectedDepositException(s"Found ${lics.size} dcterms:license elements. There should be exactly one")
    }
  }

  /**
   * Retrieves a matching license, while being liberal in what it accepts:
   *
   * - both http and https accepted
   * - a trailing slash is ignored
   *
   * @param uri the URI of the license
   * @param licenses the map from license URI-string to license File
   * @return the license File if found, otherwise None
   */
  def getMatchingLicense(uri: URI, licenses: Map[String, File]): Option[File] = {
    def withScheme(uri: URI, scheme: String) = new URI(scheme, uri.getUserInfo, uri.getHost, uri.getPort, uri.getPath, uri.getQuery, uri.getFragment)

    val httpUri =
      if (uri.getScheme == "https") withScheme(uri, "http")
      else uri

    licenses.get(httpUri.toASCIIString).orElse {
      if (httpUri.toASCIIString.last == '/') licenses.get(httpUri.toASCIIString.init)
      else None
    }
  }

  def hasXsiType(e: Node, attributeNamespace: String, attributeValue: String): Boolean =
   e.head.attribute("http://www.w3.org/2001/XMLSchema-instance", "type") match {
     case Some(Seq(n)) => n.text.split("\\:") match {
       case Array(pref, label) => e.head.getNamespace(pref) == attributeNamespace && label == attributeValue
       case _ => false
     }
     case _ => false
   }

  def getLicenseMimeType(licenseFileName: String): MimeType =
    licenseFileName.split("\\.").last match {
      case "txt" => "text/plain"
      case "html" => "text/html"
      case ext => throw RejectedDepositException(s"Unknown extension for license: .$ext")
    }

  def copyAdditionalLicense(template: String, rightsHolder: String, year: String, sdo: File): Try[File] = Try {
{'fraction_non_alphanumeric': 0.1212856276531231, 'fraction_numerical': 0.007155852031534263, 'mean_word_length': 3.714694110920526, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '680175', 'n_tokens_mistral': 2609, 'n_tokens_neox': 2508, 'n_words': 700}