<commit_before>/**
 * Copyright (C) 2015-2016 DANS - Data Archiving and Networked Services (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package nl.knaw.dans.easy.stage

import java.io.File
import java.util

import nl.knaw.dans.common.lang.dataset.AccessCategory._
import nl.knaw.dans.easy.stage.EasyStageDataset._
import nl.knaw.dans.easy.stage.lib.Constants.DATASET_SDO
import org.apache.commons.io.FileUtils
import org.apache.commons.io.FileUtils.deleteDirectory
import org.apache.commons.io.filefilter.{DirectoryFileFilter, FileFileFilter}
import org.scalatest.{FlatSpec, Matchers}

import scala.util.{Failure, Success}

class EasyStageDatasetSpec extends FlatSpec with Matchers {

  private val tmpProps = new File("cfg/application.properties")

  "createFileAndFolderSdos" should "stumble if the data folder does not exist" in {

    val bagitDir = new File("target/test/bag")
    val dataDir = new File(bagitDir, "data")
    val sdoSetDir = new File("target/test/someSDO")
    implicit val s = createSettings(bagitDir, sdoSetDir)
    bagitDir.mkdirs()

    createFileAndFolderSdos(dataDir, DATASET_SDO, ANONYMOUS_ACCESS) shouldBe a[Failure[_]]

    deleteDirectory(bagitDir)
  }

  it should "be happy with an empty data folder" in {

    val bagitDir = new File("target/test/bag")
    val dataDir = new File(bagitDir, "data")
    val sdoSetDir = new File("target/test/someSDO")
    implicit val s = createSettings(bagitDir, sdoSetDir)
    dataDir.mkdirs()

    createFileAndFolderSdos(dataDir, DATASET_SDO, ANONYMOUS_ACCESS) shouldBe a[Success[_]]
    sdoSetDir.exists() shouldBe false

    deleteDirectory(bagitDir)
  }

  it should "create file rights computed from dataset access rights" in {

    createProps()
    val bagitDir = new File("src/test/resources/dataset-bags/no-additional-license")
    val dataDir = new File(bagitDir, "data")
    val sdoSetDir = new File("target/test/sdoSet")
    val fileMetadataFile = new File(sdoSetDir, "quicksort_hs/EASY_FILE_METADATA")
    implicit val s = createSettings(bagitDir, sdoSetDir)

    createFileAndFolderSdos(dataDir, DATASET_SDO, OPEN_ACCESS_FOR_REGISTERED_USERS) shouldBe a[Success[_]]
    FileUtils.readFileToString(fileMetadataFile) should include ("<visibleTo>KNOWN</visibleTo>")
    FileUtils.readFileToString(fileMetadataFile) should include ("<accessibleTo>KNOWN</accessibleTo>")
    deleteDirectory(sdoSetDir)

    createFileAndFolderSdos(dataDir, DATASET_SDO, ANONYMOUS_ACCESS) shouldBe a[Success[_]]
    FileUtils.readFileToString(fileMetadataFile) should include ("<visibleTo>ANONYMOUS</visibleTo>")
    FileUtils.readFileToString(fileMetadataFile) should include ("<accessibleTo>ANONYMOUS</accessibleTo>")
    deleteDirectory(sdoSetDir)

    tmpProps.delete()
  }

  "run" should "create SDO sets for all available test bags" in {

    createProps()
    val sdoSetDir = new File("target/test/sdoSet")
    for (bag <- getTestBags.toArray) {
      val bagitDir = new File("src/test/resources/dataset-bags/no-additional-license")
      implicit val s = createSettings(bagitDir, sdoSetDir)

      EasyStageDataset.run(s) shouldBe a[Success[_]]
      sdoSetDir.exists() shouldBe true

      deleteDirectory(sdoSetDir)
    }
    tmpProps.delete()
  }

  def createProps() = FileUtils.write(tmpProps, "owner=dsowner\nredirect-unset-url=http://unset.dans.knaw.nl")

  def createSettings(bagitDir: File, sdoSetDir: File): Settings = {
    Settings(
      ownerId = "dpositor",
      bagitDir = bagitDir,
      sdoSetDir = sdoSetDir,
      isMendeley = false,
      URN = Some("someUrn"),
      DOI = Some("doei"),
      disciplines = Map[String, String]("D10000" -> "easy-discipline:1")
    )
  }

  def getTestBags: util.Collection[File] = {
    val ff = new FileFileFilter {
      override def accept(pathname: File): Boolean = false
    }
    val df = new DirectoryFileFilter {
      override def accept(pathname: File): Boolean = true
    }
    FileUtils.listFilesAndDirs(new File("src/test/resources/dataset-bags/no-additional-license"), ff, df)
  }
}<commit_msg>Validate test bags of stage-dataset (#82)<commit_after>/**
 * Copyright (C) 2015-2016 DANS - Data Archiving and Networked Services (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package nl.knaw.dans.easy.stage

import java.io.File
import java.util

import nl.knaw.dans.common.lang.dataset.AccessCategory._
import nl.knaw.dans.easy.stage.EasyStageDataset._
import nl.knaw.dans.easy.stage.lib.Constants.DATASET_SDO
import org.apache.commons.io.FileUtils
import org.apache.commons.io.FileUtils.deleteDirectory
import org.apache.commons.io.filefilter.{DirectoryFileFilter, FileFileFilter}
import org.scalatest.{FlatSpec, Matchers}

import scala.util.{Failure, Success}

class EasyStageDatasetSpec extends FlatSpec with Matchers {

  private val tmpProps = new File("cfg/application.properties")

  "createFileAndFolderSdos" should "stumble if the data folder does not exist" in {

    val bagitDir = new File("target/test/bag")
    val dataDir = new File(bagitDir, "data")
    val sdoSetDir = new File("target/test/someSDO")
    implicit val s = createSettings(bagitDir, sdoSetDir)
    bagitDir.mkdirs()

    createFileAndFolderSdos(dataDir, DATASET_SDO, ANONYMOUS_ACCESS) shouldBe a[Failure[_]]

    deleteDirectory(bagitDir)
  }

  it should "be happy with an empty data folder" in {

    val bagitDir = new File("target/test/bag")
    val dataDir = new File(bagitDir, "data")
    val sdoSetDir = new File("target/test/someSDO")
    implicit val s = createSettings(bagitDir, sdoSetDir)
    dataDir.mkdirs()

    createFileAndFolderSdos(dataDir, DATASET_SDO, ANONYMOUS_ACCESS) shouldBe a[Success[_]]
    sdoSetDir.exists() shouldBe false

    deleteDirectory(bagitDir)
  }

  it should "create file rights computed from dataset access rights" in {

    createProps()
    val bagitDir = new File("src/test/resources/dataset-bags/no-additional-license")
    val dataDir = new File(bagitDir, "data")
    val sdoSetDir = new File("target/test/sdoSet")
    val fileMetadataFile = new File(sdoSetDir, "quicksort_hs/EASY_FILE_METADATA")
    implicit val s = createSettings(bagitDir, sdoSetDir)

    createFileAndFolderSdos(dataDir, DATASET_SDO, OPEN_ACCESS_FOR_REGISTERED_USERS) shouldBe a[Success[_]]
    FileUtils.readFileToString(fileMetadataFile) should include ("<visibleTo>KNOWN</visibleTo>")
    FileUtils.readFileToString(fileMetadataFile) should include ("<accessibleTo>KNOWN</accessibleTo>")
    deleteDirectory(sdoSetDir)

    createFileAndFolderSdos(dataDir, DATASET_SDO, ANONYMOUS_ACCESS) shouldBe a[Success[_]]
    FileUtils.readFileToString(fileMetadataFile) should include ("<visibleTo>ANONYMOUS</visibleTo>")
    FileUtils.readFileToString(fileMetadataFile) should include ("<accessibleTo>ANONYMOUS</accessibleTo>")
    deleteDirectory(sdoSetDir)

    tmpProps.delete()
  }

  "run" should "create SDO sets from test bags (puddings to be eaten by easy-ingest)" in {

    createProps()

    val testBags = new File ("src/test/resources/dataset-bags")
    val puddingsDir = new File ("target/sdoPuddings")
    val emptyDataDir = new File(testBags,"minimal/data")
    emptyDataDir.mkdir()

    // clean up old results
    FileUtils.deleteDirectory(puddingsDir)

    // license-by-url seems to require mocking web-access, probably beyond the purpose of this test
    for (bagName <- Array("additional-license-by-text", "no-additional-license", "minimal")) {
      val sdoSetDir = new File(puddingsDir,bagName)
      implicit val settings = createSettings(new File(testBags, bagName), sdoSetDir)

      EasyStageDataset.run(settings) shouldBe a[Success[_]]
      new File(sdoSetDir, "dataset/EMD").exists() shouldBe true
      new File(sdoSetDir, "dataset/AMD").exists() shouldBe true
      new File(sdoSetDir, "dataset/cfg.json").exists() shouldBe true
      new File(sdoSetDir, "dataset/fo.xml").exists() shouldBe true
      new File(sdoSetDir, "dataset/PRSQL").exists() shouldBe true
    }
    // just a dataset SDO
    new File(puddingsDir,"minimal").listFiles().length shouldBe 1

    // both have 2 files and 2 nested folders resulting in a total of 5 SDO's
    new File(puddingsDir,"no-additional-license").listFiles().length shouldBe 5
    new File(puddingsDir,"additional-license-by-text").listFiles().length shouldBe 5

    // cleanup, leave created SDO sets as puddings to eat with easy-ingest
    emptyDataDir.delete()
    tmpProps.delete()
  }

  def createProps() = FileUtils.write(tmpProps, "owner=dsowner\nredirect-unset-url=http://unset.dans.knaw.nl")

  def createSettings(bagitDir: File, sdoSetDir: File): Settings = {
    // the user and disciplines should exist in deasy
    // to allow ingest and subsequent examination with the web-ui of the generated sdo sets
    Settings(
      ownerId = "digger001",
      bagitDir = bagitDir,
      sdoSetDir = sdoSetDir,
      isMendeley = false,
      URN = Some("someUrn"),
      DOI = Some("doei"),
      disciplines = Map[String, String](
        "D10000" -> "easy-discipline:57",
        "E10000" -> "easy-discipline:219",
        "E18000" -> "easy-discipline:226")
    )
  }
}
{'fraction_non_alphanumeric': 0.10502958579881656, 'fraction_numerical': 0.00631163708086785, 'mean_word_length': 4.152947154471545, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4332984', 'n_tokens_mistral': 3302, 'n_tokens_neox': 3156, 'n_words': 812}