<commit_before>package controllers

import exceptions.{StorageInsertionException, AccessKeyException}
import play.api.mvc._
import java.util.zip.{ZipEntry, ZipOutputStream, GZIPInputStream}
import java.io._
import org.apache.commons.io.IOUtils
import play.api.libs.iteratee.Enumerator
import play.libs.Akka
import akka.actor.Actor
import play.api.Logger
import core._
import scala.{Either, Option}
import storage.FileStorage
import util.SimpleDataSetParser
import akka.util.Duration
import java.util.concurrent.TimeUnit
import models._
import play.api.libs.Files.TemporaryFile
import eu.delving.stats.Stats
import scala.collection.JavaConverters._
import java.util.Date
import models.statistics._
import HubMongoContext.hubFileStore
import xml.Node
import play.api.libs.concurrent.Promise
import org.apache.commons.lang.StringEscapeUtils
import scala.util.matching.Regex.Match
import java.util.regex.Matcher
import plugins.DataSetPlugin
import models.statistics.DataSetStatisticsContext
import models.statistics.FieldFrequencies
import models.statistics.FieldValues
import play.api.libs.MimeTypes

/**
 * This Controller is responsible for all the interaction with the SIP-Creator.
 * Access control is done using OAuth2
 *
 * @author <NAME> <<EMAIL>>
 */

object SipCreatorEndPoint extends BoundController(HubModule) with SipCreatorEndPoint

trait SipCreatorEndPoint extends Controller with OrganizationConfigurationAware with Logging {
  this: BoundController with Controller with OrganizationConfigurationAware with Logging =>

  val organizationServiceLocator = HubModule.inject[DomainServiceLocator[OrganizationService]](name = None)

  val DOT_PLACEHOLDER = "--"

  private def basexStorage(implicit configuration: OrganizationConfiguration) = HubServices.basexStorage(configuration)

  // HASH__type[_prefix].extension
  private val FileName = """([^_]*)__([^._]*)_?([^.]*).(.*)""".r

  private var connectedUserObject: Option[HubUser] = None

  def AuthenticatedAction[A](accessToken: Option[String])(action: Action[A]): Action[A] = OrganizationConfigured {
    Action(action.parser) {
      implicit request => {
        if (accessToken.isEmpty) {
          Unauthorized("No access token provided")
        }
        else if (!HubUser.isValidToken(accessToken.get)) {
          Unauthorized("Access Key %s not accepted".format(accessToken.get))
        }
        else {
          connectedUserObject = HubUser.getUserByToken(accessToken.get)
          action(request)
        }
      }
    }
  }

  def OrganizationAction[A](orgId: String, accessToken: Option[String])
    (action: Action[A]): Action[A] = AuthenticatedAction(accessToken) {
    Action(action.parser) {
      implicit request =>
        if (orgId == null || orgId.isEmpty) {
          BadRequest("No orgId provided")
        }
        else {
          if (!organizationServiceLocator.byDomain.exists(orgId)) {
            NotFound("Unknown organization " + orgId)
          }
          else {
            action(request)
          }
        }
    }
  }

  def getConnectedUser: HubUser = connectedUserObject.getOrElse({
    log.warn("Attemtping to connect with an invalid access token")
    throw new AccessKeyException("No access token provided")
  })

  def connectedUser = getConnectedUser.userName

  def listAll(accessToken: Option[String]) = AuthenticatedAction(accessToken) {
    Action {
      implicit request =>
        val dataSets = DataSet.dao.findAllForUser(
          connectedUserObject.get.userName,
          configuration.orgId,
          DataSetPlugin.ROLE_DATASET_EDITOR
        )

        val dataSetsXml = <data-set-list>{
                    dataSets.map {
                      ds =>
                        val creator = HubUser.dao.findByUsername(ds.getCreator)
                        val lockedBy = ds.getLockedBy
                            <data-set>
                              <spec>{ds.spec}</spec>
                              <name>{ds.details.name}</name>
                              <orgId>{ds.orgId}</orgId>
                              {if (creator.isDefined) {
                              <createdBy>
                                <username>{creator.get.userName}</username>
                                <fullname>{creator.get.fullname}</fullname>
                                <email>{creator.get.email}</email>
                              </createdBy>}
                            else {
                              <createdBy>
                                <username>{ds.getCreator}</username>
                              </createdBy>}}{if (lockedBy != None) {
                              <lockedBy>
                                <username>{lockedBy.get.userName}</username>
                                <fullname>{lockedBy.get.fullname}</fullname>
                                <email>{lockedBy.get.email}</email>
                              </lockedBy>}}
                              <state>{ds.state.name}</state>
                              <recordCount>{ds.details.total_records}</recordCount>
                            </data-set>
                    }
                }</data-set-list>
        Ok(dataSetsXml)
    }
  }

  def unlock(orgId: String, spec: String, accessToken: Option[String]): Action[AnyContent] =
    OrganizationAction(orgId, accessToken) {
      Action {
        implicit request =>
          val dataSet = DataSet.dao.findBySpecAndOrgId(spec, orgId)
          if (dataSet.isEmpty) {
            val msg = "Unknown spec %s".format(spec)
            NotFound(msg)
          }
          else {
            if (dataSet.get.lockedBy == None) {<commit_msg>Passing the list of schema versions to the Sip-Creator<commit_after>package controllers

import exceptions.{StorageInsertionException, AccessKeyException}
import play.api.mvc._
import java.util.zip.{ZipEntry, ZipOutputStream, GZIPInputStream}
import java.io._
import org.apache.commons.io.IOUtils
import play.api.libs.iteratee.Enumerator
import play.libs.Akka
import akka.actor.Actor
import play.api.Logger
import core._
import scala.{Either, Option}
import storage.FileStorage
import util.SimpleDataSetParser
import akka.util.Duration
import java.util.concurrent.TimeUnit
import models._
import play.api.libs.Files.TemporaryFile
import eu.delving.stats.Stats
import scala.collection.JavaConverters._
import java.util.Date
import models.statistics._
import HubMongoContext.hubFileStore
import xml.Node
import play.api.libs.concurrent.Promise
import org.apache.commons.lang.StringEscapeUtils
import scala.util.matching.Regex.Match
import java.util.regex.Matcher
import plugins.DataSetPlugin
import models.statistics.DataSetStatisticsContext
import models.statistics.FieldFrequencies
import models.statistics.FieldValues
import play.api.libs.MimeTypes

/**
 * This Controller is responsible for all the interaction with the SIP-Creator.
 * Access control is done using OAuth2
 *
 * @author <NAME> <<EMAIL>>
 */

object SipCreatorEndPoint extends BoundController(HubModule) with SipCreatorEndPoint

trait SipCreatorEndPoint extends Controller with OrganizationConfigurationAware with Logging {
  this: BoundController with Controller with OrganizationConfigurationAware with Logging =>

  val organizationServiceLocator = HubModule.inject[DomainServiceLocator[OrganizationService]](name = None)

  val DOT_PLACEHOLDER = "--"

  private def basexStorage(implicit configuration: OrganizationConfiguration) = HubServices.basexStorage(configuration)

  // HASH__type[_prefix].extension
  private val FileName = """([^_]*)__([^._]*)_?([^.]*).(.*)""".r

  private var connectedUserObject: Option[HubUser] = None

  def AuthenticatedAction[A](accessToken: Option[String])(action: Action[A]): Action[A] = OrganizationConfigured {
    Action(action.parser) {
      implicit request => {
        if (accessToken.isEmpty) {
          Unauthorized("No access token provided")
        }
        else if (!HubUser.isValidToken(accessToken.get)) {
          Unauthorized("Access Key %s not accepted".format(accessToken.get))
        }
        else {
          connectedUserObject = HubUser.getUserByToken(accessToken.get)
          action(request)
        }
      }
    }
  }

  def OrganizationAction[A](orgId: String, accessToken: Option[String])
    (action: Action[A]): Action[A] = AuthenticatedAction(accessToken) {
    Action(action.parser) {
      implicit request =>
        if (orgId == null || orgId.isEmpty) {
          BadRequest("No orgId provided")
        }
        else {
          if (!organizationServiceLocator.byDomain.exists(orgId)) {
            NotFound("Unknown organization " + orgId)
          }
          else {
            action(request)
          }
        }
    }
  }

  def getConnectedUser: HubUser = connectedUserObject.getOrElse({
    log.warn("Attemtping to connect with an invalid access token")
    throw new AccessKeyException("No access token provided")
  })

  def connectedUser = getConnectedUser.userName

  def listAll(accessToken: Option[String]) = AuthenticatedAction(accessToken) {
    Action {
      implicit request =>
        val dataSets = DataSet.dao.findAllForUser(
          connectedUserObject.get.userName,
          configuration.orgId,
          DataSetPlugin.ROLE_DATASET_EDITOR
        )

        val dataSetsXml = <data-set-list>{
                    dataSets.map {
                      ds =>
                        val creator = HubUser.dao.findByUsername(ds.getCreator)
                        val lockedBy = ds.getLockedBy
                            <data-set>
                              <spec>{ds.spec}</spec>
                              <name>{ds.details.name}</name>
                              <orgId>{ds.orgId}</orgId>
                              {if (creator.isDefined) {
                              <createdBy>
                                <username>{creator.get.userName}</username>
                                <fullname>{creator.get.fullname}</fullname>
                                <email>{creator.get.email}</email>
                              </createdBy>}
                            else {
                              <createdBy>
                                <username>{ds.getCreator}</username>
                              </createdBy>}}{if (lockedBy != None) {
                              <lockedBy>
                                <username>{lockedBy.get.userName}</username>
                                <fullname>{lockedBy.get.fullname}</fullname>
                                <email>{lockedBy.get.email}</email>
                              </lockedBy>}}
                              <state>{ds.state.name}</state>
                              <schemaVersions>{ds.getAllMappingSchemas.map { schema =>
                                <schemaVersion>
                                  <prefix>{schema.getPrefix}</prefix>
                                  <version>{schema.getVersion}</version>
                                </schemaVersion>}}
                              </schemaVersions>
                              <recordCount>{ds.details.total_records}</recordCount>
                            </data-set>
                    }
                }</data-set-list>
        Ok(dataSetsXml)
    }
  }

  def unlock(orgId: String, spec: String, accessToken: Option[String]): Action[AnyContent] =
    OrganizationAction(orgId, accessToken) {
      Action {
        implicit request =>
          val dataSet = DataSet.dao.findBySpecAndOrgId(spec, orgId)
          if (dataSet.isEmpty) {
            val msg = "Unknown spec %s".format(spec)
            NotFound(msg)
          }
          else {
            if (dataSet.get.lockedBy == None) {
{'fraction_non_alphanumeric': 0.09916153319644079, 'fraction_numerical': 0.00017111567419575633, 'mean_word_length': 2.068784457862956, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 98, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3398690', 'n_tokens_mistral': 3113, 'n_tokens_neox': 3042, 'n_words': 593}