<commit_before>package it.polimi.genomics.repository.FSRepository

import java.io._
import java.nio.file.{Files, Paths}
import java.text.SimpleDateFormat
import java.util.Date

import it.polimi.genomics.core.DataStructures.IRDataSet
import it.polimi.genomics.core.GDMSUserClass._
import it.polimi.genomics.core.ParsingType.PARSING_TYPE
import it.polimi.genomics.core.exception.UserExceedsQuota
import it.polimi.genomics.core.{GDMSUserClass, _}
import it.polimi.genomics.repository.FSRepository.FS_Utilities.checkDsName
import it.polimi.genomics.repository.FSRepository.datasets.GMQLDataSetXML
import it.polimi.genomics.repository.GMQLExceptions._
import it.polimi.genomics.repository.{DatasetOrigin, GMQLRepository, GMQLSample, RepositoryType, Utilities => General_Utilities}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.JavaConverters._
import scala.xml.{Elem, Node, NodeSeq, XML}


/**
  * Created by abdulrahman on 16/01/2017.
  */
trait XMLDataSetRepository extends GMQLRepository{
  private final val logger: Logger = LoggerFactory.getLogger(this.getClass)
  General_Utilities()
  /**
    *
    *
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param Samples
    * @param GMQLScriptPath
    * @throws GMQLDSNotFound
    * @throws GMQLUserNotFound
    * @throws GMQLSampleNotFound
    */
  def createDs(dataSet:IRDataSet, userName: String = General_Utilities().USERNAME, Samples: java.util.List[GMQLSample], GMQLScriptPath: String,schemaType:GMQLSchemaFormat.Value, schemaCoordinateSystem: GMQLSchemaCoordinateSystem.Value, dsmeta: Map[String, String]): Unit = {
    // Check the dataset name, return if the dataset is already used in
    // the repository of the this user or the public repository.
    if (DSExists(dataSet.position, userName)) {
      logger.warn(s"The dataset (${dataSet.position})  is already registered")
      throw new GMQLNotValidDatasetNameException(s"The dataset name (${dataSet.position}) is already registered")
    }

    // Store dataset info file
    setDatasetMeta(dataSet.position,userName,dsmeta)

    val samples: List[GMQLSample] = Samples.asScala.map{ x=>if (x.meta.equals("nothing.meta")) new GMQLSample(x.name, x.name+".meta",x.ID) else x}.toList
    //create DS descriptive file of the Data set
    val gMQLDataSetXML = new GMQLDataSetXML(dataSet,userName,samples,GMQLScriptPath, schemaType, schemaCoordinateSystem, "GENERATED_"+General_Utilities().MODE )
    gMQLDataSetXML.Create()
  }


  /**
    *
    * Import Dataset into GMQL from Local file system.
    *
    * @param dataSetName String of the dataset name.
    * @param userName    String of the user name.
    * @param userClass   GDMSUserClass
    * @param Samples     List of GMQL samples [[ GMQLSample]].
    * @param schemaPath  String of the path to the xml file of the dataset schema.
    * @throws GMQLNotValidDatasetNameException
    * @throws GMQLUserNotFound
    * @throws UserExceedsQuota
    * @throws java.lang.Exception
    */
  @throws(classOf[GMQLNotValidDatasetNameException])
  override def importDs(dataSetName: String, userName: String, userClass: GDMSUserClass = GDMSUserClass.PUBLIC, Samples: java.util.List[GMQLSample], schemaPath: String): Unit = {
    checkDsName(dataSetName)

    if( isUserQuotaExceeded(userName, userClass) ) {
      throw new UserExceedsQuota()
    }

    // Files.copy((new File(schemaPath)),(new File(GMQLRepository.FS_Utilities.RepoDir + userName + "/schema/" + dataSetName + ".schema")))

    // Check the dataset name, return if the dataset is already used in
    // the repository of the this user or the public repository.
    if (DSExists(dataSetName, userName)) {
      logger.warn("The dataset name is already registered")
      throw new GMQLNotValidDatasetNameException(s"The dataset name ($dataSetName) is already registered")
    }

    val xmlFile = XML.load(schemaPath)
    val schemaFields = (xmlFile \\ "field")
    val schemaType = (xmlFile \\ "gmqlSchema").head.attribute("type").get.head.text
    val coordinateSystemAttribute = (xmlFile \\ "gmqlSchema").head.attribute("coordinate_system")
    val schemaCoordinateSystem = if (coordinateSystemAttribute.isDefined) coordinateSystemAttribute.get.head.text else "none"
    val schema = schemaFields.map { x => (x.text.trim, ParsingType.attType(x.attribute("type").get.head.text)) }.toList.asJava
    val dataSet = new IRDataSet(dataSetName, schema)
    val gMQLDataSetXML = new GMQLDataSetXML(dataSet, userName, Samples.asScala.toList, GMQLSchemaFormat.getType(schemaType), GMQLSchemaCoordinateSystem.getType(schemaCoordinateSystem), "IMPORTED_"+General_Utilities().MODE  )
    gMQLDataSetXML.Create()


    // Set upload date in dataset meta
    val date =  new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").format(new Date());
    setDatasetMeta(dataSetName, userName, Map("Upload date" -> date))
  }

  /**
    *
    *   Add sample to dataset in the repository
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param Sample
    * @throws GMQLDSNotFound
    * @throws GMQLDSException
    * @throws GMQLUserNotFound
    * @throws GMQLSampleNotFound
    */
  @throws(classOf[GMQLDSException])
  @deprecated
  override def addSampleToDS(dataSet: String, userName: String = General_Utilities().USERNAME, Sample: GMQLSample, userClass: GDMSUserClass = GDMSUserClass.PUBLIC) ={

    val exceeded  = General_Utilities().getRepository().isUserQuotaExceeded(userName, userClass)

    if( exceeded ) {
      throw new UserExceedsQuota()
    }


    val ds = new GMQLDataSetXML(dataSet,userName).loadDS()
    ds.addSample(Sample)
  }




  /**
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param sample
    * @return
    */
  @throws(classOf[GMQLSampleNotFound])
  override def getSampleMeta(dataSet: String, userName: String = General_Utilities().USERNAME, sample: GMQLSample): String = {
    val ds = new GMQLDataSetXML(dataSet,userName).loadDS()
    ds.getMeta(sample)
  }

  /**
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @return
    */
  override def getMeta(dataSet: String,userName:String = General_Utilities().USERNAME): String = {
    new GMQLDataSetXML(dataSet,userName).getMeta()
  }

  /**
    *
    * @param userName
    * @throws GMQLDSException
    * @throws GMQLUserNotFound
    */
  override def listAllDSs(userName: String = General_Utilities().USERNAME): java.util.List[IRDataSet] = {
    val dSs = new File(General_Utilities().getDataSetsDir(userName)).listFiles(new FilenameFilter() {
      def accept(dir: File, name: String): Boolean = {
        return name.endsWith(".xml")
      }
    })
    dSs.map(x=>new GMQLDataSetXML(new IRDataSet(x.getName().subSequence(0, x.getName().length() - 4).toString(),List[(String,PARSING_TYPE)]().asJava),userName).loadDS().dataSet).toList.asJava
  }

  /**
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @throws GMQLDSException
    * @return
    */
  override def DSExists(dataSet: String, userName: String = General_Utilities().USERNAME): Boolean = {
    new GMQLDataSetXML(dataSet,userName).exists()  || new GMQLDataSetXML(dataSet,"public").exists()
  }

  /**
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @throws GMQLDSException
    * @return
    */
  override def DSExistsInPublic(dataSet: String): Boolean = {
    new GMQLDataSetXML(dataSet,"public").exists()
  }

  /**<commit_msg>corrected DSExists function if the "public" user is not registered<commit_after>package it.polimi.genomics.repository.FSRepository

import java.io._
import java.nio.file.{Files, Paths}
import java.text.SimpleDateFormat
import java.util.Date

import it.polimi.genomics.core.DataStructures.IRDataSet
import it.polimi.genomics.core.GDMSUserClass._
import it.polimi.genomics.core.ParsingType.PARSING_TYPE
import it.polimi.genomics.core.exception.UserExceedsQuota
import it.polimi.genomics.core.{GDMSUserClass, _}
import it.polimi.genomics.repository.FSRepository.FS_Utilities.checkDsName
import it.polimi.genomics.repository.FSRepository.datasets.GMQLDataSetXML
import it.polimi.genomics.repository.GMQLExceptions._
import it.polimi.genomics.repository.{DatasetOrigin, GMQLRepository, GMQLSample, RepositoryType, Utilities => General_Utilities}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.JavaConverters._
import scala.util.Try
import scala.xml.{Elem, Node, NodeSeq, XML}


/**
  * Created by abdulrahman on 16/01/2017.
  */
trait XMLDataSetRepository extends GMQLRepository{
  private final val logger: Logger = LoggerFactory.getLogger(this.getClass)
  General_Utilities()
  /**
    *
    *
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param Samples
    * @param GMQLScriptPath
    * @throws GMQLDSNotFound
    * @throws GMQLUserNotFound
    * @throws GMQLSampleNotFound
    */
  def createDs(dataSet:IRDataSet, userName: String = General_Utilities().USERNAME, Samples: java.util.List[GMQLSample], GMQLScriptPath: String,schemaType:GMQLSchemaFormat.Value, schemaCoordinateSystem: GMQLSchemaCoordinateSystem.Value, dsmeta: Map[String, String]): Unit = {
    // Check the dataset name, return if the dataset is already used in
    // the repository of the this user or the public repository.
    if (DSExists(dataSet.position, userName)) {
      logger.warn(s"The dataset (${dataSet.position})  is already registered")
      throw new GMQLNotValidDatasetNameException(s"The dataset name (${dataSet.position}) is already registered")
    }

    // Store dataset info file
    setDatasetMeta(dataSet.position,userName,dsmeta)

    val samples: List[GMQLSample] = Samples.asScala.map{ x=>if (x.meta.equals("nothing.meta")) new GMQLSample(x.name, x.name+".meta",x.ID) else x}.toList
    //create DS descriptive file of the Data set
    val gMQLDataSetXML = new GMQLDataSetXML(dataSet,userName,samples,GMQLScriptPath, schemaType, schemaCoordinateSystem, "GENERATED_"+General_Utilities().MODE )
    gMQLDataSetXML.Create()
  }


  /**
    *
    * Import Dataset into GMQL from Local file system.
    *
    * @param dataSetName String of the dataset name.
    * @param userName    String of the user name.
    * @param userClass   GDMSUserClass
    * @param Samples     List of GMQL samples [[ GMQLSample]].
    * @param schemaPath  String of the path to the xml file of the dataset schema.
    * @throws GMQLNotValidDatasetNameException
    * @throws GMQLUserNotFound
    * @throws UserExceedsQuota
    * @throws java.lang.Exception
    */
  @throws(classOf[GMQLNotValidDatasetNameException])
  override def importDs(dataSetName: String, userName: String, userClass: GDMSUserClass = GDMSUserClass.PUBLIC, Samples: java.util.List[GMQLSample], schemaPath: String): Unit = {
    checkDsName(dataSetName)

    if( isUserQuotaExceeded(userName, userClass) ) {
      throw new UserExceedsQuota()
    }

    // Files.copy((new File(schemaPath)),(new File(GMQLRepository.FS_Utilities.RepoDir + userName + "/schema/" + dataSetName + ".schema")))

    // Check the dataset name, return if the dataset is already used in
    // the repository of the this user or the public repository.
    if (DSExists(dataSetName, userName)) {
      logger.warn("The dataset name is already registered")
      throw new GMQLNotValidDatasetNameException(s"The dataset name ($dataSetName) is already registered")
    }

    val xmlFile = XML.load(schemaPath)
    val schemaFields = (xmlFile \\ "field")
    val schemaType = (xmlFile \\ "gmqlSchema").head.attribute("type").get.head.text
    val coordinateSystemAttribute = (xmlFile \\ "gmqlSchema").head.attribute("coordinate_system")
    val schemaCoordinateSystem = if (coordinateSystemAttribute.isDefined) coordinateSystemAttribute.get.head.text else "none"
    val schema = schemaFields.map { x => (x.text.trim, ParsingType.attType(x.attribute("type").get.head.text)) }.toList.asJava
    val dataSet = new IRDataSet(dataSetName, schema)
    val gMQLDataSetXML = new GMQLDataSetXML(dataSet, userName, Samples.asScala.toList, GMQLSchemaFormat.getType(schemaType), GMQLSchemaCoordinateSystem.getType(schemaCoordinateSystem), "IMPORTED_"+General_Utilities().MODE  )
    gMQLDataSetXML.Create()


    // Set upload date in dataset meta
    val date =  new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").format(new Date());
    setDatasetMeta(dataSetName, userName, Map("Upload date" -> date))
  }

  /**
    *
    *   Add sample to dataset in the repository
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param Sample
    * @throws GMQLDSNotFound
    * @throws GMQLDSException
    * @throws GMQLUserNotFound
    * @throws GMQLSampleNotFound
    */
  @throws(classOf[GMQLDSException])
  @deprecated
  override def addSampleToDS(dataSet: String, userName: String = General_Utilities().USERNAME, Sample: GMQLSample, userClass: GDMSUserClass = GDMSUserClass.PUBLIC) ={

    val exceeded  = General_Utilities().getRepository().isUserQuotaExceeded(userName, userClass)

    if( exceeded ) {
      throw new UserExceedsQuota()
    }


    val ds = new GMQLDataSetXML(dataSet,userName).loadDS()
    ds.addSample(Sample)
  }




  /**
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param sample
    * @return
    */
  @throws(classOf[GMQLSampleNotFound])
  override def getSampleMeta(dataSet: String, userName: String = General_Utilities().USERNAME, sample: GMQLSample): String = {
    val ds = new GMQLDataSetXML(dataSet,userName).loadDS()
    ds.getMeta(sample)
  }

  /**
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @return
    */
  override def getMeta(dataSet: String,userName:String = General_Utilities().USERNAME): String = {
    new GMQLDataSetXML(dataSet,userName).getMeta()
  }

  /**
    *
    * @param userName
    * @throws GMQLDSException
    * @throws GMQLUserNotFound
    */
  override def listAllDSs(userName: String = General_Utilities().USERNAME): java.util.List[IRDataSet] = {
    val dSs = new File(General_Utilities().getDataSetsDir(userName)).listFiles(new FilenameFilter() {
      def accept(dir: File, name: String): Boolean = {
        return name.endsWith(".xml")
      }
    })
    dSs.map(x=>new GMQLDataSetXML(new IRDataSet(x.getName().subSequence(0, x.getName().length() - 4).toString(),List[(String,PARSING_TYPE)]().asJava),userName).loadDS().dataSet).toList.asJava
  }

  /**
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @throws GMQLDSException
    * @return
    */
  override def DSExists(dataSet: String, userName: String = General_Utilities().USERNAME): Boolean = {
    new GMQLDataSetXML(dataSet,userName).exists()  ||
    //getOrElse there is no public user
      Try( new GMQLDataSetXML(dataSet, "public").exists()).getOrElse(false)
  }

  /**
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @throws GMQLDSException
    * @return
    */
  override def DSExistsInPublic(dataSet: String): Boolean = {
    new GMQLDataSetXML(dataSet,"public").exists()
  }

  /**
{'fraction_non_alphanumeric': 0.10850217761787541, 'fraction_numerical': 0.0013886258915609417, 'mean_word_length': 4.179470415168356, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '836042', 'n_tokens_mistral': 4851, 'n_tokens_neox': 4820, 'n_words': 1135}