<commit_before>package it.polimi.genomics.repository.FSRepository

import java.io._
import java.text.SimpleDateFormat
import java.util.Date

import it.polimi.genomics.core.DataStructures.IRDataSet
import it.polimi.genomics.core.{GMQLSchemaField, GMQLSchemaFormat}
import it.polimi.genomics.repository
import it.polimi.genomics.repository.FSRepository.datasets.GMQLDataSetXML
import it.polimi.genomics.repository.GMQLExceptions.{GMQLDSException, GMQLDSNotFound, GMQLSampleNotFound, GMQLUserNotFound}
import it.polimi.genomics.repository.{GMQLRepository, GMQLSample, GMQLStatistics, Utilities => General_Utilities}
import org.apache.hadoop.fs.{FileSystem, Path}
import org.slf4j.LoggerFactory

import scala.collection.JavaConverters._
/**
  * Created by abdulrahman on 12/04/16.
  */
class DFSRepository extends GMQLRepository with XMLDataSetRepository{
  private final val logger = LoggerFactory.getLogger(this.getClass)
    repository.Utilities()

  /**
    * Add a new dataset to GMQL repository, this dataset is usually a result of a script execution.
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param userName The user name in which this dataSet belongs to.
    * @param Samples List of GMQL samples [[ GMQLSample]].
    * @param GMQLScriptPath The path to the script text file that generated this data set.
    * @param schemaType One of GMQL schema types as shown in [[ GMQLSchemaFormat]]
    */
  override def createDs(dataSet:IRDataSet, userName: String, Samples: java.util.List[GMQLSample], GMQLScriptPath: String,schemaType:GMQLSchemaFormat.Value): Unit = {
    //Create Temp folder to place the meta files temporarly in Local file system
    val tmpFolderName = General_Utilities().getTempDir(userName)+dataSet.position+"_/"
    val tmpFolder = new File(tmpFolderName)
    tmpFolder.mkdirs()

    //copy all the meta data from HDFS to Local file system
    val samples = Samples.asScala.map{x=>
      val metaFile= tmpFolderName + x.name+".meta"
      val sourceHDFSMetaFile = General_Utilities().getHDFSRegionDir(userName) + x.name+".meta"
      FS_Utilities.copyfiletoLocal(sourceHDFSMetaFile, metaFile)
      new GMQLSample(name = x.name,meta = metaFile )
    }.toList.asJava

    //create DS as a set of XML files in the local repository
    super.createDs(dataSet, userName, samples, GMQLScriptPath, schemaType)

    //clean the temp Directory
    tmpFolder.delete()
  }

  /**
    *     * Import Dataset into GMQL from Local file system.
    *
    * @param dataSetName String of the dataset name.
    * @param Samples List of GMQL samples [[ GMQLSample]].
    * @param userName String of the user name to add this dataset to.
    * @param schemaPath String of the path to the xml file of the dataset's schema.
    */
  override def importDs(dataSetName: String, userName: String, Samples: java.util.List[GMQLSample], schemaPath: String): Unit = {
    if (FS_Utilities.validate(schemaPath)) {
      val date = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date());
      val samples = Samples.asScala.map(x=> GMQLSample(x.ID,dataSetName+"_"+date+ "/"+new File(x.name).getName,dataSetName +"_"+date+ "/"+new File(x.meta).getName) ).asJava
      // Import the dataset schema and Script files to the local folder
      super.importDs(dataSetName: String, userName: String, samples ,schemaPath)

      // Copy sample and Meta data from HDFS to the local folder
      samples.asScala.map{x =>
        val HDFSfile = General_Utilities().getHDFSDSRegionDir(userName,dataSetName+"_"+date) +  new File(x.name).getName
        FS_Utilities.copyfiletoHDFS(x.name, HDFSfile)
      }

      FS_Utilities.copyfiletoHDFS(General_Utilities().getSchemaDir(userName)+dataSetName+".schema",
        General_Utilities().getHDFSRegionDir(userName)+ new Path(samples.get(0).name).getParent.toString+ "/test.schema"
      )
    } else {
      logger.info("The dataset schema does not confirm the schema style (XSD)")
    }
  }

  /**
    *  Add sample to Data Set,
    *  TODO: i did not finish it since The web interface does not use it for the moment
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param Sample GMQL sample [[ GMQLSample]].
    * @param userName String of the user name.
    */
  override def addSampleToDS(dataSet: String, userName: String, Sample: GMQLSample): Unit = ???

  /**
    * Delete Data Set from the repository, Delete XML files from local File system and delete the samples and meta files from HDFS.<commit_msg>solving bug produced from [Issue #67].<commit_after>package it.polimi.genomics.repository.FSRepository

import java.io._
import java.text.SimpleDateFormat
import java.util.Date

import it.polimi.genomics.core.DataStructures.IRDataSet
import it.polimi.genomics.core.{GMQLSchemaField, GMQLSchemaFormat}
import it.polimi.genomics.repository
import it.polimi.genomics.repository.FSRepository.datasets.GMQLDataSetXML
import it.polimi.genomics.repository.GMQLExceptions.{GMQLDSException, GMQLDSNotFound, GMQLSampleNotFound, GMQLUserNotFound}
import it.polimi.genomics.repository.{GMQLRepository, GMQLSample, GMQLStatistics, Utilities => General_Utilities}
import org.apache.hadoop.fs.{FileSystem, Path}
import org.slf4j.LoggerFactory

import scala.collection.JavaConverters._
/**
  * Created by abdulrahman on 12/04/16.
  */
class DFSRepository extends GMQLRepository with XMLDataSetRepository{
  private final val logger = LoggerFactory.getLogger(this.getClass)
    repository.Utilities()

  /**
    * Add a new dataset to GMQL repository, this dataset is usually a result of a script execution.
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param userName The user name in which this dataSet belongs to.
    * @param Samples List of GMQL samples [[ GMQLSample]].
    * @param GMQLScriptPath The path to the script text file that generated this data set.
    * @param schemaType One of GMQL schema types as shown in [[ GMQLSchemaFormat]]
    */
  override def createDs(dataSet:IRDataSet, userName: String, Samples: java.util.List[GMQLSample], GMQLScriptPath: String,schemaType:GMQLSchemaFormat.Value): Unit = {
    //Create Temp folder to place the meta files temporarly in Local file system
    val tmpFolderName = General_Utilities().getTempDir(userName)+dataSet.position+"_/"
    val tmpFolder = new File(tmpFolderName)
    tmpFolder.mkdirs()

    //copy all the meta data from HDFS to Local file system
    val samples = Samples.asScala.map{x=>
      val metaFile= tmpFolderName + x.name+".meta"
      val sourceHDFSMetaFile = General_Utilities().getHDFSRegionDir(userName) + x.name+".meta"
      FS_Utilities.copyfiletoLocal(sourceHDFSMetaFile, metaFile)
      new GMQLSample(name = x.name,meta = metaFile )
    }.toList.asJava

    //create DS as a set of XML files in the local repository
    super.createDs(dataSet, userName, samples, GMQLScriptPath, schemaType)

    //clean the temp Directory
    tmpFolder.delete()
  }

  /**
    *     * Import Dataset into GMQL from Local file system.
    *
    * @param dataSetName String of the dataset name.
    * @param Samples List of GMQL samples [[ GMQLSample]].
    * @param userName String of the user name to add this dataset to.
    * @param schemaPath String of the path to the xml file of the dataset's schema.
    */
  override def importDs(dataSetName: String, userName: String, Samples: java.util.List[GMQLSample], schemaPath: String): Unit = {
    if (FS_Utilities.validate(schemaPath)) {
      val date = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date());
      val samples = Samples.asScala.map(x=> GMQLSample(ID = x.ID, name = dataSetName+"_"+date+ "/"+new File(x.name).getName,meta = x.meta) ).asJava
      // Import the dataset schema and Script files to the local folder
      super.importDs(dataSetName: String, userName: String, samples ,schemaPath)

      // Copy sample and Meta data from HDFS to the local folder
      Samples.asScala.map{x =>
        val HDFSfile = General_Utilities().getHDFSDSRegionDir(userName,dataSetName+"_"+date) +  new File(x.name).getName
        FS_Utilities.copyfiletoHDFS(x.name, HDFSfile)
      }

      FS_Utilities.copyfiletoHDFS(General_Utilities().getSchemaDir(userName)+dataSetName+".schema",
        General_Utilities().getHDFSRegionDir(userName)+ new Path(samples.get(0).name).getParent.toString+ "/test.schema"
      )
    } else {
      logger.info("The dataset schema does not confirm the schema style (XSD)")
    }
  }

  /**
    *  Add sample to Data Set,
    *  TODO: i did not finish it since The web interface does not use it for the moment
    *
    * @param dataSet Intermediate Representation (IRDataSet) of the dataset, contains the dataset name and schema.
    * @param Sample GMQL sample [[ GMQLSample]].
    * @param userName String of the user name.
    */
  override def addSampleToDS(dataSet: String, userName: String, Sample: GMQLSample): Unit = ???

  /**
    * Delete Data Set from the repository, Delete XML files from local File system and delete the samples and meta files from HDFS.
{'fraction_non_alphanumeric': 0.09459165942658558, 'fraction_numerical': 0.001954821894005213, 'mean_word_length': 3.959073774905762, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1699682', 'n_tokens_mistral': 2766, 'n_tokens_neox': 2706, 'n_words': 835}