<commit_before>package it.polimi.genomics.core.DataStructures

import it.polimi.genomics.core.DataStructures.CoverParameters.CoverFlag.CoverFlag
import it.polimi.genomics.core.DataStructures.CoverParameters.CoverParam
import it.polimi.genomics.core.DataStructures.GroupMDParameters.Direction.Direction
import it.polimi.genomics.core.DataStructures.GroupMDParameters.TopParameter
import it.polimi.genomics.core.DataStructures.JoinParametersRD.JoinQuadruple
import it.polimi.genomics.core.DataStructures.JoinParametersRD.RegionBuilder.RegionBuilder
import it.polimi.genomics.core.DataStructures.MetaAggregate.{MetaAggregateFunction, MetaExtension}
import it.polimi.genomics.core.DataStructures.MetaGroupByCondition.MetaGroupByCondition
import it.polimi.genomics.core.DataStructures.MetaJoinCondition.MetaJoinCondition
import it.polimi.genomics.core.DataStructures.MetadataCondition.MetadataCondition
import it.polimi.genomics.core.DataStructures.RegionAggregate.{RegionFunction, RegionsToMeta, RegionsToRegion}
import it.polimi.genomics.core.DataStructures.RegionCondition.RegionCondition
import it.polimi.genomics.core.GMQLLoader
import it.polimi.genomics.core.ParsingType.PARSING_TYPE


/**
 * Dag node to represent the repository/storage reader for metadata files.
 * @param paths path from which to read the files
 * @param loader class that contains both the region and the metadata parser
 * @tparam IR Type of the input of the region parser
 * @tparam OR Type of the output of the region parser
 * @tparam IM Type of the input of the metadata parser
 * @tparam OM Type of the output of the metadata parser
 */
case class IRReadMD[IR,OR,IM,OM](var paths : List[String],
                                 loader : GMQLLoader[IR,OR,IM,OM],
                                 var dataset : IRDataSet) extends MetaOperator

/**
  * Dag node to represent the memory dataset reader for meta data.
  */
case class IRReadMEMMD(metaDS:Any) extends MetaOperator


/**
 * Dag node to represent the repository/storage reader for region data files.
 * @param paths path from which to read the files
 * @param loader class that contains both the region and the metadata parser
 * @tparam IR Type of the input of the region parser
 * @tparam OR Type of the output of the region parser
 * @tparam IM Type of the input of the metadata parser
 * @tparam OM Type of the output of the metadata parser
 */
case class IRReadRD[IR,OR,IM,OM](var paths : List[String],
                                 loader : GMQLLoader[IR,OR,IM,OM],
                                 var dataset : IRDataSet) extends RegionOperator

/**
  * Dag node to represent the memory dataset reader for region data.
  */
case class IRReadMEMRD(regionDS:Any) extends RegionOperator


/** Dag node to represent the MATERIALIZE operation of a variable's metadata into the repository/storage.
  *
  * @param path location where to store the metadata
  * @param father metadata dag of the variable to be stored
  */<commit_msg>added IRReadFileRD and IRGenerateMD<commit_after>package it.polimi.genomics.core.DataStructures

import it.polimi.genomics.core.DataStructures.CoverParameters.CoverFlag.CoverFlag
import it.polimi.genomics.core.DataStructures.CoverParameters.CoverParam
import it.polimi.genomics.core.DataStructures.GroupMDParameters.Direction.Direction
import it.polimi.genomics.core.DataStructures.GroupMDParameters.TopParameter
import it.polimi.genomics.core.DataStructures.JoinParametersRD.JoinQuadruple
import it.polimi.genomics.core.DataStructures.JoinParametersRD.RegionBuilder.RegionBuilder
import it.polimi.genomics.core.DataStructures.MetaAggregate.{MetaAggregateFunction, MetaExtension}
import it.polimi.genomics.core.DataStructures.MetaGroupByCondition.MetaGroupByCondition
import it.polimi.genomics.core.DataStructures.MetaJoinCondition.MetaJoinCondition
import it.polimi.genomics.core.DataStructures.MetadataCondition.MetadataCondition
import it.polimi.genomics.core.DataStructures.RegionAggregate.{RegionFunction, RegionsToMeta, RegionsToRegion}
import it.polimi.genomics.core.DataStructures.RegionCondition.RegionCondition
import it.polimi.genomics.core.GMQLLoader
import it.polimi.genomics.core.ParsingType.PARSING_TYPE


/**
 * Dag node to represent the repository/storage reader for metadata files.
 * @param paths path from which to read the files
 * @param loader class that contains both the region and the metadata parser
 * @tparam IR Type of the input of the region parser
 * @tparam OR Type of the output of the region parser
 * @tparam IM Type of the input of the metadata parser
 * @tparam OM Type of the output of the metadata parser
 */
case class IRReadMD[IR,OR,IM,OM](var paths : List[String],
                                 loader : GMQLLoader[IR,OR,IM,OM],
                                 var dataset : IRDataSet) extends MetaOperator

/**
  * Dag node to represent the memory dataset reader for meta data.
  */
case class IRReadMEMMD(metaDS:Any) extends MetaOperator


/**
 * Dag node to represent the repository/storage reader for region data files.
 * @param paths path from which to read the files
 * @param loader class that contains both the region and the metadata parser
 * @tparam IR Type of the input of the region parser
 * @tparam OR Type of the output of the region parser
 * @tparam IM Type of the input of the metadata parser
 * @tparam OM Type of the output of the metadata parser
 */
case class IRReadRD[IR,OR,IM,OM](var paths : List[String],
                                 loader : GMQLLoader[IR,OR,IM,OM],
                                 var dataset : IRDataSet) extends RegionOperator

/**
  * Dag node to represent the memory dataset reader for region data.
  */
case class IRReadMEMRD(regionDS:Any) extends RegionOperator


/**
  * Dag node to load single files with custom loader
  * @param path path of the file
  * @param loader class that contains both the region and the metadata parser
  * @param dataset dataset class which is a wrapper for the dataset
  * @tparam IR Type of the input of the region parser
  * @tparam OR Type of the output of the region parser
  * @tparam IM Type of the input of the metadata parser
  * @tparam OM Type of the output of the metadata parser
  */
case class IRReadFileRD[IR,OR,IM,OM](var path: String,
                                   loader: GMQLLoader[IR,OR,IM,OM],
                                   var dataset : IRDataSet) extends RegionOperator


/**
  * Dag node to generate empty metadata values for each key in the region data
  * @param regionFile region Dag node
  */
case class IRGenerateMD(regionFile: RegionOperator) extends MetaOperator


/** Dag node to represent the MATERIALIZE operation of a variable's metadata into the repository/storage.
  *
  * @param path location where to store the metadata
  * @param father metadata dag of the variable to be stored
  */
{'fraction_non_alphanumeric': 0.06812865497076023, 'fraction_numerical': 0.0, 'mean_word_length': 3.92158273381295, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6650924', 'n_tokens_mistral': 1881, 'n_tokens_neox': 1827, 'n_words': 646}