<commit_before>package com.github.amchang.ohdsi.lib

import java.nio.file.{Files, Paths}

import com.github.nscala_time.time.Imports._
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.DataFrame



/**
  * Replication of functionality in
  *
  * https://github.com/OHDSI/Era-Constructor/blob/master/v5/PostgreSQL/postgres_v5_dose_era.sql
  */
object DoseEra extends Spark with Era {

  type AncestorConceptId = Int
  type DescendantConceptId = Int
  type DrugExposureId = Int
  type PersonId = Int
  type IngredientConceptId = Int
  type UnitConceptId = String
  type DoseValue = String
  type DrugExposureStartDate = DateTime
  type DaysSupply = String
  type DrugExposureEndDate = DateTime

  // use these for none values
  private val emptyConceptId = (-1, -1)
  private val emptyDrugRecord = (-1, -1, -1, "", "", null, "", null)

  /**
    * Build Dosage Eras
    */
  def build = {
    // the entire data
    val netData = createInitialData
    val b = 1
  }

  /**
    * Create the initial set of data to use for later merging
    */
  private def createInitialData: RDD[
    (DrugExposureId, PersonId, IngredientConceptId, UnitConceptId, DoseValue,
      DrugExposureStartDate, DaysSupply, DrugExposureEndDate)
    ] = {
    val descendantConceptMappings = remainingConceptIds
    val drugExposure = loadDrugExposure
    val remainder = sparkContext.broadcast(descendantConceptMappings.toMap)
<commit_msg>delete<commit_after>package com.github.amchang.ohdsi.lib

import java.nio.file.{Files, Paths}

import com.github.nscala_time.time.Imports._
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.DataFrame



/**
  * Replication of functionality in
  *
  * https://github.com/OHDSI/Era-Constructor/blob/master/v5/PostgreSQL/postgres_v5_dose_era.sql
  */
object DoseEra extends Spark with Era {

  type AncestorConceptId = Int
  type DescendantConceptId = Int
  type DrugExposureId = Int
  type PersonId = Int
  type IngredientConceptId = Int
  type UnitConceptId = String
  type DoseValue = String
  type DrugExposureStartDate = DateTime
  type DaysSupply = String
  type DrugExposureEndDate = DateTime

  // use these for none values
  private val emptyConceptId = (-1, -1)
  private val emptyDrugRecord = (-1, -1, -1, "", "", null, "", null)

  /**
    * Build Dosage Eras
    */
  def build = {
    // the entire data
    val netData = createInitialData
  }

  /**
    * Create the initial set of data to use for later merging
    */
  private def createInitialData: RDD[
    (DrugExposureId, PersonId, IngredientConceptId, UnitConceptId, DoseValue,
      DrugExposureStartDate, DaysSupply, DrugExposureEndDate)
    ] = {
    val descendantConceptMappings = remainingConceptIds
    val drugExposure = loadDrugExposure
    val remainder = sparkContext.broadcast(descendantConceptMappings.toMap)

{'fraction_non_alphanumeric': 0.08828581243558914, 'fraction_numerical': 0.005152868430092752, 'mean_word_length': 3.629570747217806, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6717578', 'n_tokens_mistral': 988, 'n_tokens_neox': 950, 'n_words': 233}