<commit_before>#' Import data stage for Syberia model process.
#'
#' @param modelenv an environment. The current modeling environment.
#' @param import_options a list. The available import options. Will differ
#'    depending on the adapter. (default is file adapter)
#' @export
import_stage <- function(modelenv, import_options) {
  # By default, try loading from only one adapter
  if (!all(vapply(import_options, is.list, logical(1)))) {
    import_options$adapter <- import_options$adapter %||% 'file'
    import_options <-
      structure(list(import_options), .Names = import_options$adapter)
  }

  build_import_stagerunner(modelenv, import_options)$run()
  NULL
}

#' Build a stagerunner for importing data with backup sources.
#'
#' @param modelenv an environment. The current modeling environment.
#' @param import_options a list. Nested list, one adapter per list entry.
build_import_stagerunner <- function(modelenv, import_options) {
  stages <- lapply(seq_along(import_options), function(index) {
    stage <- function(modelenv) {
      # Only run if data isn't already loaded
      if (!'data' %in% ls(modelenv)) {
        attempt <- suppressWarnings(suppressMessages(
          tryCatch(fn(modelenv, opts), error = function(e) FALSE)))
      }
    }
    adapter <- names(import_options)[index]
    environment(stage)$fn <- import_adapter(adapter)
    environment(stage)$opts <- import_options[[index]]
    stage
  })

  stages[[length(stages) + 1]] <- function(modelenv) {
    if (!'data' %in% ls(modelenv))
      stop("Failed to load data from all data sources")
    modelenv$import_stage$variable_summaries <-
      statsUtils::variable_summaries(modelenv$data) 
  }
  stageRunner$new(modelenv, stages)
}

#' Fetch an import adapter.
#'
#' @param adapter character. Only supported so far are 's3' and 'file'.
#'    The default is 'file'.
#' @param opts list. The options that get passed to the import adapter.
import_adapter <- function(adapter = 'file', opts) {
  stopifnot(is.character(adapter))
  adapter <- tolower(adapter)
  if (adapter == 's3') {
    function(modelenv, opts) {
      require(s3mpi)
      if (is.character(opts)) opts <- list(file = opts)
      stopifnot(is.character(opts$file))
      modelenv$data <- s3read(opts$file)
    }
  } else {
    function(modelenv, opts) {
      if (is.character(opts)) opts <- list(file = opts)
      stopifnot(is.character(opts$file))
      modelenv$data <- read.csv(opts$file %||% opts$filename %||% opts$name %||% opts$path,
                                stringsAsFactors = FALSE)
    }
  }
}
<commit_msg>allow "file", "filename", "name", and "path" to all reference the same thing<commit_after>#' Import data stage for Syberia model process.
#'
#' @param modelenv an environment. The current modeling environment.
#' @param import_options a list. The available import options. Will differ
#'    depending on the adapter. (default is file adapter)
#' @export
import_stage <- function(modelenv, import_options) {
  # By default, try loading from only one adapter
  if (!all(vapply(import_options, is.list, logical(1)))) {
    import_options$adapter <- import_options$adapter %||% 'file'
    import_options <-
      structure(list(import_options), .Names = import_options$adapter)
  }

  build_import_stagerunner(modelenv, import_options)$run()
  NULL
}

#' Build a stagerunner for importing data with backup sources.
#'
#' @param modelenv an environment. The current modeling environment.
#' @param import_options a list. Nested list, one adapter per list entry.
build_import_stagerunner <- function(modelenv, import_options) {
  stages <- lapply(seq_along(import_options), function(index) {
    stage <- function(modelenv) {
      # Only run if data isn't already loaded
      if (!'data' %in% ls(modelenv)) {
        attempt <- suppressWarnings(suppressMessages(
          tryCatch(fn(modelenv, opts), error = function(e) FALSE)))
      }
    }
    adapter <- names(import_options)[index]
    environment(stage)$fn <- import_adapter(adapter)
    environment(stage)$opts <- import_options[[index]]
    stage
  })

  stages[[length(stages) + 1]] <- function(modelenv) {
    if (!'data' %in% ls(modelenv))
      stop("Failed to load data from all data sources")
    modelenv$import_stage$variable_summaries <-
      statsUtils::variable_summaries(modelenv$data) 
  }
  stageRunner$new(modelenv, stages)
}

#' Fetch an import adapter.
#'
#' @param adapter character. Only supported so far are 's3' and 'file'.
#'    The default is 'file'.
#' @param opts list. The options that get passed to the import adapter.
import_adapter <- function(adapter = 'file', opts) {
  stopifnot(is.character(adapter))
  adapter <- tolower(adapter)
  if (adapter == 's3') {
    function(modelenv, opts) {
      require(s3mpi)
      if (is.character(opts)) opts <- list(file = opts)
      filename <- opts$file %||% opts$filename %||% opts$name %||% opts$path
      stopifnot(is.character(filename))
      modelenv$data <- s3read(filename)
    }
  } else {
    function(modelenv, opts) {
      if (is.character(opts)) opts <- list(file = opts)
      filename <- opts$file %||% opts$filename %||% opts$name %||% opts$path
      stopifnot(is.character(filename))
      modelenv$data <- read.csv(filename, stringsAsFactors = FALSE)
    }
  }
}

{'fraction_non_alphanumeric': 0.13038548752834467, 'fraction_numerical': 0.0022675736961451248, 'mean_word_length': 3.8205828779599273, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '715475', 'n_tokens_mistral': 1670, 'n_tokens_neox': 1590, 'n_words': 434}