<commit_before>#' Data stage for syberia models
#'
#' TODO: Document this more
#' 
#' @param modelenv an environment. The persistent modeling environment.
#' @param munge_procedure a list. A list of mungepiece arguments,
#'    first preprocessed then passed to munge.
#' @export
data_stage <- function(modelenv, munge_procedure) {
  # preprocess_munge_procedure(munge_procedure)
  removed_steps <- c()
  for (i in seq_along(munge_procedure)) {
    # Check if we are storing the munging performed so far back into the file
    if (FALSE && is_save_trigger(munge_procedure[[i]])) {
      # TODO: If this was already done, remove previous munge operations
      # and this one so we "fast forward". Not sure yet how this should be
      # detected.
      munge_procedure[[i]] <- list(list(eval(bquote(function(dataframe) {
        write.csv(dataframe, .(modelenv$import_stage$file))
      })), NULL)) # the NULL is to ensure this won't fire on prediction
      removed_steps <- c(removed_steps, i)
    }

    if (is.trigger(munge_procedure[[i]])) {
      removed_steps <- c(removed_steps, i) 
    }

    # TODO: sameAs/importFrom and butWith/except triggers

  }

  munge_procedure <<- munge_procedure
  modelenv$data <- munge(modelenv$data, munge_procedure)
  # Now okay to run munging procedure
  # modelenv$data <- munge(modelenv$data, munge_procedure)
  if (length(removed_steps) > 0)
    attr(modelenv$data, 'mungepieces')[removed_steps] <- NULL
    
  NULL
}

is_save_trigger <- function(mungestep) {
  (is.character(mungestep) && tolower(mungestep) == 'save') ||
  (is.function(mungestep) && body(mungestep) == body(save))
}
<commit_msg>initial commit for integrating stagerunner into data_stage<commit_after>#' Data stage for syberia models
#'
#' TODO: Document this more
#' 
#' @param modelenv an environment. The persistent modeling environment.
#' @param munge_procedure a list. A list of mungepiece arguments,
#'    first preprocessed then passed to munge.
#' @export
data_stage <- function(modelenv, munge_procedure) {
  # preprocess_munge_procedure(munge_procedure)
  removed_steps <- vapply(munge_procedure, is.trigger, logical(1))
  # TODO: sameAs/importFrom and butWith/except triggers
  # TODO: save trigger

  modelenv$data <- munge(modelenv$data, munge_procedure)
  # Now okay to run munging procedure
  # modelenv$data <- munge(modelenv$data, munge_procedure)
  if (length(removed_steps) > 0)
    attr(modelenv$data, 'mungepieces')[removed_steps] <- NULL
    
  NULL
}

{'fraction_non_alphanumeric': 0.11137917169280258, 'fraction_numerical': 0.0012062726176115801, 'mean_word_length': 4.316239316239316, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '452813', 'n_tokens_mistral': 848, 'n_tokens_neox': 785, 'n_words': 218}