<commit_before>library(MASS)

trunc_weibull = function(x, k, lmd, lower_bound){
  return (dweibull(x, k, lmd) / (1 - pweibull(lower_bound, k, lmd)))
}

trunc_weibull_par_est = function(x, lower_bound){
  trunc_weibull_lb = function(x, k, lmd){
    return (trunc_weibull(x, k, lmd, lower_bound))
  }
  weibull_est = as.numeric((fitdistr(x, 'weibull'))$estimate)
  return (as.numeric(fitdistr(x, trunc_weibull_lb, list(k = weibull_est[1], lmd = weibull_est[2]))$estimate))
}

trunc_expon_par_est = function(x, lower_bound){
  return (1 / (mean(x) - lower_bound))
}

trunc_pareto_par_est = function(x, lower_bound){
  return (length(x) / sum(log(x) - log(lower_bound)))
}

read_file = function(path){
  dat = read.csv(path,
    colClasses = c('character', 'character', 'numeric'))
  return(dat)
}
<commit_msg>Modify path to be consistent with current folder layout<commit_after>library(MASS)

trunc_weibull = function(x, k, lmd, lower_bound){
  return (dweibull(x, k, lmd) / (1 - pweibull(lower_bound, k, lmd)))
}

trunc_weibull_par_est = function(x, lower_bound){
  trunc_weibull_lb = function(x, k, lmd){
    return (trunc_weibull(x, k, lmd, lower_bound))
  }
  weibull_est = as.numeric((fitdistr(x, 'weibull'))$estimate)
  return (as.numeric(fitdistr(x, trunc_weibull_lb, list(k = weibull_est[1], lmd = weibull_est[2]))$estimate))
}

trunc_expon_par_est = function(x, lower_bound){
  return (1 / (mean(x) - lower_bound))
}

trunc_pareto_par_est = function(x, lower_bound){
  return (length(x) / sum(log(x) - log(lower_bound)))
}

read_file = function(path){
  path = paste('./data/', path, sep = '')
  dat = read.csv(path,
    colClasses = c('character', 'character', 'numeric'))
  return(dat)
}

{'fraction_non_alphanumeric': 0.1645047169811321, 'fraction_numerical': 0.0047169811320754715, 'mean_word_length': 5.40377358490566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '711269', 'n_tokens_mistral': 704, 'n_tokens_neox': 695, 'n_words': 130}