<commit_before>#' Read Single Pixel Daymet data
#' 
#' Reads Single Pixel Daymet data into a nested list, preserving
#' header data and critical file name information.
#' 
#' @param file a Daymet Single Pixel data file
#' @param site a sitename (default = \code{NULL})
#' @return A nested data structure including site meta-data, the full
#' header and the data as a `data.frame()`.
#' @keywords time series, Daymet
#' @export
#' @examples
#'
#' \dontrun{
#' # with defaults, outputting a data frame
#' # with smoothed values, overwriting the original
#'
#' # download the data
#' download_daymet(site = "Daymet",
#'                  start = 1980,
#'                  end = 1980,
#'                  internal = FALSE,
#'                  silent = TRUE)
#'
#' # read in the Daymet file
#' df = read_daymet(paste0(tempdir(),"/Daymet_1980_1980.csv"))
#'
#' # print data structure
#' print(str(df))
#' }

read_daymet <- function(file = NULL,
                        site = NULL){
  
  # stop on missing files
  if (is.null(file) ){
    stop('File not provided...')
  }
  
  # check if the file exists and is a daily file
  if (!file.exists(file) ){
    stop('File does not exist...')
  }
  
  # read and format header, read past the header (should the length)
  # change in the future with a few lines this then does not break
  # the script
  header = try(readLines(file, n = 30), silent = TRUE)
  
  # get the location of the true table header
  # based upon the two fields which will always be there
  # year and yday
  table_cols = grep("year,yday", header)
  
  # warning / stop if key data table header elements are not found
  if (length(table_cols) == 0){
    stop("Key table header elements are missing, Daymet format change?")
  }
  
  # if no header is present skip extraction of meta-data, fill with NULL
  if (table_cols > 1){
      
    # header is defined as everything before the
    header = tolower(header[1:(table_cols-1)])
    
    # read ancillary data from downloaded file header
    # this includes, tile nr and altitude etc. use gregexpr()
    # and regmatches to extract relevant data<commit_msg>comments<commit_after>#' Read Single Pixel Daymet data
#' 
#' Reads Single Pixel Daymet data into a nested list, preserving
#' header data and critical file name information.
#' 
#' @param file a Daymet Single Pixel data file
#' @param site a sitename (default = \code{NULL})
#' @return A nested data structure including site meta-data, the full
#' header and the data as a `data.frame()`.
#' @keywords time series, Daymet
#' @export
#' @examples
#'
#' \dontrun{
#' # with defaults, outputting a data frame
#' # with smoothed values, overwriting the original
#'
#' # download the data
#' download_daymet(site = "Daymet",
#'                  start = 1980,
#'                  end = 1980,
#'                  internal = FALSE,
#'                  silent = TRUE)
#'
#' # read in the Daymet file
#' df = read_daymet(paste0(tempdir(),"/Daymet_1980_1980.csv"))
#'
#' # print data structure
#' print(str(df))
#' }

read_daymet <- function(file = NULL,
                        site = NULL){
  
  # stop on missing files
  if (is.null(file) ){
    stop('File not provided...')
  }
  
  # check if the file exists and is a daily file
  if (!file.exists(file) ){
    stop('File does not exist...')
  }
  
  # read and format header, read past the header (should the length)
  # change in the future with a few lines this then does not break
  # the script
  header = try(readLines(file, n = 30), silent = TRUE)
  
  # get the location of the true table header
  # based upon the two fields which will always be there
  # year and yday
  table_cols = grep("year,yday", header)
  
  # warning / stop if key data table header elements are not found
  if (length(table_cols) == 0){
    stop("Key table header elements are missing, Daymet format change?")
  }
  
  # if no header is present (table_cols == 1) skip extraction of meta-data
  if (table_cols > 1){
      
    # header is defined as everything before the
    header = tolower(header[1:(table_cols-1)])
    
    # read ancillary data from downloaded file header
    # this includes, tile nr and altitude etc. use gregexpr()
    # and regmatches to extract relevant data
{'fraction_non_alphanumeric': 0.10895769321673364, 'fraction_numerical': 0.011108484991727724, 'mean_word_length': 2.922150139017609, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6764591', 'n_tokens_mistral': 1351, 'n_tokens_neox': 1211, 'n_words': 502}