<commit_before>################
#
# Code to import a file that updates a particular person's catch data.
#
# <NAME>
# <EMAIL>
# January 9, 2017
# Using: http://google-styleguide.googlecode.com/svn/trunk/google-r-style.html
#
################

rm(list=ls())   		#clean up the workspace
header <- "Import Post Season File Tool v0.3"
options(stringsAsFactors = FALSE)


source.lib.dir <- "./lib/"
if (exists("lib.dir")) {
  source.lib.dir <- lib.dir
}

if (exists("report.dir") == FALSE) {
  report.dir <- "./report/"
}

if (exists("data.dir") == FALSE) {
  data.dir <- "./data/"
}

source(file.path(source.lib.dir, "Util.r"))
source(file.path(source.lib.dir, "FramDb.r"))
source(file.path(source.lib.dir, "PscFramAdminData.r"))

#' Parses the import file format and returns the results into a list
#' The file format generally consists of a header followed by a CSV table of fishery catch.
#' Each section of the file is seperated by 4 or more dashes (e.g. ----------) and a carrage return.
#'
#' @param import.file.name The file name of the import file
#'
#' @return A list with the different sections of the import file
#' 
ParseImportFile <- function(import.file.name) {
  
  import.data <- list()
  file.text <- readChar(import.file.name, file.info(import.file.name)$size)
  
  sections <- strsplit(file.text, "[\r]?\n[-]{4,}[\r]?\n")[[1]]
  
  header <- sections[1]
  
  header.conn <- rawConnection(raw(0), "r+")
  writeBin(header, header.conn)
  seek(header.conn, 0)
  import.data$header <- read_delim(header.conn, ":", col_names=FALSE)
  close(header.conn)
  names(import.data$header) <- c("variable.name", "variable.value")
  
  catch <- sections[2]
  while (substr(catch, 1, 1) %in% c("\n", "\r")) {
    #strip blank lines from before the catch data, so that the first line is the table header
    catch <- substring(catch, 2)
  }
  #Add a new line character so that the file parses correctly
  catch <- paste0(catch, "\r\n", collapse="")

  import.data$fishery.scalars <- ReadMemoryCsv(catch)
  
  #remove blank lines
  import.data$fishery.scalars <- filter(import.data$fishery.scalars,
                                          !is.na(fram.fishery.id) & 
                                          !is.na(fram.time.step) &
                                          !is.na (fishery.flag))
  
  na.msf <- is.na(import.data$fishery.scalars$msf.catch)
  import.data$fishery.scalars$msf.catch[na.msf] <- 0
  
  if (length(sections) > 2) {
    import.data$target.escapement <- ReadMemoryCsv(sections[3])
    
    #remove blank lines
    import.data$target.escapement <- filter(import.data$target.escapement,
                                            !is.na(fram.stock.id) & 
                                            !is.na(escapement.flag))
    
  }
  
  return (import.data)
}

#' Validates the escapement flags against the escapement values for parametrization 
#' of the backward FRAM model run.
#'
#' @param target_escapement A data frame of the targent escapements from an import file
#'
#' @return A boolean, TRUE for valid or FALSE for when there is issues with the escapement flags
#'
ValidEscapementFlags <- function(target_escapement) {
  valid.esc <- TRUE
  
  esc.required <- filter(target_escapement, 
                         escapement.flag == FramTargetEscExactFlag & 
                           target.escapement == 0)
  
  if (nrow(esc.required) > 0) {
    valid.esc <- FALSE
    
    cat(sprintf("ERROR - The following stocks have target escapement but the flag identfies it as not specified.  To suggested fix is to change the escapementflag should possibly be %d.\n",
                FramTargetNotUsedFlag))
    
    esc.txt <- paste(inval.nonselect.fishery$fram.stock.name, 
                     " (", 
                     inval.nonselect.fishery$fram.stock.id, 
                     ")", 
                     collapse=", ", sep="")
    cat(esc.txt)
    cat("\n\n")
  }  
  
  esc.notused <- filter(target_escapement,
                             escapement.flag == FramTargetNotUsedFlag & target.escapement > 0)
  
  if (nrow(esc.notused) > 0) {
    valid.esc <- FALSE
    
    cat(sprintf("ERROR - The following stocks have target escapement but the flag identfies it as not specified.  To suggested fix is to change the escapementflag should possibly be %d.\n",
                FramTargetEscExactFlag))
    
    esc.txt <- paste(esc.notused$fram.stock.name, 
                     " (", 
                     esc.notused$fram.stock.id, 
                     ")", 
                     collapse=", ", sep="")
    cat(esc.txt)
    cat("\n\n")
  }  
  
  return(valid.esc)
  
}

ValidPostSeasonCatch <- function(fishery.scalars) {
  # Validates the catch data for parametrization of the FRAM model.
  # Most of the validation is related to the appropriate setting of flags and 
  # providing all the necessary parameters.
  #
  # Args:
  #   catch.data: The catch data loaded from a post season import file.
  #
  # Returns:
  #   A boolean, TRUE for valid or FALSE for when there is issues with the catch
  #
  # Exceptions:
  #   None
  #   
  valid.catch <- TRUE
  
  nonselective.flags <- as.integer(fishery.scalars$fishery.flag  / 10)
  nonselective.flags[fishery.scalars$fishery.flag < 10] <- 
    fishery.scalars$fishery.flag[fishery.scalars$fishery.flag < 10]
  
  inval.nonselect <- filter(fishery.scalars, nonselective.catch > 0, 
                            nonselective.flags != kFramNonSelectiveQuotaFlag)
  if (nrow(inval.nonselect) > 0) {
    valid.catch <- FALSE
    inval.nonselect.fishery <- unique(select(inval.nonselect, fram.fishery.name, fram.fishery.id))
    
    cat(sprintf("ERROR - The following non-selective fisheries have invalid flag, it should be %d or %d.\n",
                kFramNonSelectiveQuotaFlag,
                kFramNonSelectiveQuotaFlag * 10 + kFramMsfQuotaFlag))
    
    fishery.txt <- paste(inval.nonselect.fishery$fram.fishery.name, 
                         " (", 
                         inval.nonselect.fishery$fram.fishery.id, 
                         ")", 
                         collapse=", ", sep="")
    cat(fishery.txt)
    cat("\n\n")
  }
  
  msf.flags <- as.integer(fishery.scalars$fishery.flag %% 10)
  inval.msf <- filter(fishery.scalars, msf.catch > 0, 
                      msf.flags != kFramMsfQuotaFlag)
  
  if (nrow(inval.msf) > 0) {
    valid.catch <- FALSE
    inval.msf.fishery <- unique(select(inval.msf, fishery.name, fishery.id))
    cat(sprintf("ERROR - The following MSF fisheries have invalid flag, it should be %d or %d.\n",
                kFramMsfQuotaFlag,
                kFramNonSelectiveQuotaFlag * 10 + kFramMsfQuotaFlag))
    fishery.txt <- paste(inval.msf.fishery$fishery.name, 
                         " (", 
                         inval.msf.fishery$fishery.id, 
                         ")", 
                         collapse=", ", sep="")
    cat(fishery.txt)
    cat("\n\n")    
  }

  return (valid.catch)
}

#' Validates the mark rate information for mark-selective fisheries.
#'
#' @param fishery.scalars The catch data loaded from a post season import file.
#'
#' @return A boolean, TRUE for valid or FALSE for when there is issues with the mark rate information
#'   with the mark-selective fisheries
#'
ValidMarkInfo <- function(fishery.scalars) {

  valid.mark.info <- TRUE

  msf.flags <- as.integer(fishery.scalars$fishery.flag %% 10)
  inval.mark.info <- filter(fishery.scalars,
                            msf.flags %in% c(kFramMsfScalarFlag, kFramMsfQuotaFlag),
                            !(mark.release.rate > 0),
                            !(mark.missid.rate > 0),
                            !(unmark.missid.rate > 0),
                            !(mark.incidental.rate > 0))
  
  if (nrow(inval.mark.info) > 0) {<commit_msg>fix variable names<commit_after>################
#
# Code to import a file that updates a particular person's catch data.
#
# <NAME>
# <EMAIL>
# January 9, 2017
# Using: http://google-styleguide.googlecode.com/svn/trunk/google-r-style.html
#
################

rm(list=ls())   		#clean up the workspace
header <- "Import Post Season File Tool v0.3"
options(stringsAsFactors = FALSE)


source.lib.dir <- "./lib/"
if (exists("lib.dir")) {
  source.lib.dir <- lib.dir
}

if (exists("report.dir") == FALSE) {
  report.dir <- "./report/"
}

if (exists("data.dir") == FALSE) {
  data.dir <- "./data/"
}

source(file.path(source.lib.dir, "Util.r"))
source(file.path(source.lib.dir, "FramDb.r"))
source(file.path(source.lib.dir, "PscFramAdminData.r"))

#' Parses the import file format and returns the results into a list
#' The file format generally consists of a header followed by a CSV table of fishery catch.
#' Each section of the file is seperated by 4 or more dashes (e.g. ----------) and a carrage return.
#'
#' @param import.file.name The file name of the import file
#'
#' @return A list with the different sections of the import file
#' 
ParseImportFile <- function(import.file.name) {
  
  import.data <- list()
  file.text <- readChar(import.file.name, file.info(import.file.name)$size)
  
  sections <- strsplit(file.text, "[\r]?\n[-]{4,}[\r]?\n")[[1]]
  
  header <- sections[1]
  
  header.conn <- rawConnection(raw(0), "r+")
  writeBin(header, header.conn)
  seek(header.conn, 0)
  import.data$header <- read_delim(header.conn, ":", col_names=FALSE)
  close(header.conn)
  names(import.data$header) <- c("variable.name", "variable.value")
  
  catch <- sections[2]
  while (substr(catch, 1, 1) %in% c("\n", "\r")) {
    #strip blank lines from before the catch data, so that the first line is the table header
    catch <- substring(catch, 2)
  }
  #Add a new line character so that the file parses correctly
  catch <- paste0(catch, "\r\n", collapse="")

  import.data$fishery.scalars <- ReadMemoryCsv(catch)
  
  #remove blank lines
  import.data$fishery.scalars <- filter(import.data$fishery.scalars,
                                          !is.na(fram.fishery.id) & 
                                          !is.na(fram.time.step) &
                                          !is.na (fishery.flag))
  
  na.msf <- is.na(import.data$fishery.scalars$msf.catch)
  import.data$fishery.scalars$msf.catch[na.msf] <- 0
  
  if (length(sections) > 2) {
    import.data$target.escapement <- ReadMemoryCsv(sections[3])
    
    #remove blank lines
    import.data$target.escapement <- filter(import.data$target.escapement,
                                            !is.na(fram.stock.id) & 
                                            !is.na(escapement.flag))
    
  }
  
  if ("comment" %notin% touper(names(import.data$fishery.scalars))) {
    import.data$fishery.scalars$comment <- ""
  }
  
  return (import.data)
}

#' Validates the escapement flags against the escapement values for parametrization 
#' of the backward FRAM model run.
#'
#' @param target_escapement A data frame of the targent escapements from an import file
#'
#' @return A boolean, TRUE for valid or FALSE for when there is issues with the escapement flags
#'
ValidEscapementFlags <- function(target_escapement) {
  valid.esc <- TRUE
  
  esc.required <- filter(target_escapement, 
                         escapement.flag == FramTargetEscExactFlag & 
                           target.escapement == 0)
  
  if (nrow(esc.required) > 0) {
    valid.esc <- FALSE
    
    cat(sprintf("ERROR - The following stocks have target escapement but the flag identfies it as not specified.  To suggested fix is to change the escapementflag should possibly be %d.\n",
                FramTargetNotUsedFlag))
    
    esc.txt <- paste(esc.required$fram.stock.name, 
                     " (", 
                     esc.required$fram.stock.id, 
                     ")", 
                     collapse=", ", sep="")
    cat(esc.txt)
    cat("\n\n")
  }  
  
  esc.notused <- filter(target_escapement,
                             escapement.flag == FramTargetNotUsedFlag & target.escapement > 0)
  
  if (nrow(esc.notused) > 0) {
    valid.esc <- FALSE
    
    cat(sprintf("ERROR - The following stocks have target escapement but the flag identfies it as not specified.  To suggested fix is to change the escapementflag should possibly be %d.\n",
                FramTargetEscExactFlag))
    
    esc.txt <- paste(esc.notused$fram.stock.name, 
                     " (", 
                     esc.notused$fram.stock.id, 
                     ")", 
                     collapse=", ", sep="")
    cat(esc.txt)
    cat("\n\n")
  }  
  
  return(valid.esc)
  
}

ValidPostSeasonCatch <- function(fishery.scalars) {
  # Validates the catch data for parametrization of the FRAM model.
  # Most of the validation is related to the appropriate setting of flags and 
  # providing all the necessary parameters.
  #
  # Args:
  #   catch.data: The catch data loaded from a post season import file.
  #
  # Returns:
  #   A boolean, TRUE for valid or FALSE for when there is issues with the catch
  #
  # Exceptions:
  #   None
  #   
  valid.catch <- TRUE
  
  nonselective.flags <- as.integer(fishery.scalars$fishery.flag  / 10)
  nonselective.flags[fishery.scalars$fishery.flag < 10] <- 
    fishery.scalars$fishery.flag[fishery.scalars$fishery.flag < 10]
  
  inval.nonselect <- filter(fishery.scalars, nonselective.catch > 0, 
                            nonselective.flags != kFramNonSelectiveQuotaFlag)
  if (nrow(inval.nonselect) > 0) {
    valid.catch <- FALSE
    inval.nonselect.fishery <- unique(select(inval.nonselect, fram.fishery.name, fram.fishery.id))
    
    cat(sprintf("ERROR - The following non-selective fisheries have invalid flag, it should be %d or %d.\n",
                kFramNonSelectiveQuotaFlag,
                kFramNonSelectiveQuotaFlag * 10 + kFramMsfQuotaFlag))
    
    fishery.txt <- paste(inval.nonselect.fishery$fram.fishery.name, 
                         " (", 
                         inval.nonselect.fishery$fram.fishery.id, 
                         ")", 
                         collapse=", ", sep="")
    cat(fishery.txt)
    cat("\n\n")
  }
  
  msf.flags <- as.integer(fishery.scalars$fishery.flag %% 10)
  inval.msf <- filter(fishery.scalars, msf.catch > 0, 
                      msf.flags != kFramMsfQuotaFlag)
  
  if (nrow(inval.msf) > 0) {
    valid.catch <- FALSE
    inval.msf.fishery <- unique(select(inval.msf, fram.fishery.name, fram.fishery.id))
    cat(sprintf("ERROR - The following MSF fisheries have invalid flag, it should be %d or %d.\n",
                kFramMsfQuotaFlag,
                kFramNonSelectiveQuotaFlag * 10 + kFramMsfQuotaFlag))
    fishery.txt <- paste(inval.msf.fishery$fram.fishery.name, 
                         " (", 
                         inval.msf.fishery$fram.fishery.id, 
                         ")", 
                         collapse=", ", sep="")
    cat(fishery.txt)
    cat("\n\n")    
  }

  return (valid.catch)
}

#' Validates the mark rate information for mark-selective fisheries.
#'
#' @param fishery.scalars The catch data loaded from a post season import file.
#'
#' @return A boolean, TRUE for valid or FALSE for when there is issues with the mark rate information
#'   with the mark-selective fisheries
#'
ValidMarkInfo <- function(fishery.scalars) {

  valid.mark.info <- TRUE

  msf.flags <- as.integer(fishery.scalars$fishery.flag %% 10)
  inval.mark.info <- filter(fishery.scalars,
                            msf.flags %in% c(kFramMsfScalarFlag, kFramMsfQuotaFlag),
                            !(mark.release.rate > 0),
                            !(mark.missid.rate > 0),
                            !(unmark.missid.rate > 0),
                            !(mark.incidental.rate > 0))
  
  if (nrow(inval.mark.info) > 0) {
{'fraction_non_alphanumeric': 0.12962726576461578, 'fraction_numerical': 0.006127138115905029, 'mean_word_length': 2.214813295034879, 'pattern_counts': {'":': 2, '<': 106, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1848924', 'n_tokens_mistral': 5326, 'n_tokens_neox': 4959, 'n_words': 1173}