<commit_before>#' A Text Loading Function
#'
#' This function chews a load of pdfs in a fixed directory
#' @keywords impact ref text
#' @export
#' @param mySettings, list of alternative boilerplate terms etc to overwrite defaults
#' @return list containing metric results, and document corpora along with supervised topic models
fromTheTop <- function (mySettings=list()) {
  library (RCurl)
  
  defaultSettings <- list (
    dirName = "C:/Martin/Data/Impact/REF submission documents/Case Studies",
    boilerPlateTerms = c(
      "Page",
      "Impact case study (REF3b)",
      "Title of case study:",
      "(indicative maximum 100 words)",
      "(indicative maximum 500 words)",
      "(indicative maximum of six references)",
      "(indicative maximum 750 words)",
      "(indicative maximum of 10 references)"
    ),
    sectionHeadings = c(
      "Summary of the impact",
      "Underpinning research",
      "References to the research",
      "Details of the impact",
      "Sources to corroborate the impact|Sources to corroborate impact|References to corroborate the impact" 
    ),
    impactSettings = list(uoa=19, profile="Impact"),
    helperFiles = list(buzzFile1=system.file("extdata", "buzz.txt", package="refimpact"), 
                       buzzFile2=system.file("extdata", "busibuzz.txt", package="refimpact")
                       ),
    topicCount = 10,
    topicIter = 10,
    keepSpaces = FALSE,
    places = TRUE,
    orgs = TRUE
  )
  
  mergedSettings <- merge.list (mySettings, defaultSettings)
  
  corpus <- loadDocs (mergedSettings$dirName)
  corpus <- primeCaseStudyDocs (corpus, mergedSettings)
  pwordCorpus <- makeProperWordCorpus (corpus, mergedSettings)
  results <- doMetrics (corpus, pwordCorpus, mergedSettings)
  
  write.csv (results, "results.csv", row.names=FALSE, na="")
  
  results
}


#' A Multiple PDF Document Loading Function
#'
#' This function chews a load of pdfs in a given directory
#' @export
#' @param dirName directory PDFs are located in
#' @param isRecursive flag whether sub-directories should be searched
#' @return TM document corpus
#' @examples
#' \dontrun{
#' loadDocs (dirName="C:/Martin/Data/Impact/REF submission documents/Templates", isRecursive=FALSE)
#' }
loadDocs <- function (dirName="C:/Martin/Data/Impact/REF submission documents/Case Studies", isRecursive=TRUE) {
  library(tm)
  library(stringr)
  
  dirPath = file.path(dirName)
  docs = Corpus (DirSource (dirPath, recursive=isRecursive), readerControl = list(reader=readPDF))
  
  docs
}


#' Corpus quality control and metadata attachment<commit_msg>link to raw buzzword lists<commit_after>#' A Text Loading Function
#'
#' This function chews a load of pdfs in a fixed directory
#' @keywords impact ref text
#' @export
#' @param mySettings, list of alternative boilerplate terms etc to overwrite defaults
#' @return list containing metric results, and document corpora along with supervised topic models
fromTheTop <- function (mySettings=list()) {
  library (RCurl)
  
  defaultSettings <- list (
    dirName = "C:/Martin/Data/Impact/REF submission documents/Case Studies",
    boilerPlateTerms = c(
      "Page",
      "Impact case study (REF3b)",
      "Title of case study:",
      "(indicative maximum 100 words)",
      "(indicative maximum 500 words)",
      "(indicative maximum of six references)",
      "(indicative maximum 750 words)",
      "(indicative maximum of 10 references)"
    ),
    sectionHeadings = c(
      "Summary of the impact",
      "Underpinning research",
      "References to the research",
      "Details of the impact",
      "Sources to corroborate the impact|Sources to corroborate impact|References to corroborate the impact" 
    ),
    impactSettings = list(uoa=19, profile="Impact"),
    helperFiles = list(buzzFile1=system.file("extdata", "buzz.txt", package="refimpact"), 
                       buzzFile2=system.file("extdata", "busibuzz.txt", package="refimpact")
                       ),
    topicCount = 10,
    topicIter = 10,
    keepSpaces = FALSE,
    places = TRUE,
    orgs = TRUE
  )
  
  mergedSettings <- merge.list (mySettings, defaultSettings)
  
  corpus <- loadDocs (mergedSettings$dirName)
  corpus <- primeCaseStudyDocs (corpus, mergedSettings)
  pwordCorpus <- makeProperWordCorpus (corpus, mergedSettings)
  results <- doMetrics (corpus, pwordCorpus, mergedSettings)
  
  write.csv (results$metricTable, "results.csv", row.names=FALSE, na="")
  
  results
}


#' A Multiple PDF Document Loading Function
#'
#' This function chews a load of pdfs in a given directory
#' @export
#' @param dirName directory PDFs are located in
#' @param isRecursive flag whether sub-directories should be searched
#' @return TM document corpus
#' @examples
#' \dontrun{
#' loadDocs (dirName="C:/Martin/Data/Impact/REF submission documents/Templates", isRecursive=FALSE)
#' }
loadDocs <- function (dirName="C:/Martin/Data/Impact/REF submission documents/Case Studies", isRecursive=TRUE) {
  library(tm)
  library(stringr)
  
  dirPath = file.path(dirName)
  docs = Corpus (DirSource (dirPath, recursive=isRecursive), readerControl = list(reader=readPDF))
  
  docs
}


#' Corpus quality control and metadata attachment
{'fraction_non_alphanumeric': 0.10901749663526245, 'fraction_numerical': 0.007690828686791002, 'mean_word_length': 3.759377859103385, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5675680', 'n_tokens_mistral': 1643, 'n_tokens_neox': 1484, 'n_words': 469}