<commit_before>require 'open3'
require 'shellwords'

class Filesystem

  # TODO Move this to a config file?
  # /nfs/07 => false
  # /nfs/gpfs/UNAME/template => true
  # /nfs/08/bmcmichael/ood_templ/5/ => true
  #
  # FIXME: I believe that /nfs/home/efranz will be the pattern for the new file
  # system home directory paths, so we should allow for these as well
  #
  # FIXME: regex is probably overkill here since we now are checking the size of
  # the directory
  # COPY_SAFE_PATH_REGEX = %r{^/nfs/([0-9]{2}|gpfs)/\w+/.+}
  # COPY_SAFE_PATH_ERROR_MESSAGE = "The specified path must be under a home directory or in project space."
  MAX_COPY_SAFE_DIR_SIZE = 1024*1024*1024
  MAX_COPY_SAFE_DU_TIMEOUT_SECONDS = 10
  MAX_COPY_TIMEOUT_MESSAGE = "Timeout occurred when trying to determine directory size. " \
    "Size must be computable in less than #{MAX_COPY_SAFE_DU_TIMEOUT_SECONDS} seconds. " \
    "Either directory has too many files or the file system is currently slow (if so, please try again later)."

  # Returns an http URI path to the cloudcmd filesystem link
  def fs(path)
    OodAppkit.files.url(path: path).to_s
  end

  # Returns an http URI path to the cloudcmd api link
  def api(path)
    OodAppkit.files.api(path: path).to_s
  end

  # Verify that this path is safe to copy recursively from
  #
  # Matches a pathname on the system to prevent root file system copiesa
  # FIXME: this should be a validation on template when creating a new template
  # unfortunately the template's source path and @source for the template Source
  # directory are two very different things and so naming is confusing...
  def validate_path_is_copy_safe(path)
    # FIXME: consider using http://ruby-doc.org/stdlib-2.2.0/libdoc/timeout/rdoc/Timeout.html
    stdout, stderr, status = Open3.capture3 "timeout #{MAX_COPY_SAFE_DU_TIMEOUT_SECONDS}s du -cbs #{Shellwords.escape(path)}"
    return false, MAX_COPY_TIMEOUT_MESSAGE if status.exitstatus == 124
    return false, "Error with status #{status} occurred when trying to determine directory size: #{stderr}" unless status.success?

    safe, error = true, nil
    size = stdout.split.first

    if size.blank?
      safe, error = false, "Failed to properly parse the output of the du command."
    elsif size.to_i > MAX_COPY_SAFE_DIR_SIZE
      safe, error = false, "The directory is too large to copy. The directory should be less than #{MAX_COPY_SAFE_DIR_SIZE} bytes."
    # FIXME: regex is probably overkill here
    # elsif (path =~ COPY_SAFE_PATH_REGEX).nil?
    #   safe, error = false, COPY_SAFE_PATH_ERROR_MESSAGE
    end

    return safe, error
  end

  # Get the disk usage of a path in bytes, nil if path is invalid
  def path_size (path)
    if Dir.exist? path
      Integer(`du -s -b #{path}`.split('/')[0])
    end
  end
end<commit_msg>remove commented out regex related code<commit_after>require 'open3'
require 'shellwords'

class Filesystem

  MAX_COPY_SAFE_DIR_SIZE = 1024*1024*1024
  MAX_COPY_SAFE_DU_TIMEOUT_SECONDS = 10
  MAX_COPY_TIMEOUT_MESSAGE = "Timeout occurred when trying to determine directory size. " \
    "Size must be computable in less than #{MAX_COPY_SAFE_DU_TIMEOUT_SECONDS} seconds. " \
    "Either directory has too many files or the file system is currently slow (if so, please try again later)."

  # Returns an http URI path to the cloudcmd filesystem link
  def fs(path)
    OodAppkit.files.url(path: path).to_s
  end

  # Returns an http URI path to the cloudcmd api link
  def api(path)
    OodAppkit.files.api(path: path).to_s
  end

  # Verify that this path is safe to copy recursively from
  #
  # Matches a pathname on the system to prevent root file system copiesa
  # FIXME: this should be a validation on template when creating a new template
  # unfortunately the template's source path and @source for the template Source
  # directory are two very different things and so naming is confusing...
  def validate_path_is_copy_safe(path)
    # FIXME: consider using http://ruby-doc.org/stdlib-2.2.0/libdoc/timeout/rdoc/Timeout.html
    stdout, stderr, status = Open3.capture3 "timeout #{MAX_COPY_SAFE_DU_TIMEOUT_SECONDS}s du -cbs #{Shellwords.escape(path)}"
    return false, MAX_COPY_TIMEOUT_MESSAGE if status.exitstatus == 124
    return false, "Error with status #{status} occurred when trying to determine directory size: #{stderr}" unless status.success?

    safe, error = true, nil
    size = stdout.split.first

    if size.blank?
      safe, error = false, "Failed to properly parse the output of the du command."
    elsif size.to_i > MAX_COPY_SAFE_DIR_SIZE
      safe, error = false, "The directory is too large to copy. The directory should be less than #{MAX_COPY_SAFE_DIR_SIZE} bytes."
    end

    return safe, error
  end

  # Get the disk usage of a path in bytes, nil if path is invalid
  def path_size (path)
    if Dir.exist? path
      Integer(`du -s -b #{path}`.split('/')[0])
    end
  end
end
{'fraction_non_alphanumeric': 0.07982937233394272, 'fraction_numerical': 0.011375177737152143, 'mean_word_length': 3.6496694995278567, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3793695', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1522, 'n_words': 577}