<commit_before># rails runner script/upload_tax_documents_to_aws.rb 'folder_path' -e production

folder_path = ARGV[0]

if File.directory?(folder_path)
  raise "No directory exists with the given name: #{folder_path}"
elsif Dir.entries(folder_path).size < 1
  raise "Unable to find files/folders in the given path: #{folder_path}"
end

field_names  = %w(uploaded_file_name)

def upload_to_s3(file_name, csv)
  base_name = File.basename(file_name)
  Aws::S3Storage.save(file_name, "tax-documents", base_name)
  @counter += 1
  csv << [
    base_name
  ]
end

def process_folder(folder, csv)
  Dir.entries(folder).each do |file|
    next unless File.file?(file)
    upload_to_s3(file, csv)
  end
end

file_name = "#{Rails.root}/upload_tax_documents_to_aws_#{Time.now.strftime('%m_%d_%Y')}.csv"

CSV.open(file_name, "w", force_quotes: true) do |csv|
  @counter = 0
  csv << field_names

  Dir.entries(folder_path).each do |folder|
    next if folder == '.' or folder == '..'
    begin
      next unless File.directory?(folder)
      process_folder(folder, csv)
    rescue => e
      puts 'Unable to process file or folder, error reason: #{e.backtrace}' unless Rails.env.test?
    end
  end
  puts 'Uploaded #{@counter} number of pdfs to S3' unless Rails.env.test?
end
<commit_msg>Refs #39611 CSV file name change and code refactor to access folder with files<commit_after># rails runner script/upload_tax_documents_to_aws.rb '/Users/saikumar/dchbx/GLUE/gluedb/irs_documents/irs_docs2,/Users/saikumar/dchbx/GLUE/gluedb/irs_documents/irs_docs3' -e production

folder_paths = ARGV[0]

field_names  = %w(doc_uri uploaded_file_name)

def upload_to_s3(file_name, csv)
  base_name = File.basename(file_name)
  doc_uri = Aws::S3Storage.save(file_name, "tax-documents", base_name)
  if doc_uri
    @counter += 1
    csv << [
      doc_uri,
      base_name
    ]
  end
end

file_name = "#{Rails.root}/upload_tax_documents_to_aws_#{Time.now.strftime('%m_%d_%Y_%H_%M_%S')}.csv"

CSV.open(file_name, "w", force_quotes: true) do |csv|
  @counter = 0
  csv << field_names
  folder_paths.split(',').each do |folder_path|
    folder_path = folder_path.strip
    begin
      Dir.foreach(folder_path).each do |file_name|
        next if file_name == ('.' || '..')
        file_name = folder_path + file_name.insert(0, '/')
        upload_to_s3(file_name, csv) if File.file?(file_name)
      end
    rescue => e
      puts "Unable to process file or folder, error reason: #{e.backtrace}" unless Rails.env.test?
    end
  end

  puts "Uploaded #{@counter} number of pdfs to S3" unless Rails.env.test?
end

{'fraction_non_alphanumeric': 0.12514529252227818, 'fraction_numerical': 0.008911274699728787, 'mean_word_length': 4.003875968992248, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1066260', 'n_tokens_mistral': 985, 'n_tokens_neox': 954, 'n_words': 217}