<commit_before>#! /usr/bin/ruby

# script to upload files to S3

require 'rubygems'
require 'aws/s3'

# needs to have these ENV vars set!!!!!
bucket_name = ENV['S3_BUCKET_NAME']
aws_account_id = ENV['AMAZON_ACCESS_KEY_ID']
aws_secret_key = ENV['AMAZON_SECRET_ACCESS_KEY']

datestamp = ARGV[0]
policy_name = ARGV[1]
filepath = ARGV[2]
filename = File.basename(filepath)
hostname = `hostname`.chomp

remote_path = "/#{datestamp}/#{hostname}/#{policy_name}/#{filename}"

# debugging information:
puts "bucket_name: #{bucket_name}"
puts "datestamp: #{datestamp}"
puts "policy_name: #{policy_name}"
puts "filepath: #{filepath}"
puts "remote_path: #{remote_path}\n"

# make connection:
AWS::S3::Base.establish_connection!(
    :access_key_id     => aws_account_id,
    :secret_access_key => aws_secret_key
  )

# THE BUCKET BETTER FUCKING EXIST!
# make sure the bucket exists
#AWS::S3::Bucket.create(bucket_name)

# upload file to the bucket<commit_msg>upload.rb now prints debugging information on STDERR rather than STDOUT<commit_after>#! /usr/bin/ruby

# script to upload files to S3

require 'rubygems'
require 'aws/s3'

# needs to have these ENV vars set!!!!!
bucket_name = ENV['S3_BUCKET_NAME']
aws_account_id = ENV['AMAZON_ACCESS_KEY_ID']
aws_secret_key = ENV['AMAZON_SECRET_ACCESS_KEY']

datestamp = ARGV[0]
policy_name = ARGV[1]
filepath = ARGV[2]
filename = File.basename(filepath)
hostname = `hostname`.chomp

remote_path = "/#{datestamp}/#{hostname}/#{policy_name}/#{filename}"

# debugging information:
STDERR.puts "bucket_name: #{bucket_name}"
STDERR.puts "datestamp: #{datestamp}"
STDERR.puts "policy_name: #{policy_name}"
STDERR.puts "filepath: #{filepath}"
STDERR.puts "remote_path: #{remote_path}\n"

# make connection:
AWS::S3::Base.establish_connection!(
    :access_key_id     => aws_account_id,
    :secret_access_key => aws_secret_key
  )

# THE BUCKET BETTER FUCKING EXIST!
# make sure the bucket exists
#AWS::S3::Bucket.create(bucket_name)

# upload file to the bucket
{'fraction_non_alphanumeric': 0.13688978359335682, 'fraction_numerical': 0.00805234021137393, 'mean_word_length': 6.151079136690647, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4382429', 'n_tokens_mistral': 755, 'n_tokens_neox': 750, 'n_words': 162}