<commit_before>#! /usr/bin/ruby

# script to upload files to S3

require 'rubygems'
require 'aws/s3'

# needs to have these ENV vars set!!!!!
bucket_name = ENV['S3_BUCKET_NAME']
aws_account_id = ENV['AMAZON_ACCESS_KEY_ID']
aws_secret_key = ENV['AMAZON_SECRET_ACCESS_KEY']

datestamp = ARGV[0]
policy_name = ARGV[1]
filepath = ARGV[2]
filename = File.basename(filepath)
hostname = `hostname`.chomp

remote_path = "/#{datestamp}/#{hostname}/#{policy_name}/#{filename}"

# debugging information:
STDERR.puts "bucket_name: #{bucket_name}"
STDERR.puts "datestamp: #{datestamp}"
STDERR.puts "policy_name: #{policy_name}"
STDERR.puts "filepath: #{filepath}"
STDERR.puts "remote_path: #{remote_path}\n"

# make connection:
AWS::S3::Base.establish_connection!(
  :access_key_id     => aws_account_id,
  :secret_access_key => aws_secret_key
)

# THE BUCKET BETTER FUCKING EXIST!
# make sure the bucket exists
#AWS::S3::Bucket.create(bucket_name)

# upload file to the bucket
AWS::S3::S3Object.store(remote_path, open(filepath), bucket_name)
<commit_msg>added some todo. fixed bug if bucket doesn't exist<commit_after>#! /usr/bin/ruby

# script to upload files to S3

require 'rubygems'
require 'aws/s3'

# needs to have these ENV vars set!!!!!
bucket_name = ENV['S3_BUCKET_NAME']
aws_account_id = ENV['AMAZON_ACCESS_KEY_ID']
aws_secret_key = ENV['AMAZON_SECRET_ACCESS_KEY']

datestamp = ARGV[0]
policy_name = ARGV[1]
filepath = ARGV[2]

# TODO: sanity checks. print usage if the above doesn't work out.
filename = File.basename(filepath)
hostname = `hostname`.chomp

remote_path = "/#{datestamp}/#{hostname}/#{policy_name}/#{filename}"

# debugging information:
STDERR.puts "bucket_name: #{bucket_name}"
STDERR.puts "datestamp: #{datestamp}"
STDERR.puts "policy_name: #{policy_name}"
STDERR.puts "filepath: #{filepath}"
STDERR.puts "remote_path: #{remote_path}\n"

# make connection:
AWS::S3::Base.establish_connection!(
  :access_key_id     => aws_account_id,
  :secret_access_key => aws_secret_key
)

# upload file to the bucket
begin
  AWS::S3::S3Object.store(remote_path, open(filepath), bucket_name)
rescue AWS::S3::NoSuchBucket
  # THE BUCKET BETTER FUCKING EXIST!
  # make sure the bucket exists
  AWS::S3::Bucket.create(bucket_name)
  retry
end

{'fraction_non_alphanumeric': 0.13731609451627286, 'fraction_numerical': 0.009362460989745877, 'mean_word_length': 6.262135922330097, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5283870', 'n_tokens_mistral': 860, 'n_tokens_neox': 851, 'n_words': 184}