<commit_before># http://jmlacroix.com/archives/cloudfront-publishing.html

local   = ARGV[0]
s3_dest = ARGV[1]

if local == nil || s3_dest == nil 
  puts "syntax aws_s3_sync.rb local_source s3_dest"
  exit
end

# invalidate = "#{Dir.pwd}/aws_cf_invalidate.rb"
# if !File.exists?(invalidate)
#   puts "please download the aws_cf_invalidate.rb script"
#   exit
# end

s3_dest   = s3_dest.split('/')
s3_bucket = s3_dest.shift
s3_path   = s3_dest.join('/')

s3_path += '/' unless s3_dest.length == 0

# add far future expires and gzip headers for all .js and .css files
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --exclude '*' --rinclude '.css$|.js$' --acl-public --add-header "Expires: Tue, 19 Jan 2038 03:14:07 GMT" --add-header "Content-Encoding: gzip" ]

# add far future expires header for all images
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --exclude '*' --rinclude '.ico$|.jpg$|.png$|.gif$' --acl-public --add-header "Expires: Tue, 19 Jan 2038 03:14:07 GMT" ]

# add gzip headers for all html files
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --exclude '*' --rinclude '.html$' --acl-public --add-header "Content-Encoding: gzip" ]

# upload anything else
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --rexclude '.html$|.css$|.js$|.ico$|.jpg$|.png$|.gif$' --acl-public ]

# delete files that were removed
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --delete-removed ]

# invalidate all files on CDN
# files = %x[ cd _site && find . -type f ].split("\n").map do |f| 
#   s3_path + f[2,f.length]
# end
# 
# %x[ ruby #{invalidate} #{files.join(' ')} ]<commit_msg>added cache-control headers for js and css<commit_after># http://jmlacroix.com/archives/cloudfront-publishing.html

local   = ARGV[0]
s3_dest = ARGV[1]

if local == nil || s3_dest == nil 
  puts "syntax aws_s3_sync.rb local_source s3_dest"
  exit
end

# invalidate = "#{Dir.pwd}/aws_cf_invalidate.rb"
# if !File.exists?(invalidate)
#   puts "please download the aws_cf_invalidate.rb script"
#   exit
# end

s3_dest   = s3_dest.split('/')
s3_bucket = s3_dest.shift
s3_path   = s3_dest.join('/')

s3_path += '/' unless s3_dest.length == 0

# add far future expires and gzip headers for all .js and .css files
# cache-control private headers are added to get around proxy issues. AWS won't be serving non-gzipped versions so the Vary header doesn't apply. More info: http://code.google.com/speed/page-speed/docs/caching.html#LeverageProxyCaching
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --exclude '*' --rinclude '.css$|.js$' --acl-public --add-header "Expires: Tue, 19 Jan 2038 03:14:07 GMT" --add-header "Content-Encoding: gzip" --add-header "Cache-Control: private" ]

# add far future expires header for all images
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --exclude '*' --rinclude '.ico$|.jpg$|.png$|.gif$' --acl-public --add-header "Expires: Tue, 19 Jan 2038 03:14:07 GMT" ]

# add gzip headers for all html files
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --exclude '*' --rinclude '.html$' --acl-public --add-header "Content-Encoding: gzip" ]

# upload anything else
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --rexclude '.html$|.css$|.js$|.ico$|.jpg$|.png$|.gif$' --acl-public ]

# delete files that were removed
%x[ $(which s3cmd) sync #{local} s3://#{s3_bucket}/#{s3_path} --delete-removed ]

# invalidate all files on CDN
# files = %x[ cd _site && find . -type f ].split("\n").map do |f| 
#   s3_path + f[2,f.length]
# end
# 
# %x[ ruby #{invalidate} #{files.join(' ')} ]
{'fraction_non_alphanumeric': 0.19219052893935198, 'fraction_numerical': 0.03378565494322902, 'mean_word_length': 3.9411764705882355, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6677041', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1429, 'n_words': 366}