<commit_before>require 'zlib'
require 'fileutils'
require 'tmpdir'
require 'rubygems'
require 'thor'
require 'highline'
require 'aws-sdk'
require 'archive/tar/minitar'

class NcssCookbooks < Thor
  class_option :access_key_id, :alias => '-a'
  class_option :secret_access_key, :alias => '-s'

  desc 'create <bucket>', 'create ncss bucket'
  def create(bucket_name)
    ncss.buckets.create(bucket_name)
  end

  desc 'upload <bucket>', 'upload . to ncss bucket'
  def upload(bucket_name)
    pwd = Dir.pwd
    unless File.exists?("#{pwd}/VERSION") || File.readble?("#{pwd}/VERSION")
      abort "Error: VERSION file does not exists."
    end
    version = File.read("#{pwd}/VERSION").chomp

    bucket = ncss.buckets[bucket_name]
    unless bucket.exists?
      abort "Error: Bucket #{bucket_name} does not exist."
    end
    object = bucket.objects["v#{version}/cookbooks.tgz"]

    Dir.mktmpdir('ncss-cookbooks') do |dir|
      FileUtils.mkdir_p("#{dir}/cookbooks/")
      FileUtils.cp_r(Dir["#{pwd}/*"], "#{dir}/cookbooks/")

      FileUtils.cd(dir)
      cookbooks_tgz = Zlib::GzipWriter.new(File.open("cookbooks.tgz", "wb"))
      Archive::Tar::Minitar.pack('cookbooks', cookbooks_tgz)

      object.write(:file => "cookbooks.tgz")
      object.acl = :public_read
    end

    puts object.url_for(:read).to_s.gsub(/\?(.+)$/, '')
  end
<commit_msg>add logging.<commit_after>require 'zlib'
require 'fileutils'
require 'tmpdir'
require 'rubygems'
require 'thor'
require 'highline'
require 'aws-sdk'
require 'archive/tar/minitar'

class NcssCookbooks < Thor
  class_option :access_key_id, :alias => '-a'
  class_option :secret_access_key, :alias => '-s'

  desc 'create <bucket>', 'create ncss bucket'
  def create(bucket_name)
    ncss.buckets.create(bucket_name)
  end

  desc 'upload <bucket>', 'upload . to ncss bucket'
  option :verbose
  def upload(bucket_name)
    pwd = Dir.pwd
    unless File.exists?("#{pwd}/VERSION") || File.readble?("#{pwd}/VERSION")
      abort "Error: VERSION file does not exists."
    end
    version = File.read("#{pwd}/VERSION").chomp
    puts "Version is v#{version}"

    bucket = ncss.buckets[bucket_name]
    unless bucket.exists?
      abort "Error: Bucket #{bucket_name} does not exist."
    end
    object = bucket.objects["v#{version}/cookbooks.tgz"]

    Dir.mktmpdir('ncss-cookbooks') do |dir|
      puts "Creating directory #{dir}/cookbooks/"
      FileUtils.mkdir_p("#{dir}/cookbooks/")

      puts "Copying files from #{@pwd} to #{dir}/cookbooks/"
      FileUtils.cp_r(pwd_files, "#{dir}/cookbooks/")

      puts "Archiving #{dir}/cookbooks/ to #{dir}/cookbooks.tgz"
      FileUtils.cd(dir)
      cookbooks_tgz = Zlib::GzipWriter.new(File.open("cookbooks.tgz", "wb"))
      Archive::Tar::Minitar.pack('cookbooks', cookbooks_tgz)

      if options[:verbose]
        puts "Archive includes:"
        system "tar tzvf cookbooks.tgz"
      end

      puts "Uploading #{dir}/cookbooks.tgz to bucket:#{bucket_name}, object:#{object.key}"
      object.write(:file => "cookbooks.tgz")
      object.acl = :public_read

      puts "Temporary directory #{dir} will be deleted automatically"
    end

    puts "Uploaded to " + object.url_for(:read).to_s.gsub(/\?(.+)$/, '')
  end

{'fraction_non_alphanumeric': 0.15786202610316968, 'fraction_numerical': 0.0, 'mean_word_length': 3.645021645021645, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6398703', 'n_tokens_mistral': 1130, 'n_tokens_neox': 1107, 'n_words': 239}