<commit_before>module Awful
  module Short
    def s3(*args)
      Awful::S3.new.invoke(*args)
    end
  end

  class S3 < Cli
    no_commands do
      ## resource interface to S3 commands
      def s3_resource
        Aws::S3::Resource.new(client: s3)
      end
    end

    desc 'ls PATTERN', 'list buckets or objects'
    method_option :long, aliases: '-l', default: false, desc: 'Long listing'
    def ls(name = '.')
      if name.include?('/')
        bucket, prefix = name.split('/', 2)
        invoke 'objects', [bucket, prefix], options
      else
        invoke 'buckets', [name], options
      end
    end

    desc 'buckets [PATTERN]', 'list buckets'
    method_option :long, aliases: '-l', default: false, desc: 'Long listing'
    def buckets(name = /./)
      s3.list_buckets.buckets.select do |bucket|
        bucket.name.match(/#{name}/)
      end.tap do |list|
        if options[:long]
          print_table list.map { |b| [ b.name, b.creation_date ] }
        else
          puts list.map(&:name)
        end
      end
    end

    desc 'objects BUCKET [PATTERN]', 'list objects in bucket'
    def objects(bucket, prefix = nil)
      s3_resource.bucket(bucket).objects(prefix: prefix).map do |object|
        object.key
      end.tap { |list| puts list }
    end

    desc 'cat BUCKET/OBJECT', 'stream s3 object to stdout'
    def cat(path)
      bucket, key = path.split('/', 2)
      s3.get_object(bucket: bucket, key: key) do |chunk|
        $stdout.write(chunk)
      end
    end

    desc 'upload FILE BUCKET/OBJECT', 'upload FILE to given object'
    def upload(file, s3path)
      bucket, key = s3path.split('/', 2)
      s3_resource.bucket(bucket).object(key).upload_file(file)
    end

    desc 'remove_bucket NAME', 'delete a bucket, which must be empty'<commit_msg>ugly task to get subset of buckets by tags<commit_after>module Awful
  module Short
    def s3(*args)
      Awful::S3.new.invoke(*args)
    end
  end

  class S3 < Cli
    no_commands do
      ## resource interface to S3 commands
      def s3_resource
        Aws::S3::Resource.new(client: s3)
      end

      def get_tags(bucket_name)
        s3.get_bucket_tagging(bucket: bucket_name).tag_set
      rescue Aws::S3::Errors::NoSuchTagSet # sdk throws this if no tags
        nil
      end
    end

    desc 'ls PATTERN', 'list buckets or objects'
    method_option :long, aliases: '-l', default: false, desc: 'Long listing'
    def ls(name = '.')
      if name.include?('/')
        bucket, prefix = name.split('/', 2)
        invoke 'objects', [bucket, prefix], options
      else
        invoke 'buckets', [name], options
      end
    end

    desc 'buckets [PATTERN]', 'list buckets'
    method_option :long, aliases: '-l', default: false, desc: 'Long listing'
    def buckets(name = /./)
      s3.list_buckets.buckets.select do |bucket|
        bucket.name.match(/#{name}/)
      end.tap do |list|
        if options[:long]
          print_table list.map { |b| [ b.name, b.creation_date ] }
        else
          puts list.map(&:name)
        end
      end
    end

    desc 'objects BUCKET [PATTERN]', 'list objects in bucket'
    def objects(bucket, prefix = nil)
      s3_resource.bucket(bucket).objects(prefix: prefix).map do |object|
        object.key
      end.tap { |list| puts list }
    end

    desc 'tagged [NAME_PREFIX]', 'list buckets matching given tags'
    method_option :tags,     aliases: '-t', type: :array,  default: [],  desc: 'List of tag=value to filter'
    method_option :stack,    aliases: '-s', type: :string, default: nil, desc: 'Filter by stack name'
    method_option :resource, aliases: '-r', type: :string, default: nil, desc: 'Filter by stack resource logical id'
    def tagged(name = '.')
      conditions = options[:tags].map do |tag|
        key, value = tag.split('=')
        ->(set) { (set[0] == key) && (set[1] == value) }
      end
      if options[:stack]
        conditions << ->(set) { (set[0] == 'aws:cloudformation:stack-name') && (set[1] == options[:stack]) }
      end
      if options[:resource]
        conditions << ->(set) { (set[0] == 'aws:cloudformation:logical-id') && (set[1] == options[:resource]) }
      end

      ## get all buckets and check for a match with conditions
      s3.list_buckets.buckets.select do |b|
        b.name.match(/^#{name}/i)
      end.map do |bucket|
        tags = get_tags(bucket.name) or next
        tags.any? do |set|
          conditions.any? { |c| c.call(set) }
        end && bucket
      end.select{|b| b}.tap do |buckets|
        puts buckets.map(&:name)
      end
    end

    desc 'cat BUCKET/OBJECT', 'stream s3 object to stdout'
    def cat(path)
      bucket, key = path.split('/', 2)
      s3.get_object(bucket: bucket, key: key) do |chunk|
        $stdout.write(chunk)
      end
    end

    desc 'upload FILE BUCKET/OBJECT', 'upload FILE to given object'
    def upload(file, s3path)
      bucket, key = s3path.split('/', 2)
      s3_resource.bucket(bucket).object(key).upload_file(file)
    end

    desc 'remove_bucket NAME', 'delete a bucket, which must be empty'
{'fraction_non_alphanumeric': 0.13370143955827252, 'fraction_numerical': 0.00847958982449221, 'mean_word_length': 2.440976933514247, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2077783', 'n_tokens_mistral': 1685, 'n_tokens_neox': 1634, 'n_words': 480}