<commit_before>require "awful/version"

require 'dotenv'
require 'aws-sdk'
require 'thor'
require 'yaml'
require 'erb'

module Awful
  class Cli < Thor
    class_option :env,   aliases: '-e', default: nil, desc: 'Load environment variables from file'
    class_option :quiet, aliases: '-q', default: nil, desc: 'Quieten output'

    no_commands do
      def ec2
        @ec2 ||= Aws::EC2::Client.new
      end

      def autoscaling
        @autoscaling ||= Aws::AutoScaling::Client.new
      end

      def elb
        @elb ||= Aws::ElasticLoadBalancing::Client.new
      end

      def rds
        @rds ||= Aws::RDS::Client.new
      end

      def datapipeline
        @datapipeline ||= Aws::DataPipeline::Client.new
      end

      ## to use dynamodb-local, set DYNAMO_ENDPOINT or DYNAMODB_ENDPOINT to e.g. http://localhost:8000
      def dynamodb
        options = { endpoint: ENV['DYNAMO_ENDPOINT'] || ENV['DYNAMODB_ENDPOINT'] }.reject { |_,v| v.nil? }
        @dynamodb ||= Aws::DynamoDB::Client.new(options)
      end

      def s3
        @s3 ||= Aws::S3::Client.new
      end

      def symbolize_keys(thing)
        if thing.is_a?(Hash)
          Hash[ thing.map { |k,v| [ k.to_sym, symbolize_keys(v) ] } ]
        elsif thing.respond_to?(:map)
          thing.map { |v| symbolize_keys(v) }
        else
          thing
        end
      end

      def stringify_keys(thing)
        if thing.is_a?(Hash)
          Hash[ thing.map { |k,v| [ k.to_s, stringify_keys(v) ] } ]
        elsif thing.respond_to?(:map)
          thing.map { |v| stringify_keys(v) }
        else
          thing
        end
      end

      ## merge options with config from erb-parsed yaml stdin or file
      def load_cfg(options = {}, file = nil)
        Dotenv.overload(options[:env]) if options[:env]
        src = (file and File.read(file)) || ((not $stdin.tty?) and $stdin.read)<commit_msg>add a dynamodb_simple client for getting simple_json responses from api<commit_after>require "awful/version"

require 'dotenv'
require 'aws-sdk'
require 'thor'
require 'yaml'
require 'erb'

module Awful
  class Cli < Thor
    class_option :env,   aliases: '-e', default: nil, desc: 'Load environment variables from file'
    class_option :quiet, aliases: '-q', default: nil, desc: 'Quieten output'

    no_commands do
      def ec2
        @ec2 ||= Aws::EC2::Client.new
      end

      def autoscaling
        @autoscaling ||= Aws::AutoScaling::Client.new
      end

      def elb
        @elb ||= Aws::ElasticLoadBalancing::Client.new
      end

      def rds
        @rds ||= Aws::RDS::Client.new
      end

      def datapipeline
        @datapipeline ||= Aws::DataPipeline::Client.new
      end

      ## to use dynamodb-local, set DYNAMO_ENDPOINT or DYNAMODB_ENDPOINT to e.g. http://localhost:8000
      def dynamodb
        options = {endpoint: ENV['DYNAMO_ENDPOINT'] || ENV['DYNAMODB_ENDPOINT']}.reject { |_,v| v.nil? }
        @dynamodb ||= Aws::DynamoDB::Client.new(options)
      end

      ## dynamodb client with parameter conversion turned off,
      ## for getting result as Aws::Plugins::Protocols::JsonRpc;
      ## see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataFormat.html;
      ## client method calls need simple attributes in requests also (strings not symbols, etc)
      def dynamodb_simple
        options = {
          simple_attributes: false,
          simple_json: true,
          endpoint: ENV['DYNAMO_ENDPOINT'] || ENV['DYNAMODB_ENDPOINT'],
        }.reject { |_,v| v.nil? }
        @dynamodb_simple ||= Aws::DynamoDB::Client.new(options)
      end

      def s3
        @s3 ||= Aws::S3::Client.new
      end

      def symbolize_keys(thing)
        if thing.is_a?(Hash)
          Hash[ thing.map { |k,v| [ k.to_sym, symbolize_keys(v) ] } ]
        elsif thing.respond_to?(:map)
          thing.map { |v| symbolize_keys(v) }
        else
          thing
        end
      end

      def stringify_keys(thing)
        if thing.is_a?(Hash)
          Hash[ thing.map { |k,v| [ k.to_s, stringify_keys(v) ] } ]
        elsif thing.respond_to?(:map)
          thing.map { |v| stringify_keys(v) }
        else
          thing
        end
      end

      ## merge options with config from erb-parsed yaml stdin or file
      def load_cfg(options = {}, file = nil)
        Dotenv.overload(options[:env]) if options[:env]
        src = (file and File.read(file)) || ((not $stdin.tty?) and $stdin.read)
{'fraction_non_alphanumeric': 0.12344843150530355, 'fraction_numerical': 0.0045136538027533285, 'mean_word_length': 2.3601213040181954, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '561297', 'n_tokens_mistral': 1530, 'n_tokens_neox': 1475, 'n_words': 356}