<commit_before>require 'automat/base'

module Automat::Cloudformation
  class Uploader < Automat::Base
    add_option :templatefiles, :s3path

    def validate_templates
      healthy=true
      Dir.glob(templatefiles) do |cfnfile|
        cfnstring=File.open(cfnfile, 'rb') { |f| f.read }
        if cfn.validate_template(cfnstring).has_key?(:code)
          logger.error "Validation failure on #{cfnfile}"
          healthy=false
        end
      end
      return healthy
    end

    def upload_templates
      log_options

      if validate_templates
        Dir.glob(templatefiles) do |cfnfile|
          logger.info "uploading #{cfnfile}"
          options={ :localfile => cfnfile, :s3file => "#{s3path}/#{File.basename(cfnfile)}" }
          s = Automat::S3::Uploader.new(options)
          s.upload
        end<commit_msg>added more logging to cfn validator/uploader<commit_after>require 'automat/base'

module Automat::Cloudformation
  class Uploader < Automat::Base
    add_option :templatefiles, :s3path

    def validate_templates
      healthy=true
      Dir.glob(templatefiles) do |cfnfile|
        cfnstring=File.open(cfnfile, 'rb') { |f| f.read }
        if cfn.validate_template(cfnstring).has_key?(:code)
          logger.error "Validation failure on #{cfnfile}"
          healthy=false
        else
          logger.info "Validation succeeded on #{cfnfile}"
        end
      end
      return healthy
    end

    def upload_templates
      log_options

      if validate_templates
        logger.info "Beginning to upload templates to s3"
        Dir.glob(templatefiles) do |cfnfile|
          options={ :localfile => cfnfile, :s3file => "#{s3path}/#{File.basename(cfnfile)}" }
          s = Automat::S3::Uploader.new(options)
          s.upload
        end
{'fraction_non_alphanumeric': 0.10236220472440945, 'fraction_numerical': 0.005061867266591676, 'mean_word_length': 2.6012145748987856, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2542005', 'n_tokens_mistral': 548, 'n_tokens_neox': 545, 'n_words': 120}