<commit_before># tdl-tools etdl processor
#
# Part of the Aeolus suite http://aeolusproject.org
# Licensed under the MIT license
# Copyright (C) 2013 Red Hat, Inc.
# Written By <NAME> <<EMAIL>>
#
###########################################################

require 'cloud_inst'
require 'package_system'

###############################################################
# Use various subsystems to process an etdl

class ETDLProcessor
  def launch_instance(etdl)
    puts "Connecting to deltacloud instance at #{TDLTools::CloudInst::DC_URL}...".green
    inst = TDLTools::CloudInst.new etdl.cloud_attributes

    puts "Booting up cloud instance".green
    puts "  This will take a few minutes to become fully accessible".green
    inst.launch

    if inst.address.nil?
      puts "could not find address to access instance with".red
      exit 1
    end

    puts "using address #{inst.address}\nssh:#{inst.ssh}\nscp:#{inst.scp}".green

    # TODO compare instance to etdl os

    inst
  end

  def process(etdl, instance)
    # TODO setup repos

    # yum install packages
    packages = etdl.instance_attributes[:packages].join(" ")
      puts "installing #{packages}".green
      puts instance.exec(TDLTools::PackageSystem.install_pkg_cmd(packages)).blue

    # copy files over
    files = etdl.instance_attributes[:files]
      files.each { |f|
        tf = Tempfile.new('tdl-launch')
        tf.write f[:contents]
        tf.close

        instance.cp tf.path, f[:name]
      }

    cmds = etdl.instance_attributes[:commands]
      cmds.each { |c|
        puts "running command #{c[:cmd]}"
        puts instance.exec("sudo -i #{c[:cmd]}").blue
      }
  end

  def verify(etdl, instance)
    etdl.verify_cmds.each { |v|
        puts "running verification #{v}"
        puts instance.exec("sudo -i #{v}").blue # TODO red if failed
    }
  end

  def terminate_instance(instance)
    # TODO
  end
end<commit_msg>small fixes to etdl processor<commit_after># tdl-tools etdl processor
#
# Part of the Aeolus suite http://aeolusproject.org
# Licensed under the MIT license
# Copyright (C) 2013 Red Hat, Inc.
# Written By <NAME> <<EMAIL>>
#
###########################################################

require 'cloud_inst'
require 'package_system'

###############################################################
# Use various subsystems to process an etdl

class ETDLProcessor
  def launch_instance(etdl)
    puts "Connecting to deltacloud instance at #{TDLTools::CloudInst::DC_URL}...".green
    inst = TDLTools::CloudInst.new etdl.cloud_attributes

    puts "Booting up cloud instance".green
    puts "  This will take a few minutes to become fully accessible".green
    inst.launch

    if inst.address.nil?
      puts "could not find address to access instance with".red
      exit 1
    end

    puts "using address #{inst.address}\nssh:#{inst.ssh}\nscp:#{inst.scp}".green

    # TODO compare instance to etdl os

    inst
  end

  def process(etdl, instance)
    # TODO setup repos

    # yum install packages
    packages = etdl.instance_attributes[:packages].join(" ")
      puts "installing #{packages}".green
      puts instance.exec(TDLTools::PackageSystem.install_pkg_cmd(packages)).blue

    # copy files over
    files = etdl.instance_attributes[:files]
      files.each { |f|
        tf = Tempfile.new('tdl-launch')
        tf.write f[:contents]
        tf.close

        instance.cp tf.path, f[:name]
      }

    cmds = etdl.instance_attributes[:commands]
      cmds.each { |c|
        puts "running command #{c[:cmd]}"
        puts instance.exec("#{c[:cmd]}").blue
      }
  end

  def verify(etdl, instance)
    etdl.verify_cmds.each { |v|
        puts "running verification #{v}"
        output = instance.exec("#{v}")
        if $?.to_i == 0
          puts output.blue
          puts "  ...success!".blue
        else
          puts output.red
          puts "  ... #{v} failed".red
        end
    }
  end

  def terminate_instance(instance)
    # TODO
  end
end
{'fraction_non_alphanumeric': 0.16729464958553128, 'fraction_numerical': 0.00276312484300427, 'mean_word_length': 2.9939819458375125, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '880938', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1189, 'n_words': 329}