<commit_before>require 'delegate'
require 'yaml'
require 'erb'
require 'pp'

require 'active_support/core_ext/hash'

class Masamune::Configuration
  extend Forwardable
  include Masamune::HasContext

  attr_accessor :quiet
  attr_accessor :verbose
  attr_accessor :debug
  attr_accessor :no_op
  attr_accessor :dry_run

  COMMANDS = %w(hive hadoop_streaming hadoop_filesystem elastic_mapreduce s3cmd postgres postgres_admin)
  COMMANDS.each do |command|
    attr_accessor command
    define_method(command) do
      instance_variable_get("@#{command}").symbolize_keys!
    end
  end

  def initialize(context)
    self.context   = context
    self.quiet    = false
    self.verbose  = false
    self.debug    = false
    self.no_op    = false
    self.dry_run  = false
    @templates    = Hash.new { |h,k| h[k] = {} }

    COMMANDS.each do |command|
      instance_variable_set("@#{command}", {})
    end
  end

  def load(file)
    @load_once ||= begin
      load_yaml_erb_file(file).each_pair do |command, value|
        if COMMANDS.include?(command)
          send("#{command}=", value)
        elsif command == 'paths'
          add_paths(value)
        end
      end
      logger.debug("Loaded configuration #{file}")
    end
  end

  def to_s
    io = StringIO.new
    rep = {"path" => filesystem.paths}
    COMMANDS.each do |command|
      rep[command] = send(command)
    end
    PP.pp(rep, io)
    io.string
  end

  def debug=(debug)
    @debug = debug
    context.reload_logger!
  end

  def bind_template(section, template, input_args = {})
    free_command = load_template(section, template)[:command].split(/\s+/)
    [].tap do |bind_command|
      free_command.each do |free_expr|
        if free_expr =~ /(%.*)/
          free_param = $1
          bind_command << free_expr.gsub!(free_param, bind_param(section, template, free_param, input_args))
        elsif param = free_expr
          bind_command << param
        end
      end
    end
  end

  def as_options
    opts = []
    opts << '--quiet'   if quiet
    opts << '--verbose' if verbose
    opts << '--debug'   if debug
    opts << '--no_op'   if no_op
    opts << '--dry_run' if dry_run
    opts
  end

  def_delegators :filesystem, :add_path, :get_path

  def load_yaml_erb_file(file)
    YAML.load(ERB.new(File.read(file)).result(binding))
  end

  class << self
    def default_config_file=(config_file)
      @default_config_file = config_file
    end

    def default_config_file
      @default_config_file ||= File.join(File.expand_path('../../../', __FILE__), 'config', 'masamune.yml.erb')
    end
  end

  def default_config_file
    self.class.default_config_file
  end

  private

  def load_template(section, template_name)
    @templates[section][template_name] ||= begin
      raise ArgumentError, "no configuration section #{section}" unless COMMANDS.include?(section.to_s)
      raise ArgumentError, 'no template_name' unless template_name
      templates = send(section).fetch(:templates, {}).symbolize_keys!
      template = templates[template_name.to_sym] or raise ArgumentError, "no template for #{template_name}"
      template.symbolize_keys!
      template.has_key?(:command) or raise ArgumentError, "no command for template #{template_name}"
      template[:default] ||= {}
      template[:default] = Hash[ template[:default].collect { |key,val| [key.to_sym, val.to_s] } ]
      template
    end
  end

  def bind_param(section, template, free_param, input_args = {})
    default = load_template(section, template).fetch(:default, {})
    param = free_param[/(?<=%).*/].to_sym
    default.merge(input_args.symbolize_keys || {})[param] or raise ArgumentError, "no param for #{free_param}"
  end

  def add_paths(paths)
    paths.each do |value|
      symbol, path, options = *value.to_a.flatten
      filesystem.add_path(symbol, path, options)
    end<commit_msg>Rename for consistency<commit_after>require 'delegate'
require 'yaml'
require 'erb'
require 'pp'

require 'active_support/core_ext/hash'

class Masamune::Configuration
  extend Forwardable
  include Masamune::HasContext

  attr_accessor :quiet
  attr_accessor :verbose
  attr_accessor :debug
  attr_accessor :no_op
  attr_accessor :dry_run

  COMMANDS = %w(hive hadoop_streaming hadoop_filesystem elastic_mapreduce s3cmd postgres postgres_admin)
  COMMANDS.each do |command|
    attr_accessor command
    define_method(command) do
      instance_variable_get("@#{command}").symbolize_keys!
    end
  end

  def initialize(context)
    self.context   = context
    self.quiet    = false
    self.verbose  = false
    self.debug    = false
    self.no_op    = false
    self.dry_run  = false
    @templates    = Hash.new { |h,k| h[k] = {} }

    COMMANDS.each do |command|
      instance_variable_set("@#{command}", {})
    end
  end

  def load(file)
    @load_once ||= begin
      load_yaml_erb_file(file).each_pair do |command, value|
        if COMMANDS.include?(command)
          send("#{command}=", value)
        elsif command == 'paths'
          load_paths(value)
        end
      end
      logger.debug("Loaded configuration #{file}")
    end
  end

  def to_s
    io = StringIO.new
    rep = {"path" => filesystem.paths}
    COMMANDS.each do |command|
      rep[command] = send(command)
    end
    PP.pp(rep, io)
    io.string
  end

  def debug=(debug)
    @debug = debug
    context.reload_logger!
  end

  def bind_template(section, template, input_args = {})
    free_command = load_template(section, template)[:command].split(/\s+/)
    [].tap do |bind_command|
      free_command.each do |free_expr|
        if free_expr =~ /(%.*)/
          free_param = $1
          bind_command << free_expr.gsub!(free_param, bind_param(section, template, free_param, input_args))
        elsif param = free_expr
          bind_command << param
        end
      end
    end
  end

  def as_options
    opts = []
    opts << '--quiet'   if quiet
    opts << '--verbose' if verbose
    opts << '--debug'   if debug
    opts << '--no_op'   if no_op
    opts << '--dry_run' if dry_run
    opts
  end

  def_delegators :filesystem, :add_path, :get_path

  def load_yaml_erb_file(file)
    YAML.load(ERB.new(File.read(file)).result(binding))
  end

  class << self
    def default_config_file=(config_file)
      @default_config_file = config_file
    end

    def default_config_file
      @default_config_file ||= File.join(File.expand_path('../../../', __FILE__), 'config', 'masamune.yml.erb')
    end
  end

  def default_config_file
    self.class.default_config_file
  end

  private

  def load_template(section, template_name)
    @templates[section][template_name] ||= begin
      raise ArgumentError, "no configuration section #{section}" unless COMMANDS.include?(section.to_s)
      raise ArgumentError, 'no template_name' unless template_name
      templates = send(section).fetch(:templates, {}).symbolize_keys!
      template = templates[template_name.to_sym] or raise ArgumentError, "no template for #{template_name}"
      template.symbolize_keys!
      template.has_key?(:command) or raise ArgumentError, "no command for template #{template_name}"
      template[:default] ||= {}
      template[:default] = Hash[ template[:default].collect { |key,val| [key.to_sym, val.to_s] } ]
      template
    end
  end

  def bind_param(section, template, free_param, input_args = {})
    default = load_template(section, template).fetch(:default, {})
    param = free_param[/(?<=%).*/].to_sym
    default.merge(input_args.symbolize_keys || {})[param] or raise ArgumentError, "no param for #{free_param}"
  end

  def load_paths(paths)
    paths.each do |value|
      symbol, path, options = *value.to_a.flatten
      add_path(symbol, path, options)
    end
{'fraction_non_alphanumeric': 0.11690833764888658, 'fraction_numerical': 0.0005178663904712584, 'mean_word_length': 3.1266025641025643, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3924974', 'n_tokens_mistral': 2551, 'n_tokens_neox': 2492, 'n_words': 569}