lib/pupa/runner.rb
def run(args, overrides = {})
      rest = opts.parse!(args)

      @options = OpenStruct.new(options.to_h.merge(overrides))

      if options.actions.empty?
        options.actions = %w(scrape import)
      end
      if options.tasks.empty?
        options.tasks = @processor_class.tasks
      end

      processor = @processor_class.new(options.output_dir,
        pipelined: options.pipelined,
        cache_dir: options.cache_dir,
        expires_in: options.expires_in,
        value_max_bytes: options.value_max_bytes,
        memcached_username: options.memcached_username,
        memcached_password: options.memcached_password,
        database_url: options.database_url,
        validate: options.validate,
        level: options.level,
        faraday_options: options.faraday_options,
        options: Hash[*rest])

      options.actions.each do |action|
        unless action == 'scrape' || processor.respond_to?(action)
          abort %(`#{action}` is not a #{opts.program_name} action. See `#{opts.program_name} --help` for a list of available actions.)
        end
      end

      if %w(DEBUG INFO).include?(options.level)
        puts "processor: #{@processor_class}"
        puts "actions: #{options.actions.join(', ')}"
        puts "tasks: #{options.tasks.join(', ')}"
      end

      if options.level == 'DEBUG'
        %w(output_dir pipelined cache_dir expires_in value_max_bytes memcached_username memcached_password database_url validate level).each do |option|
          puts "#{option}: #{options[option]}"
        end
        unless rest.empty?
          puts "options: #{rest.join(' ')}"
        end
      end

      exit if options.dry_run

      report = {
        plan: {
          processor: @processor_class,
          options: Marshal.load(Marshal.dump(options)).to_h,
          arguments: rest,
        },
        start: Time.now.utc,
      }

      if options.actions.delete('scrape')
        processor.store.clear
        report[:scrape] = {}
        options.tasks.each do |task_name|
          report[:scrape][task_name] = processor.dump_scraped_objects(task_name)
        end
      end

      options.actions.each do |action|
        processor.send(action)
        if processor.report.key?(action.to_sym)
          report.update(action.to_sym => processor.report[action.to_sym])
        end
      end

      if %w(DEBUG INFO).include?(options.level)
        report[:end] = Time.now.utc
        report[:time] = report[:end] - report[:start]
        puts JSON.dump(report)
      end
    end
def run(args, overrides = {})
      rest = opts.parse!(args)

      @options = OpenStruct.new(options.to_h.merge(overrides))

      if options.actions.empty?
        options.actions = %w(scrape import)
      end
      if options.tasks.empty?
        options.tasks = @processor_class.tasks
      end

      processor = @processor_class.new(options.output_dir,
        pipelined: options.pipelined,
        cache_dir: options.cache_dir,
        expires_in: options.expires_in,
        value_max_bytes: options.value_max_bytes,
        memcached_username: options.memcached_username,
        memcached_password: options.memcached_password,
        database_url: options.database_url,
        validate: options.validate,
        level: options.level,
        faraday_options: options.faraday_options,
        options: Hash[*rest])

      options.actions.each do |action|
        unless action == 'scrape' || processor.respond_to?(action)
          abort %(`#{action}` is not a #{opts.program_name} action. See `#{opts.program_name} --help` for a list of available actions.)
        end
      end

      if %w(DEBUG INFO).include?(options.level)
        puts "processor: #{@processor_class}"
        puts "actions: #{options.actions.join(', ')}"
        puts "tasks: #{options.tasks.join(', ')}"
      end

      if options.level == 'DEBUG'
        %w(output_dir pipelined cache_dir expires_in value_max_bytes memcached_username memcached_password database_url validate level).each do |option|
          puts "#{option}: #{options[option]}"
        end
        unless rest.empty?
          puts "options: #{rest.join(' ')}"
        end
      end

      exit if options.dry_run

      report = {
        plan: {
          processor: @processor_class,
          options: Marshal.load(Marshal.dump(options)).to_h,
          arguments: rest,
        },
        start: Time.now.utc,
      }

      if options.actions.delete('scrape')
        processor.store.clear
        report[:scrape] = {}
        options.tasks.each do |task_name|
          report[:scrape][task_name] = processor.dump_scraped_objects(task_name)
        end
      end

      options.actions.each do |action|
        processor.send(action)
        if processor.report.key?(action.to_sym)
          report.update(action.to_sym => processor.report[action.to_sym])
        end
      end

      if %w(DEBUG INFO).include?(options.level)
        report[:end] = Time.now.utc
        report[:time] = report[:end] - report[:start]
        puts JSON.dump(report)
      end
    end
Runs the action.

 @example Run from a command-line script

     runner.run(ARGV)

 @example Override the command-line options

     runner.run(ARGV, expires_in: 3600) # 1 hour

 @param [Array] args command-line arguments
 @param [Hash] overrides any overridden options