<commit_before>require 'palletjack'
require 'fileutils'
require 'optparse'
require 'singleton'
require 'rugged'

module PalletJack
  # Superclass for PalletJack tool implementations
  #
  # Provides convenience methods for option parsing, file generation,
  # and warehouse structure management.
  #
  # Example:
  #   require 'palletjack/tool'
  #
  #   class MyTool < PalletJack::Tool
  #     def parse_options(parser)
  #       parser.on('-o DIR', '--output DIR',
  #                 'output directory',
  #                 String) {|dir| options[:output] = dir }
  #
  #       required_option :output
  #     end
  #
  #     attr_reader :state
  #
  #     def process
  #       @state = {}
  #       jack.each(kind:'system') do |sys|
  #         @state[sys.name] = sys
  #       end
  #     end
  #
  #     def output
  #       @state.each do |name, data|
  #         config_dir :output, name
  #         config_file :output, name, "dump.yaml" do |file|
  #           file << data.to_yaml
  #         end
  #       end
  #     end
  #   end
  #
  #   if __FILE__ == $0
  #     MyTool.run
  #   end

  class Tool
    include Singleton

    # :call-seq:
    # run
    #
    # Main tool framework driver.
    #
    # Run the entire tool; setup, process and output. Actual tools
    # will want to use this function, while testing and other
    # activities that require poking around in internal state will
    # want to run the partial functions instead.
    #
    # Example:
    #
    #   if MyTool.standalone?(__FILE__)
    #     MyTool.run
    #   end

    def self.run(&block)
      instance.setup
      instance.process
      instance.output
    end

    # Predicate for detecting if we are being invoked as a standalone
    # tool, or loaded by e.g. a test framework.

    def self.standalone?(file)
      File::basename(file) == File::basename($0)
    end

    # Generate data in an internal format, saving it for later testing
    # or writing to disk by #output.
    #
    # Override this function in specific tool classes.
    #
    # Example:
    #
    #   class MyTool < PalletJack::Tool
    #     def process
    #       @systems = Set.new
    #       jack.each(kind:'system') do |s|
    #         @systems << s
    #       end
    #     end
    #   end

    def process
    end

    # Output data in its final format, probably to disk or stdout.
    #
    # Example:
    #
    #   class MyTool < PalletJack::Tool
    #     def output
    #       @systems.each do |s|
    #         puts s.name
    #       end
    #     end
    #   end

    def output
    end

    # Return the command line argument list to be used. Replace this
    # method when testing.

    def argv
      ARGV
    end

    # Set up the singleton instance
    #
    # Default setup will add options for --warehouse and --help to the
    # OptionParser, and set the banner to something useful.
    #
    # Any exceptions raised during option parsing will abort execution
    # with usage information.

    def setup
      @parser = OptionParser.new
      @options = {}
      @option_checks = []

      @parser.banner = "Usage: #{$PROGRAM_NAME} -w <warehouse> [options]"
      @parser.separator ''
      @parser.on('-w DIR', '--warehouse DIR',
                 'warehouse directory', String) {|dir|
        @options[:warehouse] = dir }
      @parser.on_tail('-h', '--help', 'display usage information') {
        raise OptionParser::ParseError }

      parse_options(@parser)

      @parser.parse!(argv)
      @option_checks.each {|check| check.call }
    rescue
      abort(usage)
    end

    # Additional option parsing
    #
    # The default instance initalization will add option parsing for
    # <tt>-w</tt>/<tt>--warehouse</tt> and <tt>-h</tt>/<tt>--help</tt>,
    # and a simple banner string.
    #
    # Implementations needing more options than the default, a more<commit_msg>Only rescue OptionParser::ParseError<commit_after>require 'palletjack'
require 'fileutils'
require 'optparse'
require 'singleton'
require 'rugged'

module PalletJack
  # Superclass for PalletJack tool implementations
  #
  # Provides convenience methods for option parsing, file generation,
  # and warehouse structure management.
  #
  # Example:
  #   require 'palletjack/tool'
  #
  #   class MyTool < PalletJack::Tool
  #     def parse_options(parser)
  #       parser.on('-o DIR', '--output DIR',
  #                 'output directory',
  #                 String) {|dir| options[:output] = dir }
  #
  #       required_option :output
  #     end
  #
  #     attr_reader :state
  #
  #     def process
  #       @state = {}
  #       jack.each(kind:'system') do |sys|
  #         @state[sys.name] = sys
  #       end
  #     end
  #
  #     def output
  #       @state.each do |name, data|
  #         config_dir :output, name
  #         config_file :output, name, "dump.yaml" do |file|
  #           file << data.to_yaml
  #         end
  #       end
  #     end
  #   end
  #
  #   if __FILE__ == $0
  #     MyTool.run
  #   end

  class Tool
    include Singleton

    # :call-seq:
    # run
    #
    # Main tool framework driver.
    #
    # Run the entire tool; setup, process and output. Actual tools
    # will want to use this function, while testing and other
    # activities that require poking around in internal state will
    # want to run the partial functions instead.
    #
    # Example:
    #
    #   if MyTool.standalone?(__FILE__)
    #     MyTool.run
    #   end

    def self.run(&block)
      instance.setup
      instance.process
      instance.output
    end

    # Predicate for detecting if we are being invoked as a standalone
    # tool, or loaded by e.g. a test framework.

    def self.standalone?(file)
      File::basename(file) == File::basename($0)
    end

    # Generate data in an internal format, saving it for later testing
    # or writing to disk by #output.
    #
    # Override this function in specific tool classes.
    #
    # Example:
    #
    #   class MyTool < PalletJack::Tool
    #     def process
    #       @systems = Set.new
    #       jack.each(kind:'system') do |s|
    #         @systems << s
    #       end
    #     end
    #   end

    def process
    end

    # Output data in its final format, probably to disk or stdout.
    #
    # Example:
    #
    #   class MyTool < PalletJack::Tool
    #     def output
    #       @systems.each do |s|
    #         puts s.name
    #       end
    #     end
    #   end

    def output
    end

    # Return the command line argument list to be used. Replace this
    # method when testing.

    def argv
      ARGV
    end

    # Set up the singleton instance
    #
    # Default setup will add options for --warehouse and --help to the
    # OptionParser, and set the banner to something useful.
    #
    # Any exceptions raised during option parsing will abort execution
    # with usage information.

    def setup
      @parser = OptionParser.new
      @options = {}
      @option_checks = []

      @parser.banner = "Usage: #{$PROGRAM_NAME} -w <warehouse> [options]"
      @parser.separator ''
      @parser.on('-w DIR', '--warehouse DIR',
                 'warehouse directory', String) {|dir|
        @options[:warehouse] = dir }
      @parser.on_tail('-h', '--help', 'display usage information') {
        raise OptionParser::ParseError }

      parse_options(@parser)

      @parser.parse!(argv)
      @option_checks.each {|check| check.call }
    rescue OptionParser::ParseError => error
      if error.args.empty?
        abort(usage)
      else
        abort("#{error}\n\n#{usage}")
      end
    end

    # Additional option parsing
    #
    # The default instance initalization will add option parsing for
    # <tt>-w</tt>/<tt>--warehouse</tt> and <tt>-h</tt>/<tt>--help</tt>,
    # and a simple banner string.
    #
    # Implementations needing more options than the default, a more
{'fraction_non_alphanumeric': 0.10201495374477253, 'fraction_numerical': 0.000506906602458497, 'mean_word_length': 1.9702672186676704, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6575272', 'n_tokens_mistral': 2444, 'n_tokens_neox': 2376, 'n_words': 730}