<commit_before>module Wukong

  # Provides methods for supporting the running of Wukong processors
  # and dataflows entirely locally, without any frameworks like Hadoop
  # or Storm.
  #
  # This module is actually a plugin for Wukong.
  module Local
    include Plugin

    # Configures the given +settings+ object with all settings
    # specific to Wukong::Local for the given program +name+.
    #
    # @param [Configliere::Param] settings the settings to configure
    # @param [String] program the name of the currently executing program
    def self.configure settings, program
      case program
      when 'wu-local'
        settings.define :run,      description: "Name of the processor or dataflow to use. Defaults to basename of the given path.", flag: 'r'
        settings.define :tcp_port, description: "Consume TCP requests on the given port instead of lines over STDIN", type: Integer, flag: 't'
        
        settings.define :from, description: "Parse input from given data format (json, tsv, &c.) before processing", type: String
        settings.define :to,   description: "Convert input to given data format (json, tsv, &c.) before emitting", type: String

        settings.define :consumes, description: "Parse input as instances of given model class before processing", type: Class
      end
    end

    # Boots Wukong::Local using the given +settings+ at the given
    # +root.
    #
    # @param [Configliere::Param] settings the settings to use to boot
    # @param [String] root the root directory to boot in
    def self.boot(settings, root)
    end
    
  end
end

require_relative('local/runner')
<commit_msg>no need to say 'type: String' when making a Configliere define -- it's assumed, and it causes a bug anyway...<commit_after>module Wukong

  # Provides methods for supporting the running of Wukong processors
  # and dataflows entirely locally, without any frameworks like Hadoop
  # or Storm.
  #
  # This module is actually a plugin for Wukong.
  module Local
    include Plugin

    # Configures the given +settings+ object with all settings
    # specific to Wukong::Local for the given program +name+.
    #
    # @param [Configliere::Param] settings the settings to configure
    # @param [String] program the name of the currently executing program
    def self.configure settings, program
      case program
      when 'wu-local'
        settings.define :run,      description: "Name of the processor or dataflow to use. Defaults to basename of the given path.", flag: 'r'
        settings.define :tcp_port, description: "Consume TCP requests on the given port instead of lines over STDIN", type: Integer, flag: 't'
        
        settings.define :from, description: "Parse input from given data format (json, tsv, &c.) before processing"
        settings.define :to,   description: "Convert input to given data format (json, tsv, &c.) before emitting"

        settings.define :consumes, description: "Parse input as instances of given model class before processing", type: Class
      end
    end

    # Boots Wukong::Local using the given +settings+ at the given
    # +root.
    #
    # @param [Configliere::Param] settings the settings to use to boot
    # @param [String] root the root directory to boot in
    def self.boot(settings, root)
    end
    
  end
end

require_relative('local/runner')

{'fraction_non_alphanumeric': 0.07568533969010727, 'fraction_numerical': 0.0, 'mean_word_length': 3.2120451693851946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2598609', 'n_tokens_mistral': 907, 'n_tokens_neox': 881, 'n_words': 410}