<commit_before>require 'stockboy/configuration'
require 'stockboy/exceptions'
require 'stockboy/configurator'
require 'stockboy/template_file'
require 'stockboy/filter_chain'
require 'stockboy/candidate_record'

module Stockboy

  # This class wraps up the main interface for the process of fetching,
  # parsing and sorting data. When used with a predefined template file, you
  # can pass the name of the template to define it. This is the common way
  # to use Stockboy:
  #
  #   job = Stockboy::Job.define('my_template')
  #   if job.process
  #     job.records[:update].each do |r|
  #       # ...
  #     end
  #     job.records[:cancel].each do |r|
  #       # ...
  #     end
  #   end
  #
  class Job

    # Defines the data source for receiving data
    #
    # @return [Provider]
    #
    attr_accessor :provider

    # Defines the format for parsing received data
    #
    # @return [Reader]
    #
    attr_accessor :reader

    # Configures the mapping & translation of raw data fields
    #
    # @return [AttributeMap]
    #
    attr_reader :attributes

    # List of filters for sorting processed records
    #
    # @return [FilterChain]
    #
    # Filters are applied in order, first match will capture the record.
    # Records that don't match any
    #
    attr_reader :filters

    attr_reader :triggers

    # Lists of records grouped by filter key
    #
    # @return [Hash{Symbol=>Array}]
    #
    attr_reader :records

    # List of records not matched by any filter
    #
    # @return [Array<CandidateRecord>]
    #
    attr_reader :unfiltered_records

    # List of all records, filtered or not
    #
    # @return [Array<CandidateRecord>]
    #
    attr_reader :all_records

    # Initialize a new job
    #
    # @param [Hash] params
    # @option params [Provider]           :provider
    # @option params [Reader]             :reader
    # @option params [AttributeMap]       :attributes
    # @option params [Array,FilterChain]  :filters
    # @yield instance for further configuration or processing
    #
    def initialize(params={}, &block)
      @provider   = params[:provider]
      @reader     = params[:reader]
      @attributes = params[:attributes] || AttributeMap.new
      @filters    = FilterChain.new params[:filters]
      @triggers   = Hash.new { |h,k| h[k] = [] }
      @triggers.replace params[:triggers] if params[:triggers]
      yield self if block_given?
      reset
    end

    # Instantiate a job configured by DSL template file
    #
    # @param template_name [String] File basename from template load path
    # @yield instance for further configuration or processing
    # @see Configuration#template_load_paths
    #
    def self.define(template_name)
      return nil unless template = TemplateFile.read(template_name)
      job = Configurator.new(template, TemplateFile.find(template_name)).to_job
      yield job if block_given?
      job
    end

    # Fetch data and process it into groups of filtered records
    #
    # @return [Boolean] Success or failure
    #
    def process
      with_query_caching do
        load_records
        yield @records if block_given?
      end
      provider.errors.empty?
    end

    # Count of all processed records
    #
    # @!attribute [r] total_records
    # @return [Fixnum]
    #
    def total_records
      @all_records.size
    end

    # Counts of processed records grouped by filter key
    #
    # @return [Hash{Symbol=>Fixnum}]
    #
    def record_counts
      @records.reduce(Hash.new) { |a, (k,v)| a[k] = v.size; a }
    end<commit_msg>Expose provider data methods on job<commit_after>require 'stockboy/configuration'
require 'stockboy/exceptions'
require 'stockboy/configurator'
require 'stockboy/template_file'
require 'stockboy/filter_chain'
require 'stockboy/candidate_record'

module Stockboy

  # This class wraps up the main interface for the process of fetching,
  # parsing and sorting data. When used with a predefined template file, you
  # can pass the name of the template to define it. This is the common way
  # to use Stockboy:
  #
  #   job = Stockboy::Job.define('my_template')
  #   if job.process
  #     job.records[:update].each do |r|
  #       # ...
  #     end
  #     job.records[:cancel].each do |r|
  #       # ...
  #     end
  #   end
  #
  class Job

    # Defines the data source for receiving data
    #
    # @return [Provider]
    #
    attr_accessor :provider

    # Defines the format for parsing received data
    #
    # @return [Reader]
    #
    attr_accessor :reader

    # Configures the mapping & translation of raw data fields
    #
    # @return [AttributeMap]
    #
    attr_reader :attributes

    # List of filters for sorting processed records
    #
    # @return [FilterChain]
    #
    # Filters are applied in order, first match will capture the record.
    # Records that don't match any
    #
    attr_reader :filters

    attr_reader :triggers

    # Lists of records grouped by filter key
    #
    # @return [Hash{Symbol=>Array}]
    #
    attr_reader :records

    # List of records not matched by any filter
    #
    # @return [Array<CandidateRecord>]
    #
    attr_reader :unfiltered_records

    # List of all records, filtered or not
    #
    # @return [Array<CandidateRecord>]
    #
    attr_reader :all_records

    # Initialize a new job
    #
    # @param [Hash] params
    # @option params [Provider]           :provider
    # @option params [Reader]             :reader
    # @option params [AttributeMap]       :attributes
    # @option params [Array,FilterChain]  :filters
    # @yield instance for further configuration or processing
    #
    def initialize(params={}, &block)
      @provider   = params[:provider]
      @reader     = params[:reader]
      @attributes = params[:attributes] || AttributeMap.new
      @filters    = FilterChain.new params[:filters]
      @triggers   = Hash.new { |h,k| h[k] = [] }
      @triggers.replace params[:triggers] if params[:triggers]
      yield self if block_given?
      reset
    end

    # Instantiate a job configured by DSL template file
    #
    # @param template_name [String] File basename from template load path
    # @yield instance for further configuration or processing
    # @see Configuration#template_load_paths
    #
    def self.define(template_name)
      return nil unless template = TemplateFile.read(template_name)
      job = Configurator.new(template, TemplateFile.find(template_name)).to_job
      yield job if block_given?
      job
    end

    # Fetch data and process it into groups of filtered records
    #
    # @return [Boolean] Success or failure
    #
    def process
      with_query_caching do
        load_records
        yield @records if block_given?
      end
      provider.errors.empty?
    end

    def data
      provider.data
    end

    def data?(reduction=:all, &comparison)
      provider.data?(reduction, &comparison)
    end

    # Count of all processed records
    #
    # @!attribute [r] total_records
    # @return [Fixnum]
    #
    def total_records
      @all_records.size
    end

    # Counts of processed records grouped by filter key
    #
    # @return [Hash{Symbol=>Fixnum}]
    #
    def record_counts
      @records.reduce(Hash.new) { |a, (k,v)| a[k] = v.size; a }
    end
{'fraction_non_alphanumeric': 0.08755317689035268, 'fraction_numerical': 0.0, 'mean_word_length': 2.544747081712062, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2534471', 'n_tokens_mistral': 2174, 'n_tokens_neox': 2101, 'n_words': 668}