<commit_before>module Clerk
  class Base
    include ActiveModel::Validations

    def initialize
      @transformed_values = Array.new
    end

    def load(data)
      @raw_values = data.freeze
    end

    def results
      Enumerator.new do |yielder|
        @transformed_values.each do |value|
          yielder << value
        end
      end
    end

    def errors
      @template.errors || []
    end

    def self.apply_template(data)
      @raw_values.each_with_index do |value, index|
        @transformed_values[index] ||= @template.apply(value)
      end
    end

    def self.template
      @template ||= Clerk::Template.new 
      #We could do `block_given?` here but it might be better to let it error if nothing is provided
      yield @template 
      @template
    end

  end
end<commit_msg>ActiveModel validation support, hooray<commit_after>module Clerk
  class ResultSet
    include ActiveModel::Validations

    def initialize(data)
      @data = data.freeze
    end

    def read_attribute_for_validation(attribute)
      @data[attribute]
    end

    def self.name
      self.class.to_s
    end

    def self.copy_validations_from(klass)
      dup = klass.validators.dup
      dup.each do |validator|
        validates_with validator.class, validator.options.dup.merge({:attributes => validator.attributes.dup})
      end
    end
  end

  class Base
    include ActiveModel::Validations

    class << self
      attr_accessor :_result_set_klass
    end

    def initialize
      clear_transformed_data!
      result_klass = Class.new(Clerk::ResultSet)
      result_klass.copy_validations_from(self.class)
      self.class._result_set_klass = result_klass
    end

    def load(data)
      clear_transformed_data!
      raw_values = data.freeze
      @transformed_values = raw_values.map do |record|
        templated_data = self.class.apply_template(record) 
        transformed_data = self.class.apply_transformations(templated_data)
        self.class.result_sets(transformed_data)
      end
      self
    end

    def valid?
      @transformed_values.all? do |sets| 
        sets.all? { |set| set.valid? } 
      end
    end

    def errors
      error_messages = Hash.new
      @transformed_values.each_with_index do |sets, index|
        messages = sets.map { |set| set.errors.full_messages }.flatten.uniq
        error_messages[index + 1] = messages unless messages.empty?
      end
      error_messages
    end

    def self.apply_template(data)
      data
    end

    def self.apply_transformations(data)
      data
    end

    def self.template
      @template ||= Clerk::Template.new 
      yield @template if block_given?
      @template
    end

    private

    def self.result_sets(record)
      data = record.freeze
      sets = Array.new
      if template.has_grouped_element?
        grouped_data = data.reject! { |key, value| value.kind_of? Clerk::TemplateGroup }
        grouped_data.each do |key, values|
          values.each do |value|
            sets.push _result_set_klass.new(data.merge(value))
          end
        end
      else
        sets.push _result_set_klass.new(data)
      end
      sets
    end

    def clear_transformed_data!
      @transformed_values = Array.new
    end

  end
end
{'fraction_non_alphanumeric': 0.07446153846153847, 'fraction_numerical': 0.0003076923076923077, 'mean_word_length': 2.3584710743801653, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4693378', 'n_tokens_mistral': 1008, 'n_tokens_neox': 967, 'n_words': 229}