<commit_before>module Clerk
  module Transformations
    def transform(record)
      dup = record.dup
      self.class.transformations.each do |field, transformers|
        orig_value = record.get(field)
        transformed_value = transformers.inject(orig_value) do |memo, transformer|
          transformer.tranform(memo)
        end
        record.set(field, transformed_value)
      end
    end

    module ClassMethods
      def transforms(*fields, &block)
        klz = Class.new(BlockTransformer)
        klz.block = block
        transforms_with klz, fields
      end

      def transforms_with(klass, *fields)
        fields.each do |field|
          self.transformations[field].push(klass)
        end
      end

      def clear_transformations!
        self.transformations.clear
      end

      def transformations
        @transformations ||= Hash.new { |hash, key| hash[key] = Array.new }
      end
    end

    def self.included(base)
      base.extend(ClassMethods)
    end
  end

  class BlockTransformer
    def self.block=(&block)
      @block = block
    end

    def self.transform(value)
      @block.call(value)
    end
  end
end<commit_msg>Transformation support!!1!<commit_after>module Clerk
  module Transformations
    def transform(record)
      self.class.transformations.each do |field, transformers|
        orig_value = record.get(field)
        transformed_value = transformers.inject(orig_value) do |memo, transformer|
          transformer.transform(memo)
        end
        record.set(field, transformed_value)
      end
      record 
    end

    module ClassMethods
      def transforms(*fields, &block)
        klz = Class.new(BlockTransformer)
        klz.block(&block)
        transforms_with klz, *fields
      end

      def transforms_with(klass, *fields)
        #TODO check klass for .transform method!
        fields.each do |field|
          transformations[field].push(klass)
        end
      end

      def clear_transformations!
        self.transformations.clear
      end

      def transformations
        @transformations ||= Hash.new { |hash, key| hash[key] = Array.new }
      end
    end

    def self.included(base)
      base.extend(ClassMethods)
    end
  end

  class BlockTransformer
    def self.block(&block)
      @block = block
    end

    def self.transform(value)
      @block.call(value)
    end
  end
end
{'fraction_non_alphanumeric': 0.08028583438419504, 'fraction_numerical': 0.0004203446826397646, 'mean_word_length': 2.278236914600551, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1334452', 'n_tokens_mistral': 702, 'n_tokens_neox': 677, 'n_words': 162}