<commit_before>module ActsAsFilterable
  
  module ActiveRecordExt
    
    module Base
      
      def self.included(klazz)
        klazz.extend ClassMethods
        klazz.before_validation :apply_filters
      end
      
      module ClassMethods
        
        def self.extended(klazz)
          klazz.filters.keys.each do |key|
            klazz.class_eval %{
              def self.filter_for_#{key}(*args)
                filtered_attributes[:#{key}] |= args unless args.empty?
              end
            }, __FILE__, __LINE__
          end
        end

        def filters
          @filters ||= returning Hash.new([]) do |f|
            f[:digits]    = lambda { |value| value.gsub!(/[^\d]*/, "") || value }
            f[:lowercase] = lambda { |value| value.downcase! || value }
          end.freeze
        end
        
        def filtered_attributes
          @filtered_attributes ||= Hash.new []<commit_msg>reworking macro creation, adding better (more correct) line number reporting<commit_after>module ActsAsFilterable
  
  module ActiveRecordExt
    
    module Base
      
      def self.included(klazz)
        klazz.extend ClassMethods
        klazz.before_validation :apply_filters
      end
      
      module ClassMethods
        
        def self.extended(klazz)
          klazz.filters.keys.each do |key|
            klazz.class_eval <<-MACROS, __FILE__, __LINE__ + 1
              def self.filter_for_#{key}(*args)
                filtered_attributes[:#{key}] |= args unless args.empty?
              end
            MACROS
          end
        end

        def filters
          @filters ||= returning Hash.new([]) do |f|
            f[:digits]    = lambda { |value| value.gsub!(/[^\d]*/, "") || value }
            f[:lowercase] = lambda { |value| value.downcase! || value }
          end.freeze
        end
        
        def filtered_attributes
          @filtered_attributes ||= Hash.new []
{'fraction_non_alphanumeric': 0.10057024364955935, 'fraction_numerical': 0.0005184033177812338, 'mean_word_length': 1.6842837273991655, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2950696', 'n_tokens_mistral': 543, 'n_tokens_neox': 532, 'n_words': 122}