<commit_before>module Ixtlan
  module Babel
    class HashFilter

      def initialize(context_or_options = nil)
        use(context_or_options)
      end
      
      private

      def context
        @context ||= {}
      end

      public

      def add_custom_serializers(map)
        @map = map
      end

      def default_context_key(default = nil)
        @default = default if default
        @default
      end

      def []=(key, options)
        context[key.to_sym] = options if key
      end

      def [](key)
        context[key.to_sym] if key
      end

      def use(context_or_options)
        if context_or_options
          case context_or_options
          when Symbol  
            if opts = context[context_or_options]
              @options = opts.dup
            end
          when Hash
            @options = context_or_options
          end
        else
          @options = nil
        end
        self
      end

      NO_MODEL = Object.new
      def NO_MODEL.send(*args)
        self
      end
      def NO_MODEL.[](*args)
        self
      end

      def filter(hash = {}, model = NO_MODEL, &block)
        filter_data(model, hash, options, &block) if hash
      end

      def options
        @options || context[default_context_key] || {}
      end

      private

      def filter_data(model, data, options = {}, &block)
        model = NO_MODEL if model == data
        only = options[:only].collect { |o| o.to_s } if options[:only]
        except = (options[:except] || []).collect { |e| e.to_s }

        include =
          case options[:include]
          when Array
            options[:include].collect { |i| i.to_s }
          when Hash
            Hash[options[:include].collect {|k,v| [k.to_s, v]}]
          else
            []
          end
        if model != NO_MODEL
          methods = (options[:methods] || []).collect { |e| e.to_s }
          methods.each do |m|
            data[m] = model.send(m.to_sym)
          end

          include_methods = include.is_a?(Array) ? include : include.keys 
          include_methods.each do |m|
            unless data.include?(m)
              raise "no block given to calculate the attributes from model" unless block
              models_or_model = model.send(m)
              if models_or_model.respond_to?(:collect)
                data[m] = models_or_model.collect { |i| block.call(i) }
              else
                data[m]= block.call(model.send(m))
              end
            end
          end
        end
        methods ||= []

        result = {}
        data.each do |k,v|
          case v
          when Hash
            if include.include?(k.to_s)
              case include
              when Array
                result[k.to_s] = filter_data(model.send(k), v, &block)
              when Hash
                result[k.to_s] = filter_data(model.send(k), v, include[k.to_s], &block)
              end
            end
          when Array
            if include.include?(k.to_s)
              models = model.send(k)
              j = -1
              case include
              when Array
                result[k.to_s] = v.collect do |i| 
                  j += 1
                  if i.is_a?(Array) || i.is_a?(Hash)
                    filter_data(models[j], i, &block)
                  else
                    i
                  end
                end
              when Hash
                opts = include[k]
                result[k.to_s] = v.collect { |i| j += 1; filter_data(models[j], i, opts, &block) }
              end
            end
          else
            if methods.include?(k.to_s) || (only && only.include?(k.to_s)) || (only.nil? && !except.include?(k.to_s))
              if @map && ser = @map[v.class.to_s]
                result[k.to_s] = ser.call(v)
              else
                result[k.to_s] = v
              end<commit_msg>handle Hash + Array correctly<commit_after>module Ixtlan
  module Babel
    class HashFilter

      def initialize(context_or_options = nil)
        use(context_or_options)
      end
      
      private

      def context
        @context ||= {}
      end

      public

      def add_custom_serializers(map)
        @map = map
      end

      def default_context_key(default = nil)
        @default = default if default
        @default
      end

      def []=(key, options)
        context[key.to_sym] = options if key
      end

      def [](key)
        context[key.to_sym] if key
      end

      def use(context_or_options)
        if context_or_options
          case context_or_options
          when Symbol  
            if opts = context[context_or_options]
              @options = opts.dup
            end
          when Hash
            @options = context_or_options
          end
        else
          @options = nil
        end
        self
      end

      NO_MODEL = Object.new
      def NO_MODEL.send(*args)
        self
      end
      def NO_MODEL.[](*args)
        self
      end

      def filter(hash = {}, model = NO_MODEL, &block)
        filter_data(model, hash, options, &block) if hash
      end

      def options
        @options || context[default_context_key] || {}
      end

      private

      def filter_data(model, data, options = {}, &block)
        model = NO_MODEL if model == data
        only = options[:only].collect { |o| o.to_s } if options[:only]
        except = (options[:except] || []).collect { |e| e.to_s }

        include =
          case options[:include]
          when Array
            options[:include].collect { |i| i.to_s }
          when Hash
            Hash[options[:include].collect {|k,v| [k.to_s, v]}]
          else
            []
          end
        if model != NO_MODEL
          methods = (options[:methods] || []).collect { |e| e.to_s }
          methods.each do |m|
            data[m] = model.send(m.to_sym)
          end

          include_methods = include.is_a?(Array) ? include : include.keys 
          include_methods.each do |m|
            unless data.include?(m)
              raise "no block given to calculate the attributes from model" unless block
              models_or_model = model.send(m)
              if models_or_model.is_a?(Array) && models_or_model.first.is_a?(String)
                data[m] = models_or_model
              elsif models_or_model.respond_to?(:collect)
                data[m] = models_or_model.collect { |i| block.call(i) }
              else
                data[m]= block.call(model.send(m))
              end
            end
          end
        end
        methods ||= []

        result = {}
        data.each do |k,v|
          case v
          when Hash
            if include.include?(k.to_s)
              case include
              when Array
                result[k.to_s] = filter_data(model.send(k), v, &block)
              when Hash
                result[k.to_s] = filter_data(model.send(k), v, include[k.to_s], &block)
              end
            end
          when Array
            if include.include?(k.to_s)
              models = model.send(k)
              j = -1
              case include
              when Array
                result[k.to_s] = v.collect do |i| 
                  j += 1
                  if i.is_a?(Array) || i.is_a?(Hash)
                    filter_data(models[j], i, &block)
                  else
                    i
                  end
                end
              when Hash
                opts = include[k.to_s]
                result[k.to_s] = v.collect do |i| 
                  j += 1
                  ndata = i.is_a?(Hash)? i : block.call(i)
                  filter_data(models[j], ndata, opts, &block)
                end
              end
            end
          else
            if methods.include?(k.to_s) || (only && only.include?(k.to_s)) || (only.nil? && !except.include?(k.to_s))
              if @map && ser = @map[v.class.to_s]
                result[k.to_s] = ser.call(v)
              else
                result[k.to_s] = v
              end
{'fraction_non_alphanumeric': 0.09986285999251963, 'fraction_numerical': 0.0007480364044383493, 'mean_word_length': 1.2835183603757472, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '644614', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2311, 'n_words': 559}