<commit_before># JsonFilter

module JsonFilter
  class Crawler
    class << self
      def do(data, key_string)
        keys = key_string.split(/(?<!\\)\./)
        cursor = data
        keys.each_with_index do |key, index|
          index = nil
          if key =~ /.+\[\d+\]$/
            index = /\[(\d+)\]$/.match(key)[1].to_i
            key = /(.+)\[\d+\]$/.match(key)[1]
          end
          if cursor.has_key?(key)
            cursor = cursor[key]
          else
            yield({:error_message => "Cannot crawl key string #{key_string}"})
            return nil
          end
          if index != nil && cursor.class.name == 'Array'
            cursor = cursor[index]
          end
        end

        cursor
      end
    end
  end
end<commit_msg>-Added test for non-object while crawling<commit_after># JsonFilter

module JsonFilter
  class Crawler
    class << self
      def do(data, key_string)
        keys = key_string.split(/(?<!\\)\./)
        cursor = data
        keys.each_with_index do |key, index|
          index = nil
          if key =~ /.+\[\d+\]$/
            index = /\[(\d+)\]$/.match(key)[1].to_i
            key = /(.+)\[\d+\]$/.match(key)[1]
          end
          if cursor.class.name != 'Hash'
            yield({:error_message => "Non-object while crawling key #{key_string}"})
            return nil
          elsif cursor.has_key?(key)
            cursor = cursor[key]
          else
            yield({:error_message => "Cannot crawl key string #{key_string}"})
            return nil
          end
          if index != nil && cursor.class.name == 'Array'
            cursor = cursor[index]
          end
        end

        cursor
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.14344262295081966, 'fraction_numerical': 0.00234192037470726, 'mean_word_length': 1.462536023054755, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5781273', 'n_tokens_mistral': 516, 'n_tokens_neox': 508, 'n_words': 131}