<commit_before>
module FFI_Yajl
  class ParseError < StandardError; end
  class Parser
    attr_accessor :stack, :key_stack, :key, :finished

    attr_accessor :opts

    #
    # stack used to build up our complex object
    #
    def stack
      @stack ||= Array.new
    end

    #
    # stack to keep track of keys as we create nested hashes
    #
    def key_stack
      @key_stack ||= Array.new
    end

    def self.parse(obj, *args)
      new(*args).parse(obj)
    end

    def initialize(opts = {})
      @opts = opts ? opts.dup : {}
      # JSON gem uses 'symbolize_names' and ruby-yajl supports this as well
      @opts[:symbolize_keys] = true if @opts[:symbolize_names]
    end

    def parse(str)
      # initialization that we can do in pure ruby
      yajl_opts = {}<commit_msg>fix #15 by requiring string or IO<commit_after>
module FFI_Yajl
  class ParseError < StandardError; end
  class Parser
    attr_accessor :stack, :key_stack, :key, :finished

    attr_accessor :opts

    #
    # stack used to build up our complex object
    #
    def stack
      @stack ||= Array.new
    end

    #
    # stack to keep track of keys as we create nested hashes
    #
    def key_stack
      @key_stack ||= Array.new
    end

    def self.parse(obj, *args)
      new(*args).parse(obj)
    end

    def initialize(opts = {})
      @opts = opts ? opts.dup : {}
      # JSON gem uses 'symbolize_names' and ruby-yajl supports this as well
      @opts[:symbolize_keys] = true if @opts[:symbolize_names]
    end

    def parse(str)
      raise FFI_Yajl::ParseError, "input must be a string or IO" unless str.is_a?(String) || str.respond_to?(:read)

      # initialization that we can do in pure ruby
      yajl_opts = {}
{'fraction_non_alphanumeric': 0.0924956369982548, 'fraction_numerical': 0.0011634671320535194, 'mean_word_length': 2.4888438133874238, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3705376', 'n_tokens_mistral': 567, 'n_tokens_neox': 551, 'n_words': 180}