<commit_before># encoding: UTF-8

# require 'oj'
require 'yajl'

Oj.default_options = {
  mode: :compat
} if defined?(Oj)

module Spontaneous
  module JSON
    module OjParser
      def parse(json_string)
        ::Oj.load(json_string, symbol_keys: true)
      rescue
        nil
      end

      def encode(object)
        ::Oj.dump(object)
      end
    end

    if defined?(Yajl)
      module YajlParser
        def parser
          Yajl::Parser.new(:symbolize_keys => true)
        end
        def encoder
          Yajl::Encoder.new
        end

        def parse(json_string)
          parser.parse(json_string)
        rescue
          nil
        end

        def encode(object)
          encoder.encode(object)
        end
      end
    end

    # extend OjParser
    extend YajlParser

    def parse_json(json_string)
      Spontaneous::JSON.parse(json_string)
    end
    alias_method :decode_json, :parse_json

    def encode_json(object)
      Spontaneous::JSON.encode(object)
    end
  end
end<commit_msg>Keep support for both Oj & Yajl, just in case<commit_after># encoding: UTF-8

require 'yajl'

Oj.default_options = { mode: :compat } if defined?(Oj)

module Spontaneous
  module JSON
    if defined?(::Oj)
      module OjParser
        def parse(json_string)
          ::Oj.load(json_string, symbol_keys: true)
        rescue
          nil
        end

        def encode(object)
          ::Oj.dump(object)
        end
      end
    end

    if defined?(::Yajl)
      module YajlParser
        def parser
          Yajl::Parser.new(:symbolize_keys => true)
        end
        def encoder
          Yajl::Encoder.new
        end

        def parse(json_string)
          parser.parse(json_string)
        rescue
          nil
        end

        def encode(object)
          encoder.encode(object)
        end
      end
    end

    extend const_get ["OjParser", "YajlParser"].detect { |m| const_defined?(m) }

    def parse_json(json_string)
      Spontaneous::JSON.parse(json_string)
    end
    alias_method :decode_json, :parse_json

    def encode_json(object)
      Spontaneous::JSON.encode(object)
    end
  end
end
{'fraction_non_alphanumeric': 0.08216619981325864, 'fraction_numerical': 0.0009337068160597573, 'mean_word_length': 1.9275956284153006, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2096387', 'n_tokens_mistral': 693, 'n_tokens_neox': 682, 'n_words': 156}