<commit_before>begin
  require 'oj'
rescue LoadError
  require 'symbolizer'
  require 'json'
end

class Freddy
  class Payload
    def self.parse(payload)
      return {} if payload == 'null'

      json_handler.parse(payload)
    end

    def self.dump(payload)
      json_handler.dump(payload)
    end

    def self.json_handler
      @_json_handler ||= defined?(Oj) ? OjAdapter : JsonAdapter
    end

    class OjAdapter
      def self.parse(payload)
        Oj.strict_load(payload, symbol_keys: true)
      end

      def self.dump(payload)
        Oj.dump(payload, mode: :compat, time_format: :xmlschema)
      end
    end

    class JsonAdapter
      def self.parse(payload)
        # MRI has :symbolize_keys, but JRuby does not. Not adding it at the
        # moment.
        Symbolizer.symbolize(JSON.parse(payload))
      end
<commit_msg>Limit timestamps precision when serializing objects with Oj<commit_after>begin
  require 'oj'
rescue LoadError
  require 'symbolizer'
  require 'json'
end

class Freddy
  class Payload
    def self.parse(payload)
      return {} if payload == 'null'

      json_handler.parse(payload)
    end

    def self.dump(payload)
      json_handler.dump(payload)
    end

    def self.json_handler
      @_json_handler ||= defined?(Oj) ? OjAdapter : JsonAdapter
    end

    class OjAdapter
      PARSE_OPTIONS = { symbol_keys: true }
      DUMP_OPTIONS = { mode: :compat, time_format: :xmlschema, second_precision: 6 }

      def self.parse(payload)
        Oj.strict_load(payload, PARSE_OPTIONS)
      end

      def self.dump(payload)
        Oj.dump(payload, DUMP_OPTIONS)
      end
    end

    class JsonAdapter
      def self.parse(payload)
        # MRI has :symbolize_keys, but JRuby does not. Not adding it at the
        # moment.
        Symbolizer.symbolize(JSON.parse(payload))
      end

{'fraction_non_alphanumeric': 0.08260869565217391, 'fraction_numerical': 0.0005434782608695652, 'mean_word_length': 2.574757281553398, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2603149', 'n_tokens_mistral': 595, 'n_tokens_neox': 573, 'n_words': 151}