<commit_before>#! /usr/bin/env ruby

require 'json'

module JSON

  class PP_KEY < String

    def eql?(*)
      false
    end

  end

  class PP_OBJECT < Hash

    def []=(k, v)
      super(PP_KEY.new(k), v)
    end

  end

  def self.pp(source, opt = {})
    pretty_generate(parse(source, opt.merge(object_class: PP_OBJECT)))
  rescue ParserError => err
    warn "#{err.class}: #{err}"
    source
  end

end

if $0 == __FILE__
  if ARGV.include?('-h') || ARGV.include?('--help')
    abort "Usage: #{$0} [-h|--help] [-l|--lines] [SOURCE...]"
  end

  args = ARGV.delete('--lines') || ARGV.delete('-l') ? ARGF : [ARGF.read]
  args.each { |source| puts JSON.pp(source) }
end<commit_msg>json_pp.rb: Added option for JSON canonicalization.<commit_after>#! /usr/bin/env ruby

require 'json'

module JSON

  extend self

  PP_OPT = :_json_pp_canonical

  class PP_KEY < String

    def eql?(*)
      false
    end

  end

  class PP_OBJECT < Hash

    def []=(k, v)
      super(PP_KEY.new(k), v)
    end

  end

  def pp(source, opt = {})
    obj = parse(source, opt.merge(object_class: PP_OBJECT))
    pretty_generate(opt[PP_OPT] ? send(PP_OPT, obj) : obj)
  rescue ParserError => err
    warn "#{err.class}: #{err}"
    source
  end

  def pc(source, opt = {})
    pp(source, opt.merge(PP_OPT => true))
  end

  private

  def _json_pp_canonical(obj)
    case obj
      when Hash
        obj.class.new.tap { |res|
          obj.keys.sort.each { |k| res[k] = send(PP_OPT, obj[k]) } }
      when Array
        obj.map { |v| send(PP_OPT, v) }.sort_by(&:to_s)
      else
        obj
    end
  end

end

if $0 == __FILE__
  if ARGV.include?('-h') || ARGV.include?('--help')
    abort "Usage: #{$0} [-h|--help] [-l|--lines] [-c|--canon] [SOURCE...]"
  end

  meth = ARGV.delete('--canon') || ARGV.delete('-c') ? :pc : :pp
  args = ARGV.delete('--lines') || ARGV.delete('-l') ? ARGF : [ARGF.read]

  args.each { |source| puts JSON.send(meth, source) }
end
{'fraction_non_alphanumeric': 0.1789203084832905, 'fraction_numerical': 0.002056555269922879, 'mean_word_length': 2.583793738489871, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6360032', 'n_tokens_mistral': 788, 'n_tokens_neox': 755, 'n_words': 178}