<commit_before>-- This is a naive, unoptimized way of checking two
-- JSON objects for equality. It uses python's json
-- module to load and re-dump the json object while
-- sorting its keys, and then calculates the object
-- hash from it.
--
CREATE OR REPLACE FUNCTION json_hash( a json ) RETURNS INTEGER AS $$
  import json
  hsh = hash(json.dumps(json.loads(a), sort_keys=True, separators=(',', ':')))
  return hsh & (2**32-1) - 2**31
$$ LANGUAGE plpythonu STRICT IMMUTABLE;

CREATE OR REPLACE FUNCTION json_eq( a json, b json ) RETURNS BOOLEAN AS $$
  SELECT json_hash(a) = json_hash(b);
$$ LANGUAGE SQL STRICT IMMUTABLE;

CREATE OPERATOR = (<commit_msg>Use a 32-bit FNV hash instead of coercing python's one<commit_after>-- This is a naive, unoptimized way of checking two
-- JSON objects for equality. It uses python's json
-- module to load and re-dump the json object while
-- sorting its keys, and then calculates the object
-- hash from it.
--
CREATE OR REPLACE FUNCTION json_hash( a json ) RETURNS INTEGER AS $$
  import json

  # http://www.isthe.com/chongo/tech/comp/fnv/#FNV-1a
  def fnv1a(str):
    hval = 2166136261
    for s in str:
      hval = hval ^ ord(s)
      hval = (hval * 16777619) % (2**32)
    return hval

  hsh = fnv1a(json.dumps(json.loads(a), sort_keys=True, separators=(',', ':')))
  return hsh - 2**31

$$ LANGUAGE plpythonu STRICT IMMUTABLE;

CREATE OR REPLACE FUNCTION json_eq( a json, b json ) RETURNS BOOLEAN AS $$
  SELECT json_hash(a) = json_hash(b);
$$ LANGUAGE SQL STRICT IMMUTABLE;

CREATE OPERATOR = (
{'fraction_non_alphanumeric': 0.1145631067961165, 'fraction_numerical': 0.02330097087378641, 'mean_word_length': 3.6566265060240966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3575773', 'n_tokens_mistral': 579, 'n_tokens_neox': 539, 'n_words': 192}