<commit_before>#include "common.h"

/* 
  "Murmur"hash provided by Austin, <EMAIL>
#define MURMUR_POSTMIX
*/

uint32_t murmur_hash(char *key, size_t length)
{
  const unsigned int m= 0x7fd652ad;
  const int r= 16;

  uint32_t h= 0xdeadbeef;

  while(length >= 4)
  {
    h += *(unsigned int *)key;
    h *= m;
    h ^= h >> r;

    key += 4;
    length -= 4;
  }

  switch(length)
  {
  case 3:
    h += key[2] << 16;
  case 2:
    h += key[1] << 8;
  case 1:
  default:
    h += key[0];
    h *= m;
    h ^= h >> r;
  };

#ifdef MURMUR_POSTMIX

  h *= m;
  h ^= h >> 10;
  h *= m;
  h ^= h >> 17;

#endif

  return h;
}
<commit_msg>Update on murmur<commit_after>#include "common.h"

/* 
  "Murmur"hash provided by Austin, <EMAIL>
*/

#define MIX(h,k,m) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; }

uint32_t murmur_hash(char *key, size_t length)
{
  const uint32_t m= 0x5bd1e995;
  const int r= 16;
  uint32_t h= length * m;

  while(length >= 4)
  {
    uint32_t k = *(uint32_t*)key;
    MIX(h,k,m);

    key += 4;
    length -= 4;
  }

  if (length)
  {
    uint32_t k= 0;

    switch(length)
    {
    case 3: k += key[2] << 16;
    case 2: k += key[1] << 8;
    case 1: k += key[0];
    };
    MIX(h,k,m);
  }

  h *= m;
  h ^= h >> 10;
  h *= m;
  h ^= h >> 17;

  return h;
}

{'fraction_non_alphanumeric': 0.17561683599419448, 'fraction_numerical': 0.04716981132075472, 'mean_word_length': 1.7635270541082164, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4379788', 'n_tokens_mistral': 681, 'n_tokens_neox': 602, 'n_words': 159}