<commit_before>{-# LANGUAGE BangPatterns #-}
module Hash (mainhash, sumhash) where

import qualified Data.ByteString.Char8 as B
import Data.Bits
import Data.Char (ord)

sumhash :: B.ByteString -> Int
sumhash = B.foldl' (\a b -> a + (fromIntegral (ord b))) 0

perlhash :: B.ByteString -> Int
perlhash !s = let v0 = B.foldl' pcombine 0 s
                  v1 = v0 `seq` v0 + (v0 `shiftL` 3)
                  v2 = v1 `seq` v1 `xor` (v1 `shiftR` 11)
              in  v2 + (v2 `shiftL` 15)
{-# INLINE perlhash #-}

mainhash :: B.ByteString -> Int
mainhash !s = B.foldl' (\i c -> ((i `shiftL` 5) + i) `xor` (ord c)) 5381 s
{-# INLINE mainhash #-}

pcombine :: Int -> Char -> Int
pcombine !val !c = let v1 = val + (fromIntegral (ord c))
                       v2 = v1 + v1 `shiftL` 10
                   in  v2 `xor` (v2 `shiftR` 6)
{-# INLINE pcombine #-}

<commit_msg>Some minor tinkering for future Data.IntMap-based hash experiment.<commit_after>{-# LANGUAGE BangPatterns #-}
module Hash (mainhash, sumhash) where

import qualified Data.ByteString.Char8 as B
import Data.Bits
import Data.Char (ord)

data Hashed a = Hashed { hshDat :: !a, hshVal :: !Int } deriving (Eq,Show)

toHashed v = Hashed v (hashVal v)

class Hashable a where
   hashVal :: a -> Int

instance Hashable (Hashed a) where
    hashVal = hshVal

instance Hashable B.ByteString where
    hashVal = mainhash

sumhash :: B.ByteString -> Int
sumhash = B.foldl' (\a b -> a + (fromIntegral (ord b))) 0

perlhash :: B.ByteString -> Int
perlhash !s = let v0 = B.foldl' pcombine 0 s
                  v1 = v0 `seq` v0 + (v0 `shiftL` 3)
                  v2 = v1 `seq` v1 `xor` (v1 `shiftR` 11)
              in  v2 + (v2 `shiftL` 15)
{-# INLINE perlhash #-}

mainhash :: B.ByteString -> Int
mainhash !s = B.foldl' (\i c -> ((i `shiftL` 5) + i) `xor` (ord c)) 5381 s
{-# INLINE mainhash #-}

pcombine :: Int -> Char -> Int
pcombine !val !c = let v1 = val + (fromIntegral (ord c))
                       v2 = v1 + v1 `shiftL` 10
                   in  v2 `xor` (v2 `shiftR` 6)
{-# INLINE pcombine #-}


{'fraction_non_alphanumeric': 0.14570179698882954, 'fraction_numerical': 0.03205439533754249, 'mean_word_length': 2.3715220949263505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4687027', 'n_tokens_mistral': 824, 'n_tokens_neox': 775, 'n_words': 259}