<commit_before>-- general utilities not really connected with the main focus of this project

module Utils where

import Text.Printf

data Hidden a = Hidden { unhide :: a }

class TypeTag a where
    getTypeTag :: a -> String -- get 'name' of a type

instance (TypeTag a) => Show (Hidden a) where
    show (Hidden foo) = (printf "Hidden {<%s>}" (getTypeTag foo) :: String)

instance Eq (Hidden a) where
    _ == _ = True<commit_msg>add shuffle, ofType to  Utils<commit_after>-- general utilities not really connected with the main focus of this project

module Utils where

import Text.Printf
import System.Random.MWC
import Data.List (sortBy)
import Data.Ord (comparing)
import Control.Monad (replicateM)


data Hidden a = Hidden { unhide :: a }

class TypeTag a where
    getTypeTag :: a -> String -- get 'name' of a type

instance (TypeTag a) => Show (Hidden a) where
    show (Hidden foo) = (printf "Hidden {<%s>}" (getTypeTag foo) :: String)

instance Eq (Hidden a) where
    _ == _ = True

shuffle :: [a] -> IO [a]
shuffle lst = withSystemRandom . asGenST $ \ gen -> do
  ixs <- replicateM (length lst) (uniform gen)
  return $ map snd $ sortBy (comparing fst) $ (zip (ixs::[Int]) lst)
  
ofType :: a -> a -> a
ofType a _ = a
{'fraction_non_alphanumeric': 0.11697806661251016, 'fraction_numerical': 0.0, 'mean_word_length': 3.463768115942029, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '857015', 'n_tokens_mistral': 403, 'n_tokens_neox': 393, 'n_words': 162}