<commit_before>module Process (process) where
import Control.Exception (evaluate, throw)
import System.FilePath.Glob
import Data.Aeson (decode)
import Data.Aeson.Types
import Data.Text (pack, unpack, Text)
import Data.ByteString.Lazy as B (readFile, ByteString)
-- import Data.HashMap as M (lookup, Map)

-- | Do actual processing
-- >>> process "Name" "../test_data/"
-- "ok"
process :: String -> String -> IO ()
process field folder = do
    -- putStrLn ("Processing field '" ++ field ++ "' in " ++ folder)
    files <- globDir1 (compile "*.json") folder
    values <- mapM (processFile field) files
    let notEmpty = [x | x <- values, x /= ""]
    -- mapM_ putStrLn $ notEmpty
    print "ok"

processFile :: String -> String -> IO String
processFile field filename = do
    contents <- B.readFile filename
    let json = parseJson contents
    let value = getFieldValue json field
    evaluate value

parseJson :: ByteString -> Object
parseJson src = case (decode src :: Maybe Object) of
    Just v  -> v
    Nothing -> error "Malformed JSON"

getFieldValue :: Object -> String -> String<commit_msg>Sorted frequencies<commit_after>module Process (process) where
import Control.Exception (evaluate, throw)
import System.FilePath.Glob
import Data.Aeson (decode)
import Data.Aeson.Types
import Data.List (sortBy)
import Data.Ord (comparing)
import Data.Text (Text, pack, unpack)
import Data.ByteString.Lazy as B (ByteString, readFile)
import qualified Data.Map as M (Map, fromListWith, toList)

-- | Do actual processing
-- >>> process "Name" "../test_data/"
-- "ok"
process :: String -> String -> IO ()
process field folder = do
    files <- globDir1 (compile "*.json") folder
    values <- mapM (processFile field) files
    let notEmpty = [x | x <- values, x /= ""]
    let result = sortedFrequencies notEmpty
    -- flip mapM_ result $ \(k, v) -> putStrLn (k ++ " " ++ show v)
    print "ok"

sortedFrequencies :: [String] -> [(String, Int)]
sortedFrequencies m = reverse $ sortBy (comparing snd) freq
  where freq = M.toList $ frequencies m

frequencies :: [String] -> M.Map String Int
frequencies items = M.fromListWith (+) (map (\i -> (i, 1)) items)

processFile :: String -> String -> IO String
processFile field filename = do
    contents <- B.readFile filename
    let json = parseJson contents
    let value = getFieldValue json field
    evaluate value

parseJson :: ByteString -> Object
parseJson src = case (decode src :: Maybe Object) of
    Just v  -> v
    Nothing -> error "Malformed JSON"

getFieldValue :: Object -> String -> String
{'fraction_non_alphanumeric': 0.12534273403838622, 'fraction_numerical': 0.0011750881316098707, 'mean_word_length': 3.4417391304347826, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2205623', 'n_tokens_mistral': 767, 'n_tokens_neox': 748, 'n_words': 279}