<commit_before>module HEPModel where

import Data.Map (Map)
import qualified Data.Map as M
import Data.HMCMC.Dist


type ProcessName = String
type RegionName = String

-- simple histogram type
type Hist a = [a]

sumHists :: Num a => [Hist a] -> Hist a
sumHists = foldr1 (zipWith (+))

-- a sample's normalization is consistent across regions
type Process = Map RegionName (Hist Double)

type Region = Map ProcessName (Hist Double)

-- a variation alters a model in some particular way.
type Variation = Double -> Model -> Model

type Model = Map RegionName Region

type Dataset = Map RegionName (Hist Int)

totalPrediction :: Model -> Process
totalPrediction = fmap (sumHists . M.elems)


-- the poisson likelihood of a model given the input data
modelPoissonLH :: Dataset -> Model -> Double
modelPoissonLH ds m = product . M.elems $ M.intersectionWith poissonProbs (totalPrediction m) ds<commit_msg>not sure what I was doing; picking this back up.<commit_after>module HEPModel where

import Data.Map (Map)
import qualified Data.Map as M
import Data.HMCMC.Dist


type ProcessName = String
type RegionName = String

-- simple histogram type
type Hist a = [a]

sumHists :: Num a => [Hist a] -> Hist a
sumHists = foldr1 (zipWith (+))

-- a process's normalization is consistent across regions
type Process = Map RegionName (Hist Double)

type Region = Map ProcessName (Hist Double)


data SystematicVariation =
    | Normalization Map RegionName Region -> 

type Dataset = Map RegionName (Hist Int)

-- HEPModel a takes a map of regions, a set of parameters
type HEPModel a = Map RegionName Region -> a -> Double

-- a ModelParameter alters a model in some particular way and has a prior
-- distribution in its parameter of type a
data ModelParameter a = ModelParameter {
    alter :: Model -> a -> Model,
    priorProb :: a -> Double
}


totalPrediction :: Model -> Process
totalPrediction = fmap (sumHists . M.elems) . regions


-- the poisson likelihood of a model given the input data
modelPoissonLH :: Dataset -> Model -> Double
modelPoissonLH ds m = product . M.elems $ M.intersectionWith poissonProbs (totalPrediction m) ds
{'fraction_non_alphanumeric': 0.08180535966149506, 'fraction_numerical': 0.0009402914903620122, 'mean_word_length': 3.858447488584475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3540693', 'n_tokens_mistral': 620, 'n_tokens_neox': 597, 'n_words': 268}