<commit_before>{-# LANGUAGE QuasiQuotes                #-}
{-# LANGUAGE TemplateHaskell            #-}
{-# LANGUAGE TypeFamilies               #-}
{-# LANGUAGE GeneralizedNewtypeDeriving #-}
{-# LANGUAGE FlexibleContexts           #-}
module Model where

import Yesod.Persist
import Yesod.Goodies.Markdown
import Data.List (nub)
import Data.Time (UTCTime)

import qualified Data.Text as T

-- | Generate data base instances for post meta-data
share2 mkPersist (mkMigrate "migratePosts") [persist|
    Post
        slug  T.Text  Eq      Update
        date  UTCTime    Desc
        title T.Text          Update
        descr Markdown        Update

        UniquePost slug

    Tag
        post PostId Eq
        name T.Text    Asc

        UniqueTag post name
    |]

data Document = Document
    { post :: Post
    , tags :: [Tag]
    }

instance Eq Document where
    (Document p1 _) == (Document p2 _) = postSlug p1 == postSlug p2

data Collection = Collection
    { name      :: T.Text
    , documents :: [Document]
    }

collect :: [Document] -> T.Text -> (Document -> Bool) -> Collection
collect docs n p = Collection n (filter p docs)

collectByTagName :: [Document] -> [Collection]
collectByTagName docs = map (\tag -> collect docs tag (hasTag tag)) . nub . map tagName $ concatMap tags docs

hasTag :: T.Text -> Document -> Bool
hasTag t d = t `elem` (map tagName $ tags d)<commit_msg>add User/Ident defs for normal auth<commit_after>{-# LANGUAGE QuasiQuotes                #-}
{-# LANGUAGE TemplateHaskell            #-}
{-# LANGUAGE TypeFamilies               #-}
{-# LANGUAGE GeneralizedNewtypeDeriving #-}
{-# LANGUAGE FlexibleContexts           #-}
module Model where

import Yesod.Persist
import Yesod.Goodies.Markdown
import Data.List (nub)
import Data.Time (UTCTime)

import qualified Data.Text as T

-- | Generate data base instances for post meta-data
share2 mkPersist (mkMigrate "migratePosts") [persist|
    Post
        slug  T.Text  Eq      Update
        date  UTCTime    Desc
        title T.Text          Update
        descr Markdown        Update

        UniquePost slug

    Tag
        post PostId Eq
        name T.Text    Asc

        UniqueTag post name

    User
        username      T.Text Maybe Update Asc
        email         T.Text Maybe Update
        admin         Bool default=false Eq Update

    Ident
        ident T.Text Asc
        user  UserId Eq

        UniqueIdent ident
    |]

data Document = Document
    { post :: Post
    , tags :: [Tag]
    }

instance Eq Document where
    (Document p1 _) == (Document p2 _) = postSlug p1 == postSlug p2

data Collection = Collection
    { name      :: T.Text
    , documents :: [Document]
    }

collect :: [Document] -> T.Text -> (Document -> Bool) -> Collection
collect docs n p = Collection n (filter p docs)

collectByTagName :: [Document] -> [Collection]
collectByTagName docs = map (\tag -> collect docs tag (hasTag tag)) . nub . map tagName $ concatMap tags docs

hasTag :: T.Text -> Document -> Bool
hasTag t d = t `elem` (map tagName $ tags d)
{'fraction_non_alphanumeric': 0.09219160104986876, 'fraction_numerical': 0.0032808398950131233, 'mean_word_length': 2.179353493222106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '577876', 'n_tokens_mistral': 958, 'n_tokens_neox': 915, 'n_words': 306}