<commit_before>{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE RecordWildCards #-}

module Types
    ( ShkoloUrl
    , ArticleMetaData(..)
    , parseTime'
    ) where

import Data.Time
import qualified Data.Text.Lazy as LT
import Data.Aeson
import GHC.Generics
import Data.Csv
import Control.Monad

type ShkoloUrl = String

data ArticleMetaData = ArticleMetaData
    { _title :: LT.Text
    , _publishedAt :: UTCTime
    , _link :: LT.Text
    } deriving (Show, Generic)

instance ToJSON ArticleMetaData

instance FromRecord ArticleMetaData where
  parseRecord v
    | length v == 3 =
        ArticleMetaData
          <$>  v .! 0
          <*> (v .! 1 >>= parseTime')
          <*>  v .! 2
    | otherwise = mzero


instance ToRecord ArticleMetaData where
  toRecord ArticleMetaData{..} = record [
    toField _title,
    toField (unparseTime _publishedAt),
    toField _link
    ]

parseTime' :: (Monad m, ParseTime t) => LT.Text -> m t
parseTime' = parseTimeM False defaultTimeLocale fmt . LT.unpack

unparseTime :: UTCTime -> LT.Text
unparseTime = LT.pack . formatTime defaultTimeLocale fmt<commit_msg>Switch from UTCTime to ZonedTime<commit_after>{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE RecordWildCards #-}

module Types
    ( ShkoloUrl
    , ArticleMetaData(..)
    , parseTime'
    ) where

import Data.Time
import qualified Data.Text.Lazy as LT
import Data.Aeson
import GHC.Generics
import Data.Csv
import Control.Monad

type ShkoloUrl = String

data ArticleMetaData = ArticleMetaData
    { _title :: LT.Text
    , _publishedAt :: ZonedTime
    , _link :: LT.Text
    } deriving (Show, Generic)

instance ToJSON ArticleMetaData

instance FromRecord ArticleMetaData where
  parseRecord v
    | length v == 3 =
        ArticleMetaData
          <$>  v .! 0
          <*> (v .! 1 >>= parseTime')
          <*>  v .! 2
    | otherwise = mzero


instance ToRecord ArticleMetaData where
  toRecord ArticleMetaData{..} = record [
    toField _title,
    toField (unparseTime _publishedAt),
    toField _link
    ]

parseTime' :: (Monad m, ParseTime t) => LT.Text -> m t
parseTime' = parseTimeM False defaultTimeLocale fmt . LT.unpack

unparseTime :: ZonedTime -> LT.Text
unparseTime = LT.pack . formatTime defaultTimeLocale fmt
{'fraction_non_alphanumeric': 0.09986973512809379, 'fraction_numerical': 0.0034737299174989146, 'mean_word_length': 3.0492091388400704, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1245654', 'n_tokens_mistral': 764, 'n_tokens_neox': 746, 'n_words': 205}