<commit_before>module ZoomHub.Storage.File
  ( getById
  , getByURL
  , create
  )
  where

import           Prelude                                  hiding (fromInteger)

import           Control.Concurrent.STM                   (TVar, atomically,
                                                           modifyTVar, readTVar)
import           Control.Exception                        (tryJust)
import           Control.Monad                            (guard)
import           Data.Aeson                               (decode)
import           Data.Aeson.Encode.Pretty                 (encodePretty')
import qualified Data.ByteString.Lazy                     as BL
import           System.AtomicWrite.Writer.LazyByteString (atomicWriteFile)
import           System.Directory                         (doesFileExist)
import           System.FilePath.Posix                    ((<.>), (</>))
import           System.IO.Error                          (isDoesNotExistError)
import           System.Posix.Files                       (createLink)

import           ZoomHub.Config                           (Config)
import qualified ZoomHub.Config                           as Config
import           ZoomHub.Pipeline                         (process)
import           ZoomHub.Storage.Internal.File            (hashURL, toFilename)
import           ZoomHub.Types.Internal.Content           (Content, contentId,
                                                           fromURL,
                                                           prettyEncodeConfig)
import           ZoomHub.Types.Internal.ContentId         (ContentId,
                                                           fromInteger, unId)


-- TODO: Introduce `ContentURL` `newtype`:
type URL = String

-- Public API
create :: Config -> String -> IO Content
create config contentURL = do
  newContentId <- createNewId config
  let newContent = fromURL newContentId contentURL<commit_msg>Use `URL` type alias<commit_after>module ZoomHub.Storage.File
  ( getById
  , getByURL
  , create
  )
  where

import           Prelude                                  hiding (fromInteger)

import           Control.Concurrent.STM                   (TVar, atomically,
                                                           modifyTVar, readTVar)
import           Control.Exception                        (tryJust)
import           Control.Monad                            (guard)
import           Data.Aeson                               (decode)
import           Data.Aeson.Encode.Pretty                 (encodePretty')
import qualified Data.ByteString.Lazy                     as BL
import           System.AtomicWrite.Writer.LazyByteString (atomicWriteFile)
import           System.Directory                         (doesFileExist)
import           System.FilePath.Posix                    ((<.>), (</>))
import           System.IO.Error                          (isDoesNotExistError)
import           System.Posix.Files                       (createLink)

import           ZoomHub.Config                           (Config)
import qualified ZoomHub.Config                           as Config
import           ZoomHub.Pipeline                         (process)
import           ZoomHub.Storage.Internal.File            (hashURL, toFilename)
import           ZoomHub.Types.Internal.Content           (Content, contentId,
                                                           fromURL,
                                                           prettyEncodeConfig)
import           ZoomHub.Types.Internal.ContentId         (ContentId,
                                                           fromInteger, unId)


-- TODO: Introduce `ContentURL` `newtype`:
type URL = String

-- Public API
create :: Config -> URL -> IO Content
create config contentURL = do
  newContentId <- createNewId config
  let newContent = fromURL newContentId contentURL
{'fraction_non_alphanumeric': 0.05945668887749872, 'fraction_numerical': 0.0, 'mean_word_length': 1.0849358974358974, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1179184', 'n_tokens_mistral': 840, 'n_tokens_neox': 799, 'n_words': 200}