<commit_before>#!/usr/bin/env nix-shell
#! nix-shell -p "haskellPackages.ghcWithPackages (pkgs: with pkgs; [mtl text])" -i runhaskell

-- This is a databrary.conf generator
--TODO commented out components of example.conf must be added to
--confContent
--TODO File needs to be in ByteString

{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE TemplateHaskell #-}

import System.Process (callProcess)
import System.Directory ( doesDirectoryExist
                        , createDirectoryIfMissing
                        , doesFileExist
                        , getCurrentDirectory)
import Data.ByteString (ByteString)
import qualified Data.ByteString as ByteString
import qualified Data.ByteString.Char8 (unlines)
import qualified Data.ByteString.Char8 as ByteString8
import Data.Monoid ((<>))

main :: IO ()
main = do
  projectRoot <- getCurrentDirectory  
  let databraryConf = "databrary.conf"
  createFileIfMissing databraryConf $ confContent (ByteString8.pack projectRoot)

confContent :: ByteString -> ByteString
confContent absPath = ByteString8.unlines 
  [ "##See example.conf for complete conf options."
  , "secret = \"bob\""
  , "port = 8000"
  , "ssl {"
  , "}"
  , "log {"
  , "  messages {"
  , "    file = \"stderr\""
  , "   rotate = 100"
  , "  }"
  , "  access {"
  , "   file = \"stdout\""
  , "   rotate = 100"
  , "  }"
  , "}"
  , "#comment out host, user = databrary , db = databrary, and pass when using nix-build"
  , "db {"
  , "  #host = \"localhost\""
  , "  sock = \"./databrary-nix-db/work/.s.PGSQL.5432\""
  , "  port = 5432"
  , "  #user = \"databrary\""
  , "  user = \"postgres\""
  , "  #pass = \"<PASSWORD>\""
  , "  #db = \"databrary\""
  , "  db = \"postgres\""
  , "}"
  , ""
  , "store {"
  , "  master = \"" <> absPath <> "/store\""
  , "  upload = \"" <> absPath <> "/upload\""
  , "  temp  = \"" <> absPath <> "/tmp\""
  , "  stage  = \"" <> absPath <> "/stage\""
  , "  cache  = \"" <> absPath <> "/cache\""
  , "  transcode {"
  , "  }"
  , "}"
  , ""
  , "solr {"
  , "  run = true"
  , "  host = \"localhost\""
  , "  port = 8983"
  , " home = \"" <> absPath <> "/solr\""
  , "  core = \"databrary_core\""
  , " log  = \"" <> absPath <> "/databrary_logs/solr_log\""
  , "}"
  , "static {"
  , " authorize = \"<EMAIL>\""
  , " assist = \"<EMAIL>\""
  , "}"
  , "ezid {"
  , "}"
  , "notification {"
  , "  filter = \"*\""
  , "  copy = \"<EMAIL>\""
  , "}"
  ]
      
createFileIfMissing :: FilePath -> ByteString -> IO ()
createFileIfMissing aFile content =
  doesFileExist aFile >>= \case
    True -> putStrLn ("Skipping " ++ show aFile ++ "(already exist)")
    False -> ByteString.writeFile aFile content<commit_msg>add trancode dir to confgen.hs<commit_after>#!/usr/bin/env nix-shell
#! nix-shell -p "haskellPackages.ghcWithPackages (pkgs: with pkgs; [mtl text])" -i runhaskell

-- This is a databrary.conf generator
--TODO commented out components of example.conf must be added to
--confContent
--TODO File needs to be in ByteString

{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE TemplateHaskell #-}

import System.Process (callProcess)
import System.Directory ( doesDirectoryExist
                        , createDirectoryIfMissing
                        , doesFileExist
                        , getCurrentDirectory)
import Data.ByteString (ByteString)
import qualified Data.ByteString as ByteString
import qualified Data.ByteString.Char8 (unlines)
import qualified Data.ByteString.Char8 as ByteString8
import Data.Monoid ((<>))

main :: IO ()
main = do
  projectRoot <- getCurrentDirectory  
  let databraryConf = "databrary.conf"
  createFileIfMissing databraryConf $ confContent (ByteString8.pack projectRoot)

confContent :: ByteString -> ByteString
confContent absPath = ByteString8.unlines 
  [ "##See example.conf for complete conf options."
  , "secret = \"bob\""
  , "port = 8000"
  , "ssl {"
  , "}"
  , "log {"
  , "  messages {"
  , "    file = \"stderr\""
  , "   rotate = 100"
  , "  }"
  , "  access {"
  , "   file = \"stdout\""
  , "   rotate = 100"
  , "  }"
  , "}"
  , "#comment out host, user = databrary , db = databrary, and pass when using nix-build"
  , "db {"
  , "  #host = \"localhost\""
  , "  sock = \"./databrary-nix-db/work/.s.PGSQL.5432\""
  , "  port = 5432"
  , "  #user = \"databrary\""
  , "  user = \"postgres\""
  , "  #pass = \"<PASSWORD>\""
  , "  #db = \"databrary\""
  , "  db = \"postgres\""
  , "}"
  , ""
  , "#comment out transcode dir when using nix-build"
  , "store {"
  , "  master = \"" <> absPath <> "/store\""
  , "  upload = \"" <> absPath <> "/upload\""
  , "  temp  = \"" <> absPath <> "/tmp\""
  , "  stage  = \"" <> absPath <> "/stage\""
  , "  cache  = \"" <> absPath <> "/cache\""
  , "  transcode {"
  , "    #dir = \"" <> absPath <> "/trans\""
  , "  }"
  , "}"
  , ""
  , "solr {"
  , "  run = true"
  , "  host = \"localhost\""
  , "  port = 8983"
  , " home = \"" <> absPath <> "/solr\""
  , "  core = \"databrary_core\""
  , " log  = \"" <> absPath <> "/databrary_logs/solr_log\""
  , "}"
  , "static {"
  , " authorize = \"<EMAIL>\""
  , " assist = \"<EMAIL>\""
  , "}"
  , "ezid {"
  , "}"
  , "notification {"
  , "  filter = \"*\""
  , "  copy = \"<EMAIL>\""
  , "}"
  ]
      
createFileIfMissing :: FilePath -> ByteString -> IO ()
createFileIfMissing aFile content =
  doesFileExist aFile >>= \case
    True -> putStrLn ("Skipping " ++ show aFile ++ "(already exist)")
    False -> ByteString.writeFile aFile content
{'fraction_non_alphanumeric': 0.1930909090909091, 'fraction_numerical': 0.009818181818181818, 'mean_word_length': 2.5172634271099743, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1108839', 'n_tokens_mistral': 2108, 'n_tokens_neox': 1973, 'n_words': 419}