<commit_before>{-# LANGUAGE BangPatterns      #-}
{-# LANGUAGE NoImplicitPrelude #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ViewPatterns      #-}

-- Module      : Main
-- Copyright   : (c) 2013-2014 <NAME> <<EMAIL>>
-- License     : This Source Code Form is subject to the terms of
--               the Mozilla Public License, v. 2.0.
--               A copy of the MPL can be found in the LICENSE file or
--               you can obtain it at http://mozilla.org/MPL/2.0/.
-- Maintainer  : <NAME> <<EMAIL>>
-- Stability   : experimental
-- Portability : non-portable (GHC extensions)

module Main (main) where

import qualified Data.Semigroup               as Semi
import           Data.String
import qualified Data.Text                    as Text
import qualified Data.Text.Lazy               as LText
import qualified Data.Text.Lazy.Encoding      as LText
import qualified Filesystem                   as FS
import qualified Filesystem.Path.CurrentOS    as Path
import           Khan.Internal                hiding (action)
import           Khan.Prelude
import           Network.HTTP.Conduit         hiding (path)
import           System.IO.Temp               (withSystemTempDirectory)
import           System.Process
import           Text.PrettyPrint.ANSI.Leijen (Pretty(..))

base :: String
base = "http://instance-data/"

output :: Text
output = "./metadata.tar.gz"

data Action
    = File { _action :: Text }
    | Dir  { _action :: Text }

instance Semi.Semigroup Action where
    (<>) a b = bool Dir File (isFile b) $ strip (_action a) <> "/" <> _action b

instance Pretty Action where
    pretty = pretty . _action

instance IsString Action where
    fromString = action . Text.pack

action :: Text -> Action
action (Text.strip -> t)
    | "/" `Text.isSuffixOf` t = Dir  t
    | otherwise               = File t

url :: Action -> String
url = mappend base . Text.unpack . _action

path :: Action -> FilePath
path a = bool (`Path.addExtension` "list") id (isFile a) $
    Path.fromText (strip (_action a))

isFile :: Action -> Bool
isFile File{} = True
isFile _      = False

strip :: Text -> Text
strip x = fromMaybe x ("/" `Text.stripSuffix` x)

main :: IO ()
main = do
    enableLogging

    log_ "Checking if running on an EC2 instance ..."
    !_ <- simpleHttp base

    withSystemTempDirectory "metadata." $ \(Path.decodeString -> d) -> do
        log_ "Starting metadata synchronisation ..."
        withManager $ \m ->
            mapM_ (retrieve d m)
                [ "latest/meta-data/"
                , "latest/user-data/"
                , "latest/dynamic"
                ]

        say "Compressing {} ..." [output]
        readProcess "tar" (map Text.unpack ["-cvf", output, toTextIgnore d]) ""
            >>= mapM_ putStrLn . lines

    log_ "Completed."
  where
    retrieve d m a = do
        say "Retrieve {}" [a]
        rq <- parseUrl (url a)
        rs <- httpLbs (rq { checkStatus = \ _ _ _ -> Nothing }) m
<commit_msg>Removing leading directory prefix when compressing output artifact<commit_after>{-# LANGUAGE BangPatterns      #-}
{-# LANGUAGE NoImplicitPrelude #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ViewPatterns      #-}

-- Module      : Main
-- Copyright   : (c) 2013-2014 <NAME> <<EMAIL>>
-- License     : This Source Code Form is subject to the terms of
--               the Mozilla Public License, v. 2.0.
--               A copy of the MPL can be found in the LICENSE file or
--               you can obtain it at http://mozilla.org/MPL/2.0/.
-- Maintainer  : <NAME> <<EMAIL>>
-- Stability   : experimental
-- Portability : non-portable (GHC extensions)

module Main (main) where

import qualified Data.Semigroup               as Semi
import           Data.String
import qualified Data.Text                    as Text
import qualified Data.Text.Lazy               as LText
import qualified Data.Text.Lazy.Encoding      as LText
import qualified Filesystem                   as FS
import qualified Filesystem.Path.CurrentOS    as Path
import           Khan.Internal                hiding (action)
import           Khan.Prelude
import           Network.HTTP.Conduit         hiding (path)
import           System.IO.Temp               (withSystemTempDirectory)
import           System.Process
import           Text.PrettyPrint.ANSI.Leijen (Pretty(..))

base :: String
base = "http://instance-data/"

output :: String
output = "./metadata.tar.gz"

data Action
    = File { _action :: Text }
    | Dir  { _action :: Text }

instance Semi.Semigroup Action where
    (<>) a b = bool Dir File (isFile b) $ strip (_action a) <> "/" <> _action b

instance Pretty Action where
    pretty = pretty . _action

instance IsString Action where
    fromString = action . Text.pack

action :: Text -> Action
action (Text.strip -> t)
    | "/" `Text.isSuffixOf` t = Dir  t
    | otherwise               = File t

url :: Action -> String
url = mappend base . Text.unpack . _action

path :: Action -> FilePath
path a = bool (`Path.addExtension` "list") id (isFile a) $
    Path.fromText (strip (_action a))

isFile :: Action -> Bool
isFile File{} = True
isFile _      = False

strip :: Text -> Text
strip x = fromMaybe x ("/" `Text.stripSuffix` x)

main :: IO ()
main = do
    enableLogging

    log_ "Checking if running on an EC2 instance ..."
    !_ <- simpleHttp base

    withSystemTempDirectory "metadata." $ \d -> do
        log_ "Starting metadata synchronisation ..."
        withManager $ \m ->
            mapM_ (retrieve (Path.decodeString d) m)
                [ "latest/meta-data/"
                , "latest/user-data/"
                , "latest/dynamic"
                ]

        say "Compressing {} ..." [output]
        readProcess "tar" ["-C", d, "-cvf", output, "."] ""
            >>= mapM_ putStrLn . lines

    log_ "Completed."
  where
    retrieve d m a = do
        say "Retrieve {}" [a]
        rq <- parseUrl (url a)
        rs <- httpLbs (rq { checkStatus = \ _ _ _ -> Nothing }) m

{'fraction_non_alphanumeric': 0.11463578382007385, 'fraction_numerical': 0.00436388049681101, 'mean_word_length': 1.9854709418837675, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2197399', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1713, 'n_words': 569}