<commit_before>{-# LANGUAGE NoImplicitPrelude #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RankNTypes        #-}
{-# LANGUAGE RecordWildCards   #-}

-- Module      : Khan.Model.S3.Bucket
-- Copyright   : (c) 2013 <NAME> <<EMAIL>>
-- License     : This Source Code Form is subject to the terms of
--               the Mozilla Public License, v. 2.0.
--               A copy of the MPL can be found in the LICENSE file or
--               you can obtain it at http://mozilla.org/MPL/2.0/.
-- Maintainer  : <NAME> <<EMAIL>>
-- Stability   : experimental
-- Portability : non-portable (GHC extensions)

module Khan.Model.S3.Bucket
    ( download
    , prune
    ) where

import           Control.Arrow
import           Data.Conduit
import qualified Data.Conduit.List         as Conduit
import           Data.Function             (on)
import qualified Data.List                 as List
import           Data.Ord
import qualified Data.Text                 as Text
import qualified Filesystem.Path.CurrentOS as Path
import           Khan.Internal
import qualified Khan.Model.S3.Object      as Object
import           Khan.Prelude
import           Network.AWS.S3
import qualified Filesystem as FS

download :: Int        -- ^ Limit
         -> Text       -- ^ Bucket
         -> Maybe Text -- ^ Prefix
         -> FilePath   -- ^ Destination dir.
         -> Bool       -- ^ Force download if file exists?
         -> Bool       -- ^ Strip prefix from key?
         -> Bool       -- ^ Skip ETag verification if file exists?
         -> AWS Bool
download n b p dir force sdest noVerify = do
    say "Paginating Bucket {} contents" [b]
    or <$> (paginate start
        $= Conduit.concatMap (filter match . gbrContents)
        $= chunked []
        $= Conduit.mapM (mapM (async . retrieve))
        $= Conduit.concatMapM (mapM wait)
        $$ Conduit.consume)
  where
    start = GetBucket b (Delimiter '/') (prefix p) 250 Nothing

    prefix (Just x) = Just . fromMaybe x $ Text.stripPrefix "/" x<commit_msg>Fox comment.<commit_after>{-# LANGUAGE NoImplicitPrelude #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RankNTypes        #-}
{-# LANGUAGE RecordWildCards   #-}

-- Module      : Khan.Model.S3.Bucket
-- Copyright   : (c) 2013 <NAME> <<EMAIL>>
-- License     : This Source Code Form is subject to the terms of
--               the Mozilla Public License, v. 2.0.
--               A copy of the MPL can be found in the LICENSE file or
--               you can obtain it at http://mozilla.org/MPL/2.0/.
-- Maintainer  : <NAME> <<EMAIL>>
-- Stability   : experimental
-- Portability : non-portable (GHC extensions)

module Khan.Model.S3.Bucket
    ( download
    , prune
    ) where

import           Control.Arrow
import           Data.Conduit
import qualified Data.Conduit.List         as Conduit
import           Data.Function             (on)
import qualified Data.List                 as List
import           Data.Ord
import qualified Data.Text                 as Text
import qualified Filesystem.Path.CurrentOS as Path
import           Khan.Internal
import qualified Khan.Model.S3.Object      as Object
import           Khan.Prelude
import           Network.AWS.S3
import qualified Filesystem as FS

download :: Int        -- ^ Concurrency
         -> Text       -- ^ Bucket
         -> Maybe Text -- ^ Prefix
         -> FilePath   -- ^ Destination dir.
         -> Bool       -- ^ Force download if file exists?
         -> Bool       -- ^ Strip prefix from key?
         -> Bool       -- ^ Skip ETag verification if file exists?
         -> AWS Bool
download n b p dir force sdest noVerify = do
    say "Paginating Bucket {} contents" [b]
    or <$> (paginate start
        $= Conduit.concatMap (filter match . gbrContents)
        $= chunked []
        $= Conduit.mapM (mapM (async . retrieve))
        $= Conduit.concatMapM (mapM wait)
        $$ Conduit.consume)
  where
    start = GetBucket b (Delimiter '/') (prefix p) 250 Nothing

    prefix (Just x) = Just . fromMaybe x $ Text.stripPrefix "/" x
{'fraction_non_alphanumeric': 0.10216256524981357, 'fraction_numerical': 0.007457121551081283, 'mean_word_length': 1.835799859055673, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5258842', 'n_tokens_mistral': 1189, 'n_tokens_neox': 1154, 'n_words': 378}