<commit_before>{-# LANGUAGE OverloadedStrings
           , GADTs
           , ParallelListComp
           , RecordWildCards
           , StandaloneDeriving
           , ExistentialQuantification
           , PatternGuards
  #-}
module Aws.SSSP where

import           Control.Applicative
import           Control.Monad
import           Control.Monad.Trans
import           Data.Char
import           Data.Either
import qualified Data.ByteString.Char8 as Bytes
import qualified Data.List as List
import           Data.Maybe
import           Data.Monoid
import           Data.Ord
import qualified Data.Set as Set
import           Data.Word
import           System.IO

import qualified Aws as Aws
import qualified Aws.Core as Aws
import qualified Aws.S3 as Aws
import qualified Blaze.ByteString.Builder as Blaze
import qualified Blaze.ByteString.Builder.Char.Utf8 as Blaze
import           Data.Attempt
import           Data.Attoparsec.Text (Parser)
import qualified Data.Attoparsec.Text as Atto
import qualified Data.Conduit as Conduit
import qualified Data.Enumerator.List as Enumerator
import           Data.Text (Text)
import qualified Data.Text as Text
import qualified Data.Text.Read as Text
import qualified Network.Wai as WWW
import qualified Network.Wai.Handler.Warp as WWW
import qualified Network.HTTP.Conduit as Conduit
import           Network.HTTP.Types (StdMethod(..))
import qualified Network.HTTP.Types as HTTP


wai :: Ctx -> WWW.Application
wai ctx@Ctx{..} req@WWW.Request{..} = do
  resolved <- liftIO $ task ctx requestMethod (resource req)
  maybe (return badTask) id $ do
    task <- resolved
    Just $ case task of
      Redirect t -> do
        i <- liftIO $ sigData 10
        let q = Aws.getObject bucket t Aws.s3ErrorResponseConsumer
            s = Aws.queryToUri (Aws.signQuery q s3 i)
            b = Blaze.fromByteString (s `Bytes.snoc` '\n')
        return $ WWW.ResponseBuilder
          status307 [("Content-Type", "text/plain"),("Location", s)] b
      Listing ts -> do
        return $ WWW.ResponseBuilder
          HTTP.status200 [("Content-Type", "text/plain")]
                         (mconcat (plusNL . s3Encode <$> ts))
      Remove ts  -> do<commit_msg>Add cache control headers for redirects.<commit_after>{-# LANGUAGE OverloadedStrings
           , GADTs
           , ParallelListComp
           , RecordWildCards
           , StandaloneDeriving
           , ExistentialQuantification
           , PatternGuards
  #-}
module Aws.SSSP where

import           Control.Applicative
import           Control.Monad
import           Control.Monad.Trans
import           Data.Char
import           Data.Either
import qualified Data.ByteString.Char8 as Bytes
import qualified Data.List as List
import           Data.Maybe
import           Data.Monoid
import           Data.Ord
import qualified Data.Set as Set
import           Data.Word
import           System.IO

import qualified Aws as Aws
import qualified Aws.Core as Aws
import qualified Aws.S3 as Aws
import qualified Blaze.ByteString.Builder as Blaze
import qualified Blaze.ByteString.Builder.Char.Utf8 as Blaze
import           Data.Attempt
import           Data.Attoparsec.Text (Parser)
import qualified Data.Attoparsec.Text as Atto
import qualified Data.Conduit as Conduit
import qualified Data.Enumerator.List as Enumerator
import           Data.Text (Text)
import qualified Data.Text as Text
import qualified Data.Text.Read as Text
import qualified Network.Wai as WWW
import qualified Network.Wai.Handler.Warp as WWW
import qualified Network.HTTP.Conduit as Conduit
import           Network.HTTP.Types (StdMethod(..))
import qualified Network.HTTP.Types as HTTP


wai :: Ctx -> WWW.Application
wai ctx@Ctx{..} req@WWW.Request{..} = do
  resolved <- liftIO $ task ctx requestMethod (resource req)
  maybe (return badTask) id $ do
    task <- resolved
    Just $ case task of
      Redirect t -> do
        let signFor = 10
        sigInfo <- liftIO $ sigData signFor
        let q = Aws.getObject bucket t Aws.s3ErrorResponseConsumer
            s = Aws.queryToUri (Aws.signQuery q s3 sigInfo)
            b = Blaze.fromByteString (s `Bytes.snoc` '\n')
            m = "max-age=" ++ show (floor signFor :: Integer)
            headers = [("Cache-Control", Bytes.pack m)
                      ,("Content-Type", "text/plain")
                      ,("Location", s)]
        return $ WWW.ResponseBuilder status307 headers b
      Listing ts -> do
        return $ WWW.ResponseBuilder
          HTTP.status200 [("Content-Type", "text/plain")]
                         (mconcat (plusNL . s3Encode <$> ts))
      Remove ts  -> do
{'fraction_non_alphanumeric': 0.07707295951504654, 'fraction_numerical': 0.006061918164104785, 'mean_word_length': 2.323741007194245, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4569796', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1286, 'n_words': 417}