<commit_before>{-# LANGUAGE FlexibleInstances #-}

module Base (
    module Development.Shake,
    module Development.Shake.FilePath,
    module Control.Applicative,
    module Data.Monoid,
    module Data.List,
    Stage (..),
    Args, arg,
    joinArgs, joinArgsWithSpaces,
    filterOut
    ) where

import Development.Shake hiding ((*>))
import Development.Shake.FilePath
import Control.Applicative
import Data.Monoid
import Data.List

data Stage = Stage0 | Stage1 | Stage2 | Stage3 deriving (Eq, Enum)

type Args = Action [String]

instance Monoid a => Monoid (Action a) where
    mempty = return mempty
    mappend p q = mappend <$> p <*> q

arg :: [String] -> Args
arg = return

intercalateArgs :: String -> Args -> Args
intercalateArgs s args = do
    as <- args
    return [intercalate s as]

joinArgsWithSpaces :: Args -> Args
joinArgsWithSpaces = intercalateArgs " "

joinArgs :: Args -> Args
joinArgs = intercalateArgs ""

filterOut :: Args -> [String] -> Args
filterOut args list = filter (`notElem` list) <$> args<commit_msg>Add splitArgs function to Base.hs.<commit_after>{-# LANGUAGE FlexibleInstances #-}

module Base (
    module Development.Shake,
    module Development.Shake.FilePath,
    module Control.Applicative,
    module Data.Monoid,
    module Data.List,
    Stage (..),
    Args, arg,
    joinArgs, joinArgsWithSpaces, splitArgs,
    filterOut
    ) where

import Development.Shake hiding ((*>))
import Development.Shake.FilePath
import Control.Applicative
import Data.Monoid
import Data.List

data Stage = Stage0 | Stage1 | Stage2 | Stage3 deriving (Eq, Enum)

type Args = Action [String]

instance Monoid a => Monoid (Action a) where
    mempty = return mempty
    mappend p q = mappend <$> p <*> q

arg :: [String] -> Args
arg = return

intercalateArgs :: String -> Args -> Args
intercalateArgs s args = do
    as <- args
    return [intercalate s as]

joinArgsWithSpaces :: Args -> Args
joinArgsWithSpaces = intercalateArgs " "

joinArgs :: Args -> Args
joinArgs = intercalateArgs ""

splitArgs :: Args -> Args
splitArgs = fmap (concatMap words)

filterOut :: Args -> [String] -> Args
filterOut args list = filter (`notElem` list) <$> args
{'fraction_non_alphanumeric': 0.10855565777368906, 'fraction_numerical': 0.0036798528058877645, 'mean_word_length': 3.569327731092437, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2789284', 'n_tokens_mistral': 686, 'n_tokens_neox': 703, 'n_words': 215}