<commit_before>{-# LANGUAGE PatternGuards, RecordWildCards #-}

-- | Parsing is a slow point, the below is optimised
module Development.Ninja.Parse(parse) where

import qualified Data.ByteString.Char8 as BS
import Development.Shake.ByteString
import Development.Ninja.Env
import Development.Ninja.Type
import Control.Monad
import Data.Char
import Data.Maybe


endsDollar :: Str -> Bool
endsDollar = BS.isSuffixOf (BS.pack "$")

dropSpace :: Str -> Str
dropSpace = BS.dropWhile isSpace

startsSpace :: Str -> Bool
startsSpace = BS.isPrefixOf (BS.pack " ")



word1 :: Str -> (Str, Str)
word1 x = (a, dropSpace b)
    where (a,b) = BS.break isSpace x

isVar :: Char -> Bool
isVar x = isAlphaNum x || x == '_'

parse :: FilePath -> IO Ninja
parse file = do env <- newEnv; parseFile file env newNinja


parseFile :: FilePath -> Env Str Str -> Ninja -> IO Ninja
parseFile file env ninja = do
    src <- if file == "-" then BS.getContents else BS.readFile file
    foldM (applyStmt env) ninja $ splitStmts src


data Stmt = Stmt Str [Str] deriving Show<commit_msg>Switch a few instances of BS.pack to BS.singleton<commit_after>{-# LANGUAGE PatternGuards, RecordWildCards #-}

-- | Parsing is a slow point, the below is optimised
module Development.Ninja.Parse(parse) where

import qualified Data.ByteString.Char8 as BS
import Development.Shake.ByteString
import Development.Ninja.Env
import Development.Ninja.Type
import Control.Monad
import Data.Char
import Data.Maybe


endsDollar :: Str -> Bool
endsDollar = BS.isSuffixOf (BS.singleton '$')

dropSpace :: Str -> Str
dropSpace = BS.dropWhile isSpace

startsSpace :: Str -> Bool
startsSpace = BS.isPrefixOf (BS.singleton ' ')



word1 :: Str -> (Str, Str)
word1 x = (a, dropSpace b)
    where (a,b) = BS.break isSpace x

isVar :: Char -> Bool
isVar x = isAlphaNum x || x == '_'

parse :: FilePath -> IO Ninja
parse file = do env <- newEnv; parseFile file env newNinja


parseFile :: FilePath -> Env Str Str -> Ninja -> IO Ninja
parseFile file env ninja = do
    src <- if file == "-" then BS.getContents else BS.readFile file
    foldM (applyStmt env) ninja $ splitStmts src


data Stmt = Stmt Str [Str] deriving Show
{'fraction_non_alphanumeric': 0.1091581868640148, 'fraction_numerical': 0.0027752081406105457, 'mean_word_length': 3.76431718061674, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '89042', 'n_tokens_mistral': 731, 'n_tokens_neox': 700, 'n_words': 246}