<commit_before>import Control.Monad (forM_, when, unless)
import Data.Algorithm.Diff (getGroupedDiff)
import Data.Algorithm.DiffOutput (ppDiff)
import Data.List (delete,isSuffixOf)
import System.IO (hPutStrLn,stderr)
import System.Directory (getDirectoryContents)
import System.Exit (ExitCode(..), exitSuccess, exitFailure)
import System.FilePath ((</>))
import System.Process (readProcessWithExitCode)

testdir, ghcUnlit, myUnlit :: String
testdir  = "/Users/pepijn/Projects/unlit/test/ghc"
ghcUnlit = "/Users/pepijn/Projects/unlit/test/ghcunlit"
myUnlit  = "/Users/pepijn/Library/Haskell/bin/unlit"

runGhcUnlit :: String -> IO String
runGhcUnlit ifile = do
  (exitCode, stdout, stderr) <- readProcessWithExitCode ghcUnlit [ifile, "-"] ""
  case exitCode of
   ExitSuccess   -> return stdout
   ExitFailure n -> return stderr

runMyUnlit :: String -> IO String
runMyUnlit ifile = do
  (exitCode, stdout, stderr) <- readProcessWithExitCode myUnlit [ifile,"-"] ""
  case exitCode of
   ExitSuccess   -> return stdout
   ExitFailure n -> return stderr

removeBlankLines :: [String] -> [String]
removeBlankLines = filter (not . null)

main :: IO ()
main = do
  files <- fmap (delete "." . delete "..") (getDirectoryContents testdir)
  forM_ files $ \file -> do

    when ("disabled" `isSuffixOf` file) $ do

      ghcResult <- runGhcUnlit (testdir </> file)
      myResult  <- runMyUnlit (testdir </> file)

      let
        ghcResult', myResult'  :: [String]
        ghcResult' = removeBlankLines . lines $ ghcResult
        myResult'  = removeBlankLines . lines $ myResult

      putStrLn $ "Testing file: " ++ file
      unless (myResult' == ghcResult') $ do

        hPutStrLn stderr $ "Error in file: " ++ file
        hPutStrLn stderr $ ppDiff $ getGroupedDiff ghcResult' myResult'

  exitSuccess<commit_msg>reenabled tests<commit_after>import Control.Monad (forM_, when, unless)
import Data.Algorithm.Diff (getGroupedDiff)
import Data.Algorithm.DiffOutput (ppDiff)
import Data.List (delete,isSuffixOf)
import System.IO (hPutStrLn,stderr)
import System.Directory (getDirectoryContents)
import System.Exit (ExitCode(..), exitSuccess, exitFailure)
import System.FilePath ((</>))
import System.Process (readProcessWithExitCode)

testdir, ghcUnlit, myUnlit :: String
testdir  = "/Users/pepijn/Projects/unlit/test/ghc"
ghcUnlit = "/Users/pepijn/Projects/unlit/test/ghcunlit"
myUnlit  = "/Users/pepijn/Library/Haskell/bin/unlit"

runGhcUnlit :: String -> IO String
runGhcUnlit ifile = do
  (exitCode, stdout, stderr) <- readProcessWithExitCode ghcUnlit [ifile, "-"] ""
  case exitCode of
   ExitSuccess   -> return stdout
   ExitFailure n -> return stderr

runMyUnlit :: String -> IO String
runMyUnlit ifile = do
  (exitCode, stdout, stderr) <- readProcessWithExitCode myUnlit [ifile,"-"] ""
  case exitCode of
   ExitSuccess   -> return stdout
   ExitFailure n -> return stderr

removeBlankLines :: [String] -> [String]
removeBlankLines = filter (not . null)

main :: IO ()
main = do
  files <- fmap (delete "." . delete "..") (getDirectoryContents testdir)
  forM_ files $ \file -> do

    when ("lhs" `isSuffixOf` file) $ do

      ghcResult <- runGhcUnlit (testdir </> file)
      myResult  <- runMyUnlit (testdir </> file)

      let
        ghcResult', myResult'  :: [String]
        ghcResult' = removeBlankLines . lines $ ghcResult
        myResult'  = removeBlankLines . lines $ myResult

      putStrLn $ "Testing file: " ++ file
      unless (myResult' == ghcResult') $ do

        hPutStrLn stderr $ "Error in file: " ++ file
        hPutStrLn stderr $ ppDiff $ getGroupedDiff ghcResult' myResult'

  exitSuccess
{'fraction_non_alphanumeric': 0.12034225779740547, 'fraction_numerical': 0.0, 'mean_word_length': 3.7064935064935063, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1309214', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1197, 'n_words': 306}