<commit_before>{-# LANGUAGE PatternGuards #-}

module Main (main) where

import Control.Monad.State
import Data.Char
import Data.List
import System.Directory
import System.Environment
import System.FilePath

import BuildInfo
import FilenameDescr
import Change
import Utils
import Tar

-- TODO:
-- * Check installed trees too
-- * Check hashbangs

sizeChangeThresholds :: [(Integer,  -- Theshold only applies if one of
                                    -- the files is at least this big
                          Integer)] -- Size changed if the larger file's
                                    -- size is at least this %age of the
                                    -- smaller file's size
sizeChangeThresholds = [(     1000, 150),
                        (50 * 1000, 110)]

main :: IO ()
main = do args <- getArgs
          (ignoreSizeChanges, p1, p2) <-
              case args of
              [p1, p2]                          -> return (False, p1, p2)
              ["--ignore-size-changes", p1, p2] -> return (True,  p1, p2)
              _ -> die ["Bad args. Need 2 filepaths."]
          doFileOrDirectory ignoreSizeChanges p1 p2

doFileOrDirectory :: Bool -> FilePath -> FilePath -> IO ()
doFileOrDirectory ignoreSizeChanges p1 p2
 = do b <- doesDirectoryExist p1
      let doit = if b then doDirectory else doFile
      doit ignoreSizeChanges p1 p2

doDirectory :: Bool -> FilePath -> FilePath -> IO ()
doDirectory ignoreSizeChanges p1 p2
 = do fs1 <- getDirectoryContents p1
      fs2 <- getDirectoryContents p2
      let isVersionChar c = isDigit c || c == '.'
          mkFileInfo "." = return []
          mkFileInfo ".." = return []
          mkFileInfo fp@('g':'h':'c':'-':x:xs)
           | isDigit x = return [(("ghc-", "VERSION", dropWhile isVersionChar xs), fp)]
           | otherwise = die ["No version number in " ++ show fp]
          mkFileInfo fp = die ["Unrecognised filename " ++ show fp]
      fss1' <- mapM mkFileInfo fs1
      fss2' <- mapM mkFileInfo fs2
      let fs1' = sort $ concat fss1'
          fs2' = sort $ concat fss2'

          putBreak = putStrLn "=========="
          extraFile d fp = do putBreak
                              putStrLn ("Extra file in " ++ show d
                                     ++ ": " ++ show fp)
          doFiles [] [] = return ()
          doFiles ((_, fp) : xs) [] = do extraFile p1 fp
                                         doFiles xs []
          doFiles [] ((_, fp) : ys) = do extraFile p2 fp
                                         doFiles [] ys
          doFiles xs@((fpc1, fp1) : xs') ys@((fpc2, fp2) : ys')
              = do case fpc1 `compare` fpc2 of
                       EQ ->
                           do putBreak
                              putStrLn $ unwords ["Doing", show fp1, show fp2]
                              doFile ignoreSizeChanges (p1 </> fp1)
                                                       (p2 </> fp2)<commit_msg>Add a "Done." line to compare<commit_after>{-# LANGUAGE PatternGuards #-}

module Main (main) where

import Control.Monad.State
import Data.Char
import Data.List
import System.Directory
import System.Environment
import System.FilePath

import BuildInfo
import FilenameDescr
import Change
import Utils
import Tar

-- TODO:
-- * Check installed trees too
-- * Check hashbangs

sizeChangeThresholds :: [(Integer,  -- Theshold only applies if one of
                                    -- the files is at least this big
                          Integer)] -- Size changed if the larger file's
                                    -- size is at least this %age of the
                                    -- smaller file's size
sizeChangeThresholds = [(     1000, 150),
                        (50 * 1000, 110)]

main :: IO ()
main = do args <- getArgs
          (ignoreSizeChanges, p1, p2) <-
              case args of
              [p1, p2]                          -> return (False, p1, p2)
              ["--ignore-size-changes", p1, p2] -> return (True,  p1, p2)
              _ -> die ["Bad args. Need 2 filepaths."]
          doFileOrDirectory ignoreSizeChanges p1 p2

doFileOrDirectory :: Bool -> FilePath -> FilePath -> IO ()
doFileOrDirectory ignoreSizeChanges p1 p2
 = do b <- doesDirectoryExist p1
      let doit = if b then doDirectory else doFile
      doit ignoreSizeChanges p1 p2

doDirectory :: Bool -> FilePath -> FilePath -> IO ()
doDirectory ignoreSizeChanges p1 p2
 = do fs1 <- getDirectoryContents p1
      fs2 <- getDirectoryContents p2
      let isVersionChar c = isDigit c || c == '.'
          mkFileInfo "." = return []
          mkFileInfo ".." = return []
          mkFileInfo fp@('g':'h':'c':'-':x:xs)
           | isDigit x = return [(("ghc-", "VERSION", dropWhile isVersionChar xs), fp)]
           | otherwise = die ["No version number in " ++ show fp]
          mkFileInfo fp = die ["Unrecognised filename " ++ show fp]
      fss1' <- mapM mkFileInfo fs1
      fss2' <- mapM mkFileInfo fs2
      let fs1' = sort $ concat fss1'
          fs2' = sort $ concat fss2'

          putBreak = putStrLn "=========="
          extraFile d fp = do putBreak
                              putStrLn ("Extra file in " ++ show d
                                     ++ ": " ++ show fp)
          doFiles [] [] = do putBreak
                             putStrLn "Done."
          doFiles ((_, fp) : xs) [] = do extraFile p1 fp
                                         doFiles xs []
          doFiles [] ((_, fp) : ys) = do extraFile p2 fp
                                         doFiles [] ys
          doFiles xs@((fpc1, fp1) : xs') ys@((fpc2, fp2) : ys')
              = do case fpc1 `compare` fpc2 of
                       EQ ->
                           do putBreak
                              putStrLn $ unwords ["Doing", show fp1, show fp2]
                              doFile ignoreSizeChanges (p1 </> fp1)
                                                       (p2 </> fp2)
{'fraction_non_alphanumeric': 0.10512129380053908, 'fraction_numerical': 0.020889487870619946, 'mean_word_length': 1.4422048539695598, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7217651', 'n_tokens_mistral': 1701, 'n_tokens_neox': 1603, 'n_words': 543}