<commit_before>
module FilenameDescr where

import Data.Either
import Data.List

import BuildInfo
import Utils
import Tar

-- We can't just compare plain filenames, because versions numbers of GHC
-- and the libaries will vary. So we use FilenameDescr instead, which
-- abstracts out the version numbers.
type FilenameDescr = [FilenameDescrBit]
data FilenameDescrBit = VersionOf String
                      | HashOf String
                      | FP String
                      | Ways
    deriving (Show, Eq, Ord)

normaliseDescr :: FilenameDescr -> FilenameDescr
normaliseDescr [] = []
normaliseDescr [x] = [x]
normaliseDescr (FP x1 : FP x2 : xs) = normaliseDescr (FP (x1 ++ x2) : xs)
normaliseDescr (x : xs) = x : normaliseDescr xs

-- Sanity check that the FilenameDescr matches the filename in the tar line
checkContent :: BuildInfo -> (FilenameDescr, TarLine) -> Errors
checkContent buildInfo (fd, tl)
 = let fn = tlFileName tl
   in case flattenFilenameDescr buildInfo fd of
      Right fn' ->
          if fn' == fn
          then []
          else ["checkContent: Can't happen: filename mismatch: " ++ show fn]
      Left errs ->
          errs

flattenFilenameDescr :: BuildInfo -> FilenameDescr
                     -> Either Errors FilePath<commit_msg>Add a kludge to the compare tool for unicode filenames<commit_after>
module FilenameDescr where

import Data.Char
import Data.Either
import Data.List

import BuildInfo
import Utils
import Tar

-- We can't just compare plain filenames, because versions numbers of GHC
-- and the libaries will vary. So we use FilenameDescr instead, which
-- abstracts out the version numbers.
type FilenameDescr = [FilenameDescrBit]
data FilenameDescrBit = VersionOf String
                      | HashOf String
                      | FP String
                      | Ways
    deriving (Show, Eq, Ord)

normaliseDescr :: FilenameDescr -> FilenameDescr
normaliseDescr [] = []
normaliseDescr [x] = [x]
normaliseDescr (FP x1 : FP x2 : xs) = normaliseDescr (FP (x1 ++ x2) : xs)
normaliseDescr (x : xs) = x : normaliseDescr xs

-- Sanity check that the FilenameDescr matches the filename in the tar line
checkContent :: BuildInfo -> (FilenameDescr, TarLine) -> Errors
checkContent buildInfo (fd, tl)
 = let fn = tlFileName tl
   in case flattenFilenameDescr buildInfo fd of
      Right fn' ->
          if fn' == fn
          then []
          else if all isAscii fn
               then ["checkContent: Can't happen: filename mismatch: "
                  ++ show fn]
               else [] -- Ugly kludge; don't worry too much if filepaths
                       -- containing non-ASCII chars have gone wrong
      Left errs ->
          errs

flattenFilenameDescr :: BuildInfo -> FilenameDescr
                     -> Either Errors FilePath
{'fraction_non_alphanumeric': 0.07537688442211055, 'fraction_numerical': 0.002871500358937545, 'mean_word_length': 2.492481203007519, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7637236', 'n_tokens_mistral': 819, 'n_tokens_neox': 782, 'n_words': 300}