<commit_before>-- | Filter a word list, removing those words that are duplicates apart from common suffixes.  |
-- | Reads SORTED list on stdin, emits filtered list on stdout.
main :: IO()
main = do
  allInput <- getContents
  putStrLn ("Got " ++ (show (myLength (lines allInput))) ++ " lines")
  putStrLn (show suffixes)
  putStrLn ("suffixes is a list of length " ++ (show (myLength suffixes)))
  putStrLn "Done."

-- | The common suffixes we will be removing
suffixes :: [[Char]]
suffixes = ["s","es","ed","ing","ly"] 

myLength :: [a] -> Integer
myLength [] = 0
myLength (_:xs) = 1 + (myLength xs)<commit_msg>Got it working, but it ain't perfect. Doesn't recognize roots, which should be easy to fix, and also, when the prefix and the suffixed word are separated by another word it doesn't recognize as suffixed, it misses the following suffixed word. But, hey....<commit_after>import Data.List

-- | Filter a word list, removing those words that are duplicates apart from common suffixes.  |
-- | Reads SORTED list on stdin, emits filtered list on stdout.
main :: IO()
main = do
  allInput <- getContents
  putStrLn ("Got " ++ (show (myLength (lines allInput))) ++ " lines")
  putStrLn ("Suffixes are:\n\t" ++ (concat (intersperse "\n\t" suffixes)))
  putStrLn ("suffixes is a list of length " ++ (show (myLength suffixes)))
  putStrLn ("Filtered list has " ++ (show (length (filterSuffixes (lines allInput)))) ++ " words.")
  putStrLn "Words:"
  putStrLn (concat (intersperse "\n" (filterSuffixes (lines allInput))))
  putStrLn "Done."

-- | The common suffixes we will be removing
suffixes :: [[Char]]
suffixes = ["s","es","ed","ing","ly"] 

-- | Just a stupid length function to show that we can use pure functions on something bound with
-- | the "<-" operator.
myLength :: [a] -> Integer
myLength [] = 0
myLength (_:xs) = 1 + (myLength xs)

-- | The actual filter function. Removes words that are like prefix apart from common suffix.
filterSuffixes :: [[Char]] -> [[Char]]
filterSuffixes [] = []
filterSuffixes [lastword] = [lastword]
filterSuffixes (prefix : [lastword])
  | (hasSuffix prefix lastword)  = [prefix]
  | otherwise                    = prefix : [lastword]
filterSuffixes (prefix : nextword : restwords)
  | (hasSuffix prefix nextword)  = filterSuffixes (prefix : restwords)
  | otherwise                    = prefix : (filterSuffixes (nextword : restwords))

-- | Returns true if the given word appends one of the common suffixes to the given prefix.
hasSuffix :: [Char] -> [Char] -> Bool
hasSuffix prefix word 
  | isPrefix prefix word   = elem (drop (length prefix) word) suffixes
  | otherwise              = False

{-
-- | Returns the common suffix that the given word appends to the given prefix or the empty string
suffix :: [Char] -> [Char] -> [Char]
suffix prefix word
  | isPrefix prefix word   = elem (drop (length prefix) word) suffixes
  | otherwise              = ""
-}

-- | Returns true if the given prefix is actually a prefix of the given word
isPrefix :: [Char] -> [Char] -> Bool
isPrefix prefix word = (take (length prefix) word) == prefix

{'fraction_non_alphanumeric': 0.12836304700162074, 'fraction_numerical': 0.0012965964343598054, 'mean_word_length': 3.6197604790419162, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1645132', 'n_tokens_mistral': 953, 'n_tokens_neox': 914, 'n_words': 372}