<commit_before>import qualified Data.Map as Map
import Data.List
import System.IO

import IgrepCashbook2

type Group       = String
type SignedPrice = String
type Summary     = Map.Map Group Int

-- Item: synonim of [String] so far
parseContents :: String -> [(Int, Item)]
parseContents c =
  map ( \(n, l) -> ( n, parseItemLine l ) ) is
  where
    ls = lines c
    ns = [ 1..( length ls ) ]
    is = selectItemLine ns ls

selectItemLine :: [Int] -> [String] -> [(Int, String)]
selectItemLine ns ls =
  filter ( \(n, l) -> isItemLine l ) $ zip ns ls

main = do
  args <- getArgs
  forM args (\a -> do
    contents <- readFile a

    let items = parseContents contents
    let items' = rejectInvalidItems items
    let ( inItems, exItems ) = classifyItems items'
    let exSummary = summarizeItems exItems
    let inSummary = summarizeItems inItems
    let exSum = sum $ Map.values exSummary
    let inSum = sum $ Map.values inSummary

    -- TODO: print summary of all files in specified in the argument
    putStrLn "# EXPENDITURES #"
    putStrLn formatSummary exSummary
    putStrLn formatItem "Sum" exSum
    putStrLn "# INCOME #"
    putStrLn formatSummary inSummary
    putStrLn formatItem "Sum" inSum )
<commit_msg>rewrite some functions to detect invalid lines<commit_after>import qualified Data.Map as Map
import Data.List
import System.IO

import IgrepCashbook2

type Group       = String
type SignedPrice = String
type Summary     = Map.Map Group Int

-- Item: synonim of [String] so far
parseContents :: String -> [(Int, Either String Item)]
parseContents c =
  map ( \(n, l) -> ( n, parseItemLine l ) ) is
  where
    ls = lines c
    ns = [ 1..( length ls ) ]
    is = selectItemLine ns ls

selectItemLine :: [Int] -> [String] -> [(Int, String)]
selectItemLine ns ls =
  filter ( \(n, l) -> isItemLine l ) $ zip ns ls

itemsAndErrors :: [(Int, Either String Item)] -> ([Item], [String])
itemsAndErrors xs = foldr f ([], []) xs
  where
    f :: (Int, Either String Item) -> ([Item], [String]) -> ([Item], [String])
    f (n, Left s) (is, ss) = (is, (s ++ " at line " ++ read n):ss)
    f (_, Right i) (is, ss) = (i:is, ss)

main = do
  args <- getArgs
  forM args (\a -> do
    contents <- readFile a

    let items = parseContents contents
    let ( items', errors ) = itemsAndErrors items
    warnErrors a errors
    let ( inItems, exItems ) = classifyItems items'
    let exSummary = summarizeItems exItems
    let inSummary = summarizeItems inItems
    let exSum = sum $ Map.values exSummary
    let inSum = sum $ Map.values inSummary

    -- TODO: print summary of all files in specified in the argument
    putStrLn "# EXPENDITURES #"
    putStrLn formatSummary exSummary
    putStrLn formatItem "Sum" exSum
    putStrLn "# INCOME #"
    putStrLn formatSummary inSummary
    putStrLn formatItem "Sum" inSum )

{'fraction_non_alphanumeric': 0.10675150229763167, 'fraction_numerical': 0.0014139271827500884, 'mean_word_length': 2.980309423347398, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '587093', 'n_tokens_mistral': 927, 'n_tokens_neox': 884, 'n_words': 333}