<commit_before>module Parse (parse, parseUnit) where

import Control.Applicative
  ( (*>)
  , (<$)
  , (<$>)
  , (<*)
  , (<*>)
  )
import System.Exit (exitFailure)
import System.FilePath (takeExtension)
import System.IO (hPutStrLn, stderr)
-- TODO: custom error messages with <?>
import Text.Parsec
  ( (<|>)
  , (<?>)
  , many
  , many1
  )
import Text.Parsec.String (Parser, parseFromFile)
import Text.Parsec.Combinator (eof)
import Text.ParserCombinators.Parsec.Char (anyChar, char, noneOf, string)

data Node = TranslationUnit [Node]
          | VimScript String
          | Comment String
          -- ...everything else
  deriving (Eq, Show)

-- | Parses a translation unit (file contents) into an AST.
parseUnit :: Parser Node
parseUnit =   TranslationUnit
          <$> many parseNode
          <*  eof

parseNode :: Parser Node
parseNode =   parseVimScript
          <|> parseComment

-- TODO: for now skip vimscript; later, parse it and use it to enrich the docs
-- or take some manual work out of specifying what's going on
parseVimScript :: Parser Node
parseVimScript =   VimScript
               <$> many1 (noneOf ['"']) -- anyChar

parseComment :: Parser Node
parseComment =   Comment
             <$> (char '"' *> many1 (noneOf ['\n']))

-- To test in the REPL:
-- import Parse (parseUnit)
-- Text.Parsec.runParser parseUnit () "this" "that"
parse :: String -> IO Node
parse fileName = parseFromFile parseUnit fileName >>= either report return
  where
    report err = do
      hPutStrLn stderr $ "Error: " ++ show err
      exitFailure<commit_msg>Facilitate testing in the REPL with `p`<commit_after>module Parse (p, parse, parseUnit) where

import Control.Applicative
  ( (*>)
  , (<$)
  , (<$>)
  , (<*)
  , (<*>)
  )
import System.Exit (exitFailure)
import System.FilePath (takeExtension)
import System.IO (hPutStrLn, stderr)
-- TODO: custom error messages with <?>
import Text.Parsec
  ( (<|>)
  , (<?>)
  , many
  , many1
  , runParser
  )
import Text.Parsec.String (Parser, parseFromFile)
import Text.Parsec.Combinator (eof)
import Text.ParserCombinators.Parsec.Char (anyChar, char, noneOf, string)

data Node = TranslationUnit [Node]
          | VimScript String
          | Comment String
          -- ...everything else
  deriving (Eq, Show)

-- | Parses a translation unit (file contents) into an AST.
parseUnit :: Parser Node
parseUnit =   TranslationUnit
          <$> many parseNode
          <*  eof

parseNode :: Parser Node
parseNode =   parseVimScript
          <|> parseComment

-- TODO: for now skip vimscript; later, parse it and use it to enrich the docs
-- or take some manual work out of specifying what's going on
parseVimScript :: Parser Node
parseVimScript =   VimScript
               <$> many1 (noneOf ['"']) -- anyChar

parseComment :: Parser Node
parseComment =   Comment
             <$> (char '"' *> many1 (noneOf ['\n']))

parse :: String -> IO Node
parse fileName = parseFromFile parseUnit fileName >>= either report return
  where
    report err = do
      hPutStrLn stderr $ "Error: " ++ show err
      exitFailure

-- | To facilitate quick testing in the console.
-- import Parse (p)
-- p "test"
p = runParser parseUnit () "(eval)"
{'fraction_non_alphanumeric': 0.12244897959183673, 'fraction_numerical': 0.0018838304552590266, 'mean_word_length': 2.972568578553616, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '563196', 'n_tokens_mistral': 1010, 'n_tokens_neox': 985, 'n_words': 315}