<commit_before>module Text.Documentalist.CommentParser.TomDoc ( TomDocParser(..)
                                               ) where

import Data.List
import Data.List.Split
import Text.Documentalist.CommentParser
import Text.Documentalist.SourceParser

-- | Parses comments in tomdoc.org format.
data TomDocParser = TomDocParser

instance CommentParser TomDocParser where
    parseDocs _ (Package pkg mods) = Right $ Package pkg $ map parseModule mods

-- | Parses all the comments in a 'Module'.
parseModule :: Module (Maybe Comment) -> Module (Maybe DocBlock)
parseModule (Module mod decls) = Module mod $ parseDecls decls

-- | Parses all the comments in a list of 'Declaration's.
parseDecls :: [Declaration (Maybe Comment)] -> [Declaration (Maybe DocBlock)]
parseDecls = map parseDecl

-- | Parses the comment of a 'Declaration'.
parseDecl :: Declaration (Maybe Comment) -> Declaration (Maybe DocBlock)
parseDecl = fmap (>>= parseComment)

-- | Parses a single comment.
parseComment :: Comment -> Maybe DocBlock
parseComment (Comment str) =
    let paras = splitOn "\n\n" str

        isResult :: String -> Bool
        isResult [] = False
        isResult str = "Returns" `isPrefixOf` str

        isParams :: String -> Bool
        isParams str
            | not (null lns) = "- " `isInfixOf` head lns
            | otherwise = False
            where lns = lines str

        extract :: (String -> Bool) -> [String] -> (Maybe String, [String])
        extract _ [] = (Nothing, [])
        extract f xs =
            let (pref, rem) = break f xs
            in if null rem
                then (Nothing, xs)
                else (Just $ head rem, pref ++ tail rem)
    
        parseComment' :: DocBlock
        parseComment' =
            let (res, paras') = extract isResult paras
                (_, paras'') = extract isParams paras'
            in DocBlock { summary = parseParagraph $ head $ paras'' ++ [""]
                        , description = map parseParagraph $ filter (not . null) $ tail $ paras'' ++ [""]
                        , parameters = []
                        , example = Nothing
                        , result = fmap (Result . parseSpans) res
                        }
    in if null paras || null (head paras)
        then Nothing
        else Just parseComment'

parseParagraph :: String -> Paragraph
parseParagraph = TextParagraph . parseSpans

parseSpans :: String -> [Span]
parseSpans str = [PlainText str]<commit_msg>Parse TomDoc example groups<commit_after>module Text.Documentalist.CommentParser.TomDoc ( TomDocParser(..)
                                               ) where

import Data.List
import Data.List.Split
import Text.Documentalist.CommentParser
import Text.Documentalist.SourceParser

-- | Parses comments in tomdoc.org format.
data TomDocParser = TomDocParser

instance CommentParser TomDocParser where
    parseDocs _ (Package pkg mods) = Right $ Package pkg $ map parseModule mods

-- | Parses all the comments in a 'Module'.
parseModule :: Module (Maybe Comment) -> Module (Maybe DocBlock)
parseModule (Module mod decls) = Module mod $ parseDecls decls

-- | Parses all the comments in a list of 'Declaration's.
parseDecls :: [Declaration (Maybe Comment)] -> [Declaration (Maybe DocBlock)]
parseDecls = map parseDecl

-- | Parses the comment of a 'Declaration'.
parseDecl :: Declaration (Maybe Comment) -> Declaration (Maybe DocBlock)
parseDecl = fmap (>>= parseComment)

-- | Parses a single comment.
parseComment :: Comment -> Maybe DocBlock
parseComment (Comment str) =
    let paras = splitOn "\n\n" str

        isResult :: String -> Bool
        isResult = ("Returns" `isPrefixOf`)

        isParams :: String -> Bool
        isParams = any ("- " `isInfixOf`) . take 1 . lines

        isExampleHeading :: String -> Bool
        isExampleHeading "Example" = True
        isExampleHeading "Example:" = True
        isExampleHeading "Examples" = True
        isExampleHeading "Examples:" = True
        isExampleHeading _ = False

        extract :: (String -> Bool) -> [String] -> (Maybe String, [String])
        extract _ [] = (Nothing, [])
        extract f xs =
            let (pref, rem) = break f xs
            in if null rem
                then (Nothing, xs)
                else (Just $ head rem, pref ++ tail rem)
    
        parseComment' :: DocBlock
        parseComment' =
            let (res, paras') = extract isResult paras
                (_, paras'') = extract isParams paras'
                (body, examples) = break isExampleHeading paras''
            in DocBlock { summary = parseParagraph $ head $ body ++ [""]
                        , description = map parseParagraph $ filter (not . null) $ drop 1 body
                        , parameters = []
                        , example = if null examples then Nothing else Just $ Code $ intercalate "\n\n" $ drop 1 examples
                        , result = fmap (Result . parseSpans) res
                        }
    in if null paras || null (head paras)
        then Nothing
        else Just parseComment'

parseParagraph :: String -> Paragraph
parseParagraph = TextParagraph . parseSpans

parseSpans :: String -> [Span]
parseSpans str = [PlainText str]
{'fraction_non_alphanumeric': 0.09846153846153846, 'fraction_numerical': 0.000576923076923077, 'mean_word_length': 2.182986536107711, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2754453', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1332, 'n_words': 477}