<commit_before>
module Paper.Haskell2.Haskell(defines, rename, fakeImplement) where

import Data.Char
import Data.List
import Data.Maybe


defines :: String -> [String]
defines = f . map lexer2 . lines
    where
        f ((x:"::":_):rest) = x : f rest
        f (("(":x:")":"::":_):rest) = x : f rest
        f (_:rest) = f rest
        f [] = []


rename :: [(String, String)] -> String -> String
rename [] = id
rename ren = concat . map f . lexer
    where f x = fromMaybe x $ lookup x ren


-- figure out which definitions have a type signature
-- but no associated body, and make one up
fakeImplement :: String -> String
fakeImplement xs = xs ++ "\n" ++ unlines (
    [operator x ++ " = undefined -- stub" | x <- def \\ imp] ++
    ["-- " ++ show def, "-- " ++ show imp])
    where (def,imp) = (definesFunction xs, implementsFunction xs)


-- more a "this function is definately defined"
-- as conservative
definesFunction :: String -> [String]
definesFunction = concatMap (f . lexer2) . filter (not . isSpace . head) . filter (not .null) . lines
    where
        f ("(":x:")":"::":_) = [x]
        f (x:"::":_) = [x]
        f _ = []


-- more a "this function is possibly implemented"
-- as conservative
implementsFunction :: String -> [String]
implementsFunction = concatMap (f . lexer2) . lines
    where
        f (x:"::":xs) = []
        f (x:xs) = [x]
        f [] = []


operator (x:xs) | isAlpha x = x:xs
operator x = "(" ++ x ++ ")"


-- concat . lexer == id
lexer :: String -> [String]
lexer [] = []<commit_msg>Lots more tweaks to get the definitions a bit better<commit_after>
module Paper.Haskell2.Haskell(defines, rename, fakeImplement) where

import Data.Char
import Data.List
import Data.Maybe


defines :: String -> [String]
defines = nub . filter validName . concatMap f . map lexer2 . classLeft . lines
    where
        f ("(":name:")":_) = [name]
        f (name:_) = [name]
        f _ = []


validName x = isAlpha (head x) && x `notElem` keyword
keyword = ["class","instance","where","data","type"]


flushLeft (x:xs) = not $ isSpace x
flushLeft [] = False


classLeft (x:xs) | "class" `isPrefixOf` x = x : a ++ classLeft xs
    where (a,b) = span (\x -> null x || isSpace (head x)) xs
classLeft ((x:_):xs) | isSpace x = classLeft xs
classLeft (x:xs) = x : classLeft xs
classLeft [] = []


rename :: [(String, String)] -> String -> String
rename [] = id
rename ren = concat . map f . lexer
    where f x = fromMaybe x $ lookup x ren


-- figure out which definitions have a type signature
-- but no associated body, and make one up
fakeImplement :: String -> String
fakeImplement xs = unlines $
    ["-- !typesigFunction " ++ show typ, "-- !implementsFunction " ++ show imp, xs] ++
    [operator x ++ " = undefined -- stub" | x <- typ \\ imp]
    where (typ,imp) = (typesigFunction xs, implementsFunction xs)


-- more a "this function is definately defined"
-- as conservative
typesigFunction :: String -> [String]
typesigFunction = nub . concatMap (f . lexer2) . filter flushLeft . lines
    where
        f ("(":x:")":"::":_) = [x]
        f (x:"::":_) = [x]
        f _ = []


-- more a "this function is possibly implemented"
-- as conservative
implementsFunction :: String -> [String]
implementsFunction = nub . concatMap (f . lexer2) . lines
    where
        f (x:"::":xs) = []
        f (x:xs) = [x]
        f [] = []


operator (x:xs) | isAlpha x = x:xs
operator x = "(" ++ x ++ ")"


-- concat . lexer == id
lexer :: String -> [String]
lexer [] = []
{'fraction_non_alphanumeric': 0.16881627973631413, 'fraction_numerical': 0.002292920607623961, 'mean_word_length': 2.7246531483457845, 'pattern_counts': {'":': 24, '<': 5, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3362412', 'n_tokens_mistral': 1173, 'n_tokens_neox': 1139, 'n_words': 390}