<commit_before>import Tokenizer
import PPTokenParser
import Text.ParserCombinators.Parsec
import Lib.Util (concatWith)

data PreprocResult =
    IncludeRequest {
        alreadyParsed :: [PPGroupPart],
        fileToInclude :: String,
        remainingTokens :: [PPGroupPart],
        parsingState :: ParsingState} |
    CompleteResult {
        resultTokens :: [PPGroupPart],
        parsingState :: ParsingState} |
    ParseErr {errorMessage :: String }
    deriving (Eq, Show)

data ParsingState = ParsingState{
    definitions :: [(PPToken, [PPToken])]
}  deriving (Eq, Show)

joinResults :: PreprocResult -> PreprocResult -> PPGroupPart -> PreprocResult
joinResults err@(ParseErr _) _ _ = err
joinResults _ err@(ParseErr _) _ = err
joinResults (CompleteResult a _) (CompleteResult x y) newTokens = (CompleteResult (a ++ x ++ [newTokens]) y)
joinResults (CompleteResult x _) (IncludeRequest a b c d) newTokens = (IncludeRequest (a ++ x ++ [newTokens]) b c d)

performPreproc :: PreprocResult -> [PPGroupPart] -> PreprocResult
performPreproc lastResult@(ParseErr _) _ = lastResult 
performPreproc lastResult@(IncludeRequest _ _ _ _) _ = lastResult
performPreproc lastResult@(CompleteResult _ _) [] = lastResult
performPreproc lastResult@(CompleteResult resultTokens parsingState) (x:xs) =
    case x of
        (PPGroupPart (If_section if_group) groupTokens) ->
            case if_group of
                (PPGroupPart (If_group if_groupType subGroups) groupTokens) ->
                    case if_groupType of
                        Ifndef -> if elem (head groupTokens) $ map fst (definitions parsingState)
                            then performPreproc lastResult xs 
                            else
                                let
                                    subResult = performPreproc lastResult subGroups
                                in
                                    performPreproc subResult xs
        text@(PPGroupPart Text_line groupTokens) ->
            let
                nextResult = performPreproc lastResult xs
            in
                joinResults lastResult nextResult text

<commit_msg>fixed bug making result tokens to be in wrong order<commit_after>import Tokenizer
import PPTokenParser
import Text.ParserCombinators.Parsec
import Lib.Util (concatWith)

data PreprocResult =
    IncludeRequest {
        alreadyParsed :: [PPGroupPart],
        fileToInclude :: String,
        remainingTokens :: [PPGroupPart],
        parsingState :: ParsingState} |
    CompleteResult {
        resultTokens :: [PPGroupPart],
        parsingState :: ParsingState} |
    ParseErr {errorMessage :: String }
    deriving (Eq, Show)

data ParsingState = ParsingState{
    definitions :: [(PPToken, [PPToken])]
}  deriving (Eq, Show)

joinResults :: PreprocResult -> PreprocResult -> PPGroupPart -> PreprocResult
joinResults err@(ParseErr _) _ _ = err
joinResults _ err@(ParseErr _) _ = err
joinResults (CompleteResult a _) (CompleteResult x y) newToken = (CompleteResult (newToken:(a ++ x)) y)
joinResults (CompleteResult x _) (IncludeRequest a b c d) newToken = (IncludeRequest (newToken:(a ++ x)) b c d)

performPreproc :: PreprocResult -> [PPGroupPart] -> PreprocResult
performPreproc lastResult@(ParseErr _) _ = lastResult 
performPreproc lastResult@(IncludeRequest _ _ _ _) _ = lastResult
performPreproc lastResult@(CompleteResult _ _) [] = lastResult
performPreproc lastResult@(CompleteResult resultTokens parsingState) (x:xs) =
    case x of
        (PPGroupPart (If_section if_group) groupTokens) ->
            case if_group of
                (PPGroupPart (If_group if_groupType subGroups) groupTokens) ->
                    case if_groupType of
                        Ifndef -> if elem (head groupTokens) $ map fst (definitions parsingState)
                            then performPreproc lastResult xs 
                            else
                                let
                                    subResult = performPreproc lastResult subGroups
                                in
                                    performPreproc subResult xs
        text@(PPGroupPart Text_line groupTokens) ->
            let
                nextResult = performPreproc lastResult xs
            in
                joinResults lastResult nextResult text


{'fraction_non_alphanumeric': 0.07395348837209302, 'fraction_numerical': 0.0, 'mean_word_length': 2.3496884735202492, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1396057', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1118, 'n_words': 309}