<commit_before>{-# Language ScopedTypeVariables #-}

module HsImport.Parse
   ( parseFile
   , lastImportSrcLine
   ) where

import qualified Data.Text.IO as TIO
import qualified Data.Text as T
import Data.Maybe (fromMaybe)
import Data.List (isPrefixOf)
import qualified Language.Haskell.Exts as HS
import Control.Applicative ((<$>))
import Control.Exception (catch, SomeException)

type Error = String

parseFile :: FilePath -> IO (Either Error (HS.ParseResult HS.Module))
parseFile file = do
   srcFile <- unlines. replaceCPPByComment . lines . T.unpack <$> TIO.readFile file
   catch (do let result = parseFileContents srcFile
             case result of
                  HS.ParseOk _ -> return $ Right result

                  HS.ParseFailed srcLoc _ -> do
                     srcResult <- parseInvalidSource (lines srcFile) 0 (HS.srcLine srcLoc)
                     return $ Right $ fromMaybe result srcResult)

         (\(e :: SomeException) -> do
            let srcLines = lines srcFile
            srcResult <- parseInvalidSource srcLines 0 (length srcLines)
            return $ maybe (Left $ show e) Right srcResult)
   where
      -- | replace CPP directives by a fake comment
      replaceCPPByComment = map $ \line ->
         if "#" `isPrefixOf` line
            then "-- fake hsimport comment"
            else line


type SrcLine = Int

-- | Expects that '[String]' starts with an import declaration and returns
--   the last source line of the import declaration, so this function
--   is for the handling of multine line import declarations.
lastImportSrcLine :: [String] -> Maybe SrcLine
lastImportSrcLine srcLines
   | null srcLines
   = Nothing

   | "import" `isPrefixOf` head srcLines
   = parseImport 1

   | otherwise
   = Nothing

   where
      parseImport lastLine
         | lastLine <= numSrcLines
         = case parseFileContents source of
                HS.ParseOk _       -> Just lastLine
                HS.ParseFailed _ _ -> parseImport (lastLine + 1)

         | otherwise
         = Nothing

         where
            source = unlines $ take lastLine srcLines

      numSrcLines = length srcLines


-- | tries to find the maximal part of the source file (from the beginning) that contains
--   valid/complete Haskell code
parseInvalidSource :: [String] -> Int -> Int -> IO (Maybe (HS.ParseResult HS.Module))
parseInvalidSource srcLines lastValidLine firstInvalidLine
   | null srcLines || lastValidLine >= firstInvalidLine = return Nothing
   | otherwise =
      catch (case parseFileContents source of
                  result@(HS.ParseOk _)
                     | (nextLine + 1) == firstInvalidLine ->
                        return $ Just result
                     | otherwise                          ->
                        parseInvalidSource srcLines nextLine firstInvalidLine

                  HS.ParseFailed _ _ -> parseInvalidSource srcLines lastValidLine nextLine)

            (\(_ :: SomeException) -> parseInvalidSource srcLines lastValidLine nextLine)
   where
      source   = unlines $ take (nextLine + 1) srcLines
      nextLine = lastValidLine + (floor ((realToFrac (firstInvalidLine - lastValidLine) / 2) :: Double) :: Int)


parseFileContents :: String -> HS.ParseResult HS.Module
parseFileContents = HS.parseFileContentsWithMode parseMode
   where
      parseMode = HS.defaultParseMode { HS.fixities = Just [] }<commit_msg>Better handling of incomplete source files<commit_after>{-# Language ScopedTypeVariables #-}

module HsImport.Parse
   ( parseFile
   , lastImportSrcLine
   ) where

import qualified Data.Text.IO as TIO
import qualified Data.Text as T
import Data.Maybe (fromMaybe)
import Data.List (isPrefixOf)
import qualified Language.Haskell.Exts as HS
import Control.Applicative ((<$>))
import Control.Exception (catch, SomeException)

type Error = String

parseFile :: FilePath -> IO (Either Error (HS.ParseResult HS.Module))
parseFile file = do
   srcFile <- unlines. replaceCPPByComment . lines . T.unpack <$> TIO.readFile file
   catch (do let result = parseFileContents srcFile
             case result of
                  HS.ParseOk _ -> return $ Right result

                  HS.ParseFailed srcLoc _ -> do
                     srcResult <- parseInvalidSource (lines srcFile) (HS.srcLine srcLoc) 0 (HS.srcLine srcLoc)
                     return $ Right $ fromMaybe result srcResult)

         (\(e :: SomeException) -> do
            let srcLines = lines srcFile
            srcResult <- parseInvalidSource srcLines (length srcLines) 0 (length srcLines)
            return $ maybe (Left $ show e) Right srcResult)
   where
      -- | replace CPP directives by a fake comment
      replaceCPPByComment = map $ \line ->
         if "#" `isPrefixOf` line
            then "-- fake hsimport comment"
            else line


type SrcLine = Int

-- | Expects that '[String]' starts with an import declaration and returns
--   the last source line of the import declaration, so this function
--   is for the handling of multine line import declarations.
lastImportSrcLine :: [String] -> Maybe SrcLine
lastImportSrcLine srcLines
   | null srcLines
   = Nothing

   | "import" `isPrefixOf` head srcLines
   = parseImport 1

   | otherwise
   = Nothing

   where
      parseImport lastLine
         | lastLine <= numSrcLines
         = case parseFileContents source of
                HS.ParseOk _       -> Just lastLine
                HS.ParseFailed _ _ -> parseImport (lastLine + 1)

         | otherwise
         = Nothing

         where
            source = unlines $ take lastLine srcLines

      numSrcLines = length srcLines


-- | tries to find the maximal part of the source file (from the beginning) that contains
--   valid/complete Haskell code
parseInvalidSource :: [String] -> Int -> Int -> Int -> IO (Maybe (HS.ParseResult HS.Module))
parseInvalidSource srcLines firstInvalidLine lastValidLine currLastLine
   | null srcLines || lastValidLine >= currLastLine = return Nothing
   | otherwise =
      catch (case parseFileContents source of
                  result@(HS.ParseOk _)
                     | (nextLine + 1) == currLastLine ->
                        return $ Just result
                     | otherwise                          ->
                        parseInvalidSource srcLines firstInvalidLine nextLine currLastLine

                  HS.ParseFailed srcLoc _
                     | HS.srcLine srcLoc == firstInvalidLine   ->
                        parseInvalidSource srcLines firstInvalidLine lastValidLine nextLine
                     | otherwise                          ->
                        parseInvalidSource srcLines firstInvalidLine nextLine currLastLine)

            (\(_ :: SomeException) -> parseInvalidSource srcLines firstInvalidLine lastValidLine nextLine)
   where
      source   = unlines $ take (nextLine + 1) srcLines
      nextLine = lastValidLine + (floor ((realToFrac (currLastLine - lastValidLine) / 2) :: Double) :: Int)


parseFileContents :: String -> HS.ParseResult HS.Module
parseFileContents = HS.parseFileContentsWithMode parseMode
   where
      parseMode = HS.defaultParseMode { HS.fixities = Just [] }
{'fraction_non_alphanumeric': 0.07432243643475832, 'fraction_numerical': 0.0019558535903883767, 'mean_word_length': 2.254090909090909, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2603074', 'n_tokens_mistral': 1887, 'n_tokens_neox': 1772, 'n_words': 608}