<commit_before>{-# LANGUAGE NoMonomorphismRestriction, FlexibleContexts #-}
module PTS.File where

import Control.Monad
import Control.Monad.Environment
import Control.Monad.Reader
import Control.Monad.State
import Control.Monad.Trans
import Control.Monad.Log
import Control.Monad.Writer

import Data.Maybe (fromMaybe)

import System.Environment
import System.IO (hPutStrLn, stderr, hFlush, stdout)
import System.Exit (exitSuccess, exitFailure)
import System.Directory (findFile)

import Parametric.Error
import Parametric.Pretty hiding (when)

import PTS.AST
import PTS.Core
import PTS.Diff
import PTS.Instances
import PTS.Normalisation
import PTS.Options
import PTS.Parser
import PTS.Substitution
import PTS.Evaluation
import PTS.Algebra
import PTS.Binding
import PTS.Module

import qualified Data.Set as Set

import Tools.Errors

deliterateLine ('>' : ' ' : line) = ' ' : ' ' : line
deliterateLine _                  = ""

deliterate text = do
  flag <- asks optLiterate
  if flag then return . unlines . map deliterateLine . lines $ text
          else return text

processFile :: (Functor m, MonadErrors [FOmegaError] m, MonadReader Options m, MonadState [(Name, Binding M)] m, MonadIO m, MonadLog m) => FilePath -> m (Maybe (Module M))
processFile file = do
  outputLine $ "process file " ++ file
  path <- asks optPath
  fild <- liftIO (findFile path file) >>= maybe (fail ("file not found: " ++ file)) return
  text <- liftIO (readFile fild)
  text <- deliterate text
  File maybeName stmts <- parseFile file text
  (imports, contents) <- execWriterT (processStmts (lines text, stmts))
  return (do
    name <- maybeName
    return (Module imports name contents))

processStmts (text, stmts) = do
  annotateCode text $ mapM_ processStmt stmts
<commit_msg>Fix typo in variable name.<commit_after>{-# LANGUAGE NoMonomorphismRestriction, FlexibleContexts #-}
module PTS.File where

import Control.Monad
import Control.Monad.Environment
import Control.Monad.Reader
import Control.Monad.State
import Control.Monad.Trans
import Control.Monad.Log
import Control.Monad.Writer

import Data.Maybe (fromMaybe)

import System.Environment
import System.IO (hPutStrLn, stderr, hFlush, stdout)
import System.Exit (exitSuccess, exitFailure)
import System.Directory (findFile)

import Parametric.Error
import Parametric.Pretty hiding (when)

import PTS.AST
import PTS.Core
import PTS.Diff
import PTS.Instances
import PTS.Normalisation
import PTS.Options
import PTS.Parser
import PTS.Substitution
import PTS.Evaluation
import PTS.Algebra
import PTS.Binding
import PTS.Module

import qualified Data.Set as Set

import Tools.Errors

deliterateLine ('>' : ' ' : line) = ' ' : ' ' : line
deliterateLine _                  = ""

deliterate text = do
  flag <- asks optLiterate
  if flag then return . unlines . map deliterateLine . lines $ text
          else return text

processFile :: (Functor m, MonadErrors [FOmegaError] m, MonadReader Options m, MonadState [(Name, Binding M)] m, MonadIO m, MonadLog m) => FilePath -> m (Maybe (Module M))
processFile file = do
  outputLine $ "process file " ++ file
  path <- asks optPath
  file <- liftIO (findFile path file) >>= maybe (fail ("file not found: " ++ file)) return
  text <- liftIO (readFile file)
  text <- deliterate text
  File maybeName stmts <- parseFile file text
  (imports, contents) <- execWriterT (processStmts (lines text, stmts))
  return (do
    name <- maybeName
    return (Module imports name contents))

processStmts (text, stmts) = do
  annotateCode text $ mapM_ processStmt stmts

{'fraction_non_alphanumeric': 0.09183385137044363, 'fraction_numerical': 0.0, 'mean_word_length': 4.175438596491228, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7444684', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1093, 'n_words': 366}