<commit_before>{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE TupleSections #-}
{-# LANGUAGE ViewPatterns #-}
{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE PartialTypeSignatures #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE ScopedTypeVariables #-}
module Main where

import Language.Haskell.GHC.ExactPrint
import Language.Haskell.GHC.ExactPrint.Annotate
import Language.Haskell.GHC.ExactPrint.Types
import Language.Haskell.GHC.ExactPrint.Utils

import Data.Data
import Data.Generics.Schemes

import HsExpr as GHC hiding (Stmt)
import qualified HsExpr as GHC (Stmt)
import HsBinds as GHC
import HsSyn hiding (Pat, Stmt)
import qualified HsSyn as GHC (Pat, Stmt)
import SrcLoc
import qualified SrcLoc as GHC
import qualified RdrName as GHC
import qualified ApiAnnotation as GHC
import qualified DynFlags      as GHC
import qualified FastString    as GHC
import qualified GHC           as GHC hiding (parseModule)
import qualified HeaderInfo    as GHC
import qualified Lexer         as GHC
import qualified MonadUtils    as GHC
import qualified Outputable    as GHC
import qualified Parser        as GHC
import qualified SrcLoc        as GHC
import qualified StringBuffer  as GHC
import qualified OccName as GHC
import Data.Generics
import Control.Monad.State

import qualified Data.Map as Map

import Debug.Trace

import Data.Monoid

import System.IO.Unsafe

import Control.Arrow

import Data.Maybe

import qualified Refact.Types as R
import Refact.Types hiding (SrcSpan)

import System.Environment
import System.Process
import System.Directory
import System.IO

main = do
  args <- getArgs
  case args of
    ["pipe", file] -> runPipe file
    ["test", file] -> runTest file

-- Types
--
type M a = State Anns a

type Module = (GHC.Located (GHC.HsModule GHC.RdrName))

type Expr = GHC.Located (GHC.HsExpr GHC.RdrName)

type Bind = HsBindLR  GHC.RdrName GHC.RdrName

type Type = GHC.Located (GHC.HsType GHC.RdrName)

type Decl = GHC.Located (GHC.HsDecl GHC.RdrName)

type Pat = GHC.LPat GHC.RdrName

type Stmt = ExprLStmt GHC.RdrName

mergeAnns :: Anns -> Anns -> Anns
mergeAnns (a, b) (c,d) = (Map.union a c, Map.union b d)

-- | Replaces an old expression with a new expression
replace :: AnnKey -> AnnKey -> Anns -> Maybe Anns
replace old new (as, keys) = do
  oldan <- Map.lookup old as
  newan <- Map.lookup new as
  return . (, keys) . Map.delete old . Map.insert new (combine oldan newan) $ as

combine :: Annotation -> Annotation -> Annotation
combine oldann newann =
  Ann (annEntryDelta oldann) (annDelta oldann) (annTrueEntryDelta oldann)
      (annPriorComments oldann ++ annPriorComments newann) (annsDP newann)


mkKey :: (Data a) => GHC.Located a -> AnnKey
mkKey (GHC.L l s) = AnnKey l (annGetConstr s) NotNeeded


-- | Shift the first output annotation into the correct place
moveAnns :: [(KeywordId, DeltaPos)] -> [(KeywordId, DeltaPos)] -> [(KeywordId, DeltaPos)]
moveAnns [] xs        = xs
moveAnns ((_, dp): _) ((kw, _):xs) = (kw,dp) : xs

-- Pipe

runPipe :: FilePath -> IO ()
runPipe file = do
  path <- canonicalizePath file
  rawhints <- getHints path<commit_msg>Maintain extra comma when doing replacement<commit_after>{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE TupleSections #-}
{-# LANGUAGE ViewPatterns #-}
{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE PartialTypeSignatures #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE ScopedTypeVariables #-}
module Main where

import Language.Haskell.GHC.ExactPrint
import Language.Haskell.GHC.ExactPrint.Annotate
import Language.Haskell.GHC.ExactPrint.Types
import Language.Haskell.GHC.ExactPrint.Utils

import Data.Data
import Data.Generics.Schemes

import HsExpr as GHC hiding (Stmt)
import qualified HsExpr as GHC (Stmt)
import HsBinds as GHC
import HsSyn hiding (Pat, Stmt)
import qualified HsSyn as GHC (Pat, Stmt)
import SrcLoc
import qualified SrcLoc as GHC
import qualified RdrName as GHC
import qualified ApiAnnotation as GHC
import qualified DynFlags      as GHC
import qualified FastString    as GHC
import qualified GHC           as GHC hiding (parseModule)
import qualified HeaderInfo    as GHC
import qualified Lexer         as GHC
import qualified MonadUtils    as GHC
import qualified Outputable    as GHC
import qualified Parser        as GHC
import qualified SrcLoc        as GHC
import qualified StringBuffer  as GHC
import qualified OccName as GHC
import Data.Generics
import Control.Monad.State

import qualified Data.Map as Map

import Debug.Trace

import Data.Monoid

import System.IO.Unsafe

import Control.Arrow

import Data.Maybe

import qualified Refact.Types as R
import Refact.Types hiding (SrcSpan)

import System.Environment
import System.Process
import System.Directory
import System.IO

main = do
  args <- getArgs
  case args of
    ["pipe", file] -> runPipe file
    ["test", file] -> runTest file

-- Types
--
type M a = State Anns a

type Module = (GHC.Located (GHC.HsModule GHC.RdrName))

type Expr = GHC.Located (GHC.HsExpr GHC.RdrName)

type Bind = HsBindLR  GHC.RdrName GHC.RdrName

type Type = GHC.Located (GHC.HsType GHC.RdrName)

type Decl = GHC.Located (GHC.HsDecl GHC.RdrName)

type Pat = GHC.LPat GHC.RdrName

type Stmt = ExprLStmt GHC.RdrName

mergeAnns :: Anns -> Anns -> Anns
mergeAnns (a, b) (c,d) = (Map.union a c, Map.union b d)

-- | Replaces an old expression with a new expression
replace :: AnnKey -> AnnKey -> Anns -> Maybe Anns
replace old new (as, keys) = do
  oldan <- Map.lookup old as
  newan <- Map.lookup new as
  return . (, keys) . Map.delete old . Map.insert new (combine oldan newan) $ as

combine :: Annotation -> Annotation -> Annotation
combine oldann newann =
  Ann (annEntryDelta oldann) (annDelta oldann) (annTrueEntryDelta oldann)
      (annPriorComments oldann ++ annPriorComments newann) (annsDP newann ++ extraComma (annsDP oldann))
  where
    extraComma [] = []
    extraComma (last -> x) = case x of
                              (G GHC.AnnComma, dp) -> [x]
                              _ -> []



mkKey :: (Data a) => GHC.Located a -> AnnKey
mkKey (GHC.L l s) = AnnKey l (annGetConstr s) NotNeeded


-- | Shift the first output annotation into the correct place
moveAnns :: [(KeywordId, DeltaPos)] -> [(KeywordId, DeltaPos)] -> [(KeywordId, DeltaPos)]
moveAnns [] xs        = xs
moveAnns ((_, dp): _) ((kw, _):xs) = (kw,dp) : xs

-- Pipe

runPipe :: FilePath -> IO ()
runPipe file = do
  path <- canonicalizePath file
  rawhints <- getHints path
{'fraction_non_alphanumeric': 0.09568044588945657, 'fraction_numerical': 0.0, 'mean_word_length': 3.620886981402003, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4904281', 'n_tokens_mistral': 2243, 'n_tokens_neox': 2186, 'n_words': 719}