<commit_before>{-# LANGUAGE DeriveGeneric #-}
module Main where

import Data.Aeson
import qualified Data.ByteString.Lazy.Char8 as LBSC
import Data.Either
import qualified Data.Map.Strict as Map
import GHC.Generics

import NanoML.Classify
import NanoML.Parser
import NanoML.Pretty
import NanoML.Types


main :: IO ()
main = interact (unlines . map generateDiff . lines)

generateDiff :: String -> String
generateDiff json = case eitherDecode (LBSC.pack json) of
  Left e -> error e
  Right (MkInSample bads' [fix'])
    | Right fix <- parseTopForm fix'
    , bads <- rights $ map parseTopForm bads'
    -> unlines . map (LBSC.unpack . encode . diffOne fix) $ bads

diffOne :: Prog -> Prog -> OutSample
diffOne fix bad = MkOutSample (render $ prettyProg bad) bad (diffProg bad fix)

data InSample = MkInSample { bad :: [String], fix :: [String] }
  deriving (Show, Generic)
instance FromJSON InSample

data OutSample = MkOutSample { prog :: String, ast :: Prog, loc :: MSrcSpan }
  deriving (Show, Generic)
instance ToJSON OutSample
<commit_msg>update generateDiff<commit_after>{-# LANGUAGE DeriveGeneric #-}
module Main where

import Data.Aeson
import qualified Data.ByteString.Lazy.Char8 as LBSC
import Data.Either
import qualified Data.Map.Strict as Map
import GHC.Generics

import NanoML.Classify
import NanoML.Parser
import NanoML.Pretty
import NanoML.Types


main :: IO ()
main = interact (unlines
                 -- TODO: add runtime flag to choose classifier?
                 . map (generateDiff by_ops)
                 . lines)

generateDiff :: (ToJSON a, Monoid a)
             => (Expr -> a) -> String -> String
generateDiff mkfs json = case eitherDecode (LBSC.pack json) of
  Left e -> error e
  Right (MkInSample bads' [fix'])
    | Right fix <- parseTopForm fix'
    , bads <- rights $ map parseTopForm bads'
    -> unlines . map (LBSC.unpack . encode . diffOne mkfs fix) $ bads

diffOne :: Monoid a => (Expr -> a) -> Prog -> Prog -> OutSample a
diffOne mkfs fix bad = MkOutSample
  { prog = render (prettyProg bad)
  , ast  = bad
  , loc  = diffProg bad fix
  , features = mconcat (map mkfsD bad)
  }
  where
  mkfsD (DFun _ _ pes) = mconcat (map (mkfs.snd) pes)
  mkfsD (DEvl _ e) = mkfs e
  mkfsD _ = mempty

data InSample = MkInSample { bad :: [String], fix :: [String] }
  deriving (Show, Generic)
instance FromJSON InSample

data OutSample a = MkOutSample
  { prog :: String
  , ast :: Prog
  , loc :: MSrcSpan
  , features :: a
  } deriving (Show, Generic)
instance ToJSON a => ToJSON (OutSample a)

{'fraction_non_alphanumeric': 0.10392701309004364, 'fraction_numerical': 0.0007933359777865926, 'mean_word_length': 3.2529510961214165, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6090764', 'n_tokens_mistral': 849, 'n_tokens_neox': 817, 'n_words': 282}