<commit_before>{-|
Module      : Jupyter.Kernel
Description : Functions for creating and serving a Jupyter kernel.
Copyright   : (c) <NAME>, 2016
License     : MIT
Maintainer  : <EMAIL>
Stability   : stable
Portability : POSIX

The 'Jupyter.Kernel' module provides an API for quickly and easily creating Jupyter kernels.

Jupyter kernels are programs which communicate using the
<https://jupyter-client.readthedocs.io/en/latest/messaging.html Jupyter messaging spec>; most kernels
are language backends that allow using a particular programming language with Jupyter frontends such as
the <http://jupyter.org/ notebook> or <http://jupyter.org/qtconsole/stable/ QtConsole>.

To run a kernel, call the 'serve' function, which provides a type-safe implementation of the Jupyter
messaging spec.

More information about the client and kernel interfaces can be found on the @jupyter@
<https://github.com/gibiansky/jupyter-haskell README>, and several example kernels may be found in 
the <https://github.com/gibiansky/jupyter-haskell/tree/master/examples examples> directory.
-}

{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE PatternSynonyms #-}
{-# LANGUAGE OverloadedStrings #-}
module Jupyter.Kernel (
  -- * Defining a kernel
  CommHandler,
  ClientRequestHandler,
  simpleKernelInfo,

  -- * Reading Kernel Profiles
  KernelProfile(..),
  readProfile,

  -- * Serving kernels
  serve,
  serveWithDynamicPorts,
  KernelCallbacks(..), 
  defaultClientRequestHandler,
  defaultCommHandler,
  ) where

-- Imports from 'base'
import           Control.Exception (bracket, catch, finally)
import           Control.Monad (forever)
import           System.Exit (exitSuccess)
import           System.IO (hPutStrLn, stderr)

-- Imports from 'bytestring'
import           Data.ByteString (ByteString)

-- Imports from 'text'
import           Data.Text (Text)

-- Imports from 'mtl'
import           Control.Monad.IO.Class (MonadIO(..))

-- Imports from 'monad-control'
import           Control.Monad.Trans.Control (liftBaseWith)

-- Imports from 'async'
import           Control.Concurrent.Async (link2, waitAny, cancel)

-- Imports from 'zeromq4-haskell'
import           System.ZMQ4.Monadic (ZMQ, Socket, Rep, Router, Pub, async, receive, send)

-- Imports from 'jupyter'
import           Jupyter.Messages (KernelOutput, Comm, ClientRequest(..), KernelReply(..),
                                   pattern ExecuteOk, pattern InspectOk, pattern CompleteOk,
                                   CursorRange(..), CodeComplete(..), CodeOffset(..), ConnectInfo(..),
                                   KernelInfo(..), LanguageInfo(..), KernelOutput(..),
                                   KernelStatus(..), KernelRequest(..), ClientReply(..))
import           Jupyter.ZeroMQ (withKernelSockets, KernelSockets(..), sendMessage, receiveMessage,
                                 KernelProfile(..), readProfile, messagingError, mkReplyHeader,
                                 threadKilledHandler, readProfile)

-- | Create the simplest possible 'KernelInfo'.
--
-- Defaults version numbers to \"0.0\", mimetype to \"text/plain\", empty banner, and a \".txt\"
-- file extension.
--
-- Mostly intended for use in tutorials and demonstrations; if publishing production kernels, make
-- sure to use the full 'KernelInfo' constructor.
--
-- >>> let kernelInfo = simpleKernelInfo "python3"
simpleKernelInfo :: Text -- ^ Kernel name, used for 'kernelImplementation' and 'languageName'.
                 -> KernelInfo
simpleKernelInfo kernelName =
  KernelInfo
    { kernelProtocolVersion = "5.0"
    , kernelBanner = ""
    , kernelImplementation = kernelName
    , kernelImplementationVersion = "0.0"
    , kernelHelpLinks = []
    , kernelLanguageInfo = LanguageInfo
      { languageName = kernelName
      , languageVersion = "0.0"
      , languageMimetype = "text/plain"
      , languageFileExtension = ".txt"
      , languagePygmentsLexer = Nothing
      , languageCodeMirrorMode = Nothing
      , languageNbconvertExporter = Nothing
      }
    }


-- | The 'KernelCallbacks' data type contains callbacks that the kernel may use to communicate with
-- the client. Specifically, it can send 'KernelOutput' and 'Comm' messages using 'sendKernelOutput'
-- and 'sendComm', respectively, which are often sent to frontends in response to 'ExecuteRequest'
-- messsages.
--
-- In addition, 'sentKernelRequest' can be used to send a 'KernelRequest' to the client, and
-- synchronously wait and receive a 'ClientReply'.
data KernelCallbacks =
       KernelCallbacks
         { sendKernelOutput :: KernelOutput -> IO () -- ^ Publish an output to all connected
                                                     -- frontends. This is the primary mechanism by
                                                     -- which a kernel shows output to the user.
         , sendComm :: Comm -> IO () -- ^ Publish a 'Comm' message to the frontends. This allows for
                                     -- entirely freeform back-and-forth communication between
                                     -- frontends and kernels, avoiding the structure of the Jupyter
                                     -- messaging protocol. This can be used for implementing custom
                                     -- features such as support for the Jupyter notebook widgets.
         , sendKernelRequest :: KernelRequest -> IO ClientReply -- ^ Send a 'KernelRequest' to the
                                                                -- client that send the first message
                                                                -- and wait for it to reply with a
                                                                -- 'ClientReply'.
         }

-- | When calling 'serve', the caller must provide a 'CommHandler'.
--
-- The handler is used when the kernel receives a 'Comm' message from a frontend; the 'Comm' message
-- is passed to the handler, along with a set of callbacks the handler may use to send messages to
-- the client.
--
-- Since 'Comm's are used for free-form communication outside the messaging spec, kernels should
-- ignore 'Comm' messages they do not expect.
--
-- The 'defaultCommHandler' handler is provided for use with kernels that wish to ignore all 'Comm'
-- messages.
type CommHandler = KernelCallbacks -> Comm -> IO ()

-- | When calling 'serve', the caller must provide a 'ClientRequestHandler'.
--
-- The handler is used when the kernel receives a 'ClientRequest' message from a frontend; the
-- 'ClientRequest' message is passed to the handler, along with a set of callbacks the handler may
-- use to send messages to the client.
--
-- The handler must return a 'KernelReply' to be sent in response to this request. 'ClientRequest'
-- and 'KernelReply' constructors come in pairs, and the output reply constructor *must* match the
-- input request constructor.
--
-- Note: When the request is a 'ExecuteRequest' with the 'executeSilent' option set to @True@, the
-- 'KernelReply' will not be sent.
type ClientRequestHandler = KernelCallbacks -> ClientRequest -> IO KernelReply

-- | Handler which ignores all 'Comm' messages sent to the kernel (and does nothing).
defaultCommHandler :: CommHandler
defaultCommHandler _ _ = return ()

-- | Handler which responds to all 'ClientRequest' messages with a default, empty reply.
defaultClientRequestHandler :: KernelProfile -- ^ The profile this kernel is running on. Used to
                                             -- respond to 'ConnectRequest's.
                            -> KernelInfo    -- ^ Information about this kernel. Used to respond to
                                             -- 'KernelInfoRequest's.
                            -> ClientRequestHandler
defaultClientRequestHandler KernelProfile { .. } kernelInfo callbacks req =
  case req of
    ExecuteRequest code _ -> do
      sendKernelOutput callbacks $ ExecuteInputOutput 0 code
      return $ ExecuteReply 0 ExecuteOk
    InspectRequest{} -> return $ InspectReply $ InspectOk Nothing
    HistoryRequest{} -> return $ HistoryReply []
    CompleteRequest _ (CodeOffset offset) ->
      return $ CompleteReply $ CompleteOk [] (CursorRange offset offset) mempty
    IsCompleteRequest{} -> return $ IsCompleteReply CodeUnknown
    CommInfoRequest{} -> return $ CommInfoReply mempty
    ShutdownRequest restart -> return $ ShutdownReply restart
    KernelInfoRequest{} -> return $ KernelInfoReply kernelInfo
    ConnectRequest{} -> return $ ConnectReply
                                   ConnectInfo
                                     { connectShellPort = profileShellPort
                                     , connectIopubPort = profileIopubPort
                                     , connectHeartbeatPort = profileHeartbeatPort
                                     , connectStdinPort = profileStdinPort
                                     , connectControlPort = profileControlPort
                                     }

-- | Indefinitely serve a kernel on the provided ports. If the ports are not open, fails with an
-- exception.
--
-- This starts several threads which listen and write to ZeroMQ sockets on the ports indicated in
-- the 'KernelProfile'. If an exception is raised and any of the threads die, the exception is
-- re-raised on the main thread. Otherwise, this listens on the kernels indefinitely.
serve :: KernelProfile         -- ^ The kernel profile specifies how to listen for client messages (ports,
                               -- transport mechanism, message signing, etc).
      -> CommHandler           -- ^ The 'Comm' handler is called when 'Comm' messages are received from a
                               -- frontend.
      -> ClientRequestHandler  -- ^The request handler is called when 'ClientRequest' messages are<commit_msg>More documentation fixes<commit_after>{-|
Module      : Jupyter.Kernel
Description : Functions for creating and serving a Jupyter kernel.
Copyright   : (c) <NAME>, 2016
License     : MIT
Maintainer  : <EMAIL>
Stability   : stable
Portability : POSIX

The 'Jupyter.Kernel' module provides an API for quickly and easily creating Jupyter kernels.

Jupyter kernels are programs which communicate using the
<https://jupyter-client.readthedocs.io/en/latest/messaging.html Jupyter messaging spec>; most kernels
are language backends that allow using a particular programming language with Jupyter frontends such as
the <http://jupyter.org/ notebook> or <http://jupyter.org/qtconsole/stable/ QtConsole>.

To run a kernel, call the 'serve' function, which provides a type-safe implementation of the Jupyter
messaging spec.

More information about the client and kernel interfaces can be found on the @jupyter@
<https://github.com/gibiansky/jupyter-haskell README>, and several example kernels may be found in 
the <https://github.com/gibiansky/jupyter-haskell/tree/master/examples examples> directory.
-}

{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE PatternSynonyms #-}
{-# LANGUAGE OverloadedStrings #-}
module Jupyter.Kernel (
  -- * Serving kernels
  serve,
  serveDynamic,
  KernelCallbacks(..),

  -- * Defining a kernel
  simpleKernelInfo,
  ClientRequestHandler,
  defaultClientRequestHandler,
  CommHandler,
  defaultCommHandler,

  -- * Reading Kernel Profiles
  KernelProfile(..),
  readProfile,
  ) where

-- Imports from 'base'
import           Control.Exception (bracket, catch, finally)
import           Control.Monad (forever)
import           System.Exit (exitSuccess)
import           System.IO (hPutStrLn, stderr)

-- Imports from 'bytestring'
import           Data.ByteString (ByteString)

-- Imports from 'text'
import           Data.Text (Text)

-- Imports from 'mtl'
import           Control.Monad.IO.Class (MonadIO(..))

-- Imports from 'monad-control'
import           Control.Monad.Trans.Control (liftBaseWith)

-- Imports from 'async'
import           Control.Concurrent.Async (link2, waitAny, cancel)

-- Imports from 'zeromq4-haskell'
import           System.ZMQ4.Monadic (ZMQ, Socket, Rep, Router, Pub, async, receive, send)

-- Imports from 'jupyter'
import           Jupyter.Messages (KernelOutput, Comm, ClientRequest(..), KernelReply(..),
                                   pattern ExecuteOk, pattern InspectOk, pattern CompleteOk,
                                   CursorRange(..), CodeComplete(..), CodeOffset(..), ConnectInfo(..),
                                   KernelInfo(..), LanguageInfo(..), KernelOutput(..),
                                   KernelStatus(..), KernelRequest(..), ClientReply(..))
import           Jupyter.ZeroMQ (withKernelSockets, KernelSockets(..), sendMessage, receiveMessage,
                                 KernelProfile(..), readProfile, messagingError, mkReplyHeader,
                                 threadKilledHandler, readProfile)

-- | Create the simplest possible 'KernelInfo'.
--
-- Defaults version numbers to \"0.0\", mimetype to \"text/plain\", empty banner, and a \".txt\"
-- file extension.
--
-- Mostly intended for use in tutorials and demonstrations; if publishing production kernels, make
-- sure to use the full 'KernelInfo' constructor.
--
-- >>> let kernelInfo = simpleKernelInfo "python3"
simpleKernelInfo :: Text -- ^ Kernel name, used for 'kernelImplementation' and 'languageName'.
                 -> KernelInfo
simpleKernelInfo kernelName =
  KernelInfo
    { kernelProtocolVersion = "5.0"
    , kernelBanner = ""
    , kernelImplementation = kernelName
    , kernelImplementationVersion = "0.0"
    , kernelHelpLinks = []
    , kernelLanguageInfo = LanguageInfo
      { languageName = kernelName
      , languageVersion = "0.0"
      , languageMimetype = "text/plain"
      , languageFileExtension = ".txt"
      , languagePygmentsLexer = Nothing
      , languageCodeMirrorMode = Nothing
      , languageNbconvertExporter = Nothing
      }
    }


-- | The 'KernelCallbacks' data type contains callbacks that the kernel may use to communicate with
-- the client. Specifically, it can send 'KernelOutput' and 'Comm' messages using 'sendKernelOutput'
-- and 'sendComm', respectively, which are often sent to frontends in response to 'ExecuteRequest'
-- messsages.
--
-- In addition, 'sentKernelRequest' can be used to send a 'KernelRequest' to the client, and
-- synchronously wait and receive a 'ClientReply'.
data KernelCallbacks =
       KernelCallbacks
         { sendKernelOutput :: KernelOutput -> IO () -- ^ Publish an output to all connected
                                                     -- frontends. This is the primary mechanism by
                                                     -- which a kernel shows output to the user.
         , sendComm :: Comm -> IO () -- ^ Publish a 'Comm' message to the frontends. This allows for
                                     -- entirely freeform back-and-forth communication between
                                     -- frontends and kernels, avoiding the structure of the Jupyter
                                     -- messaging protocol. This can be used for implementing custom
                                     -- features such as support for the Jupyter notebook widgets.
         , sendKernelRequest :: KernelRequest -> IO ClientReply -- ^ Send a 'KernelRequest' to the
                                                                -- client that send the first message
                                                                -- and wait for it to reply with a
                                                                -- 'ClientReply'.
         }

-- | When calling 'serve', the caller must provide a 'CommHandler'.
--
-- The handler is used when the kernel receives a 'Comm' message from a frontend; the 'Comm' message
-- is passed to the handler, along with a set of callbacks the handler may use to send messages to
-- the client.
--
-- Since 'Comm's are used for free-form communication outside the messaging spec, kernels should
-- ignore 'Comm' messages they do not expect.
--
-- The 'defaultCommHandler' handler is provided for use with kernels that wish to ignore all 'Comm'
-- messages.
type CommHandler = KernelCallbacks -> Comm -> IO ()

-- | When calling 'serve', the caller must provide a 'ClientRequestHandler'.
--
-- The handler is used when the kernel receives a 'ClientRequest' message from a frontend; the
-- 'ClientRequest' message is passed to the handler, along with a set of callbacks the handler may
-- use to send messages to the client.
--
-- The handler must return a 'KernelReply' to be sent in response to this request. 'ClientRequest'
-- and 'KernelReply' constructors come in pairs, and the output reply constructor *must* match the
-- input request constructor.
--
-- Note: When the request is a 'ExecuteRequest' with the 'executeSilent' option set to @True@, the
-- 'KernelReply' will not be sent.
type ClientRequestHandler = KernelCallbacks -> ClientRequest -> IO KernelReply

-- | Handler which ignores all 'Comm' messages sent to the kernel (and does nothing).
defaultCommHandler :: CommHandler
defaultCommHandler _ _ = return ()

-- | Handler which responds to all 'ClientRequest' messages with a default, empty reply.
defaultClientRequestHandler :: KernelProfile -- ^ The profile this kernel is running on. Used to
                                             -- respond to 'ConnectRequest's.
                            -> KernelInfo    -- ^ Information about this kernel. Used to respond to
                                             -- 'KernelInfoRequest's.
                            -> ClientRequestHandler
defaultClientRequestHandler KernelProfile { .. } kernelInfo callbacks req =
  case req of
    ExecuteRequest code _ -> do
      sendKernelOutput callbacks $ ExecuteInputOutput 0 code
      return $ ExecuteReply 0 ExecuteOk
    InspectRequest{} -> return $ InspectReply $ InspectOk Nothing
    HistoryRequest{} -> return $ HistoryReply []
    CompleteRequest _ (CodeOffset offset) ->
      return $ CompleteReply $ CompleteOk [] (CursorRange offset offset) mempty
    IsCompleteRequest{} -> return $ IsCompleteReply CodeUnknown
    CommInfoRequest{} -> return $ CommInfoReply mempty
    ShutdownRequest restart -> return $ ShutdownReply restart
    KernelInfoRequest{} -> return $ KernelInfoReply kernelInfo
    ConnectRequest{} -> return $ ConnectReply
                                   ConnectInfo
                                     { connectShellPort = profileShellPort
                                     , connectIopubPort = profileIopubPort
                                     , connectHeartbeatPort = profileHeartbeatPort
                                     , connectStdinPort = profileStdinPort
                                     , connectControlPort = profileControlPort
                                     }

-- | Indefinitely serve a kernel on the provided ports. If the ports are not open, fails with an
-- exception.
--
-- This starts several threads which listen and write to ZeroMQ sockets on the ports indicated in
-- the 'KernelProfile'. If an exception is raised and any of the threads die, the exception is
-- re-raised on the main thread.
--
-- Using this function generally requires a bit of setup. The most common pattern for use is as follows:
--
-- 1. In your kernel @Main.hs@, parse command line arguments. Some combination of arguments should include
-- a path to a connection file; this file can be read and parsed into a 'KernelProfile' with 'readProfile'.
-- (Often, the same executable will have a different mode that installs the kernel
-- using 'Jupyter.Install.installKernelspec').
-- 2. Set up any state your kernel may need, storing it in an 'Control.Concurrent.MVar' or 'Data.IORef.IORef'.
-- 3. Define your 'CommHandler' and 'ClientRequestHandler' handlers, which read from the state and reply
-- with any necessary messages. (These handlers /may/ be called concurrently from different threads!)
-- 4. Provide the kernel profile and handlers to the 'serve' function, which blocks indefinitely.
--
-- Example kernels may be found in the
-- <https://github.com/gibiansky/jupyter-haskell/tree/master/examples examples> directory.
serve :: KernelProfile         -- ^ The kernel profile specifies how to listen for client messages (ports,
                               -- transport mechanism, message signing, etc).
      -> CommHandler           -- ^ The 'Comm' handler is called when 'Comm' messages are received from a
                               -- frontend.
      -> ClientRequestHandler  -- ^The request handler is called when 'ClientRequest' messages are
{'fraction_non_alphanumeric': 0.08433968669673603, 'fraction_numerical': 0.0019399582908967458, 'mean_word_length': 2.3199162775720494, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 66, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 68, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6305583', 'n_tokens_mistral': 5201, 'n_tokens_neox': 4991, 'n_words': 1939}