<commit_before>package com.ibm.spark

import akka.actor.{Props, ActorSystem, ActorRef}
import com.ibm.spark.dependencies.{IvyDependencyDownloader, DependencyDownloader}
import com.ibm.spark.interpreter._
import com.ibm.spark.kernel.protocol.v5.KernelStatusType._
import com.ibm.spark.kernel.protocol.v5.MessageType.MessageType
import com.ibm.spark.kernel.protocol.v5.SocketType.SocketType
import com.ibm.spark.kernel.protocol.v5._
import com.ibm.spark.kernel.protocol.v5.dispatch.StatusDispatch
import com.ibm.spark.kernel.protocol.v5.handler._
import com.ibm.spark.kernel.protocol.v5.interpreter.InterpreterActor
import com.ibm.spark.kernel.protocol.v5.interpreter.tasks.InterpreterTaskFactory
import com.ibm.spark.kernel.protocol.v5.magic.MagicManager
import com.ibm.spark.kernel.protocol.v5.relay.{ExecuteRequestRelay, KernelMessageRelay}
import com.ibm.spark.kernel.protocol.v5.security.SignatureManagerActor
import com.ibm.spark.kernel.protocol.v5.socket._
import com.ibm.spark.kernel.protocol.v5.stream.KernelMessageStream
import com.ibm.spark.magic.MagicLoader
import com.ibm.spark.magic.builtin.BuiltinLoader
import com.ibm.spark.magic.dependencies.DependencyMap
import com.ibm.spark.security.KernelSecurityManager
import com.ibm.spark.utils.{GlobalStreamState, LogLike}
import com.typesafe.config.Config
import org.apache.spark.{SparkConf, SparkContext}
import collection.JavaConverters._
import play.api.libs.json.Json

case class SparkKernelBootstrap(config: Config) extends LogLike {

  private val DefaultAppName                          = SparkKernelInfo.banner
  private val DefaultActorSystemName                  = "spark-kernel-actor-system"

  private var socketFactory: ServerSocketFactory            = _
  private var heartbeatActor: ActorRef                = _
  private var shellActor: ActorRef                    = _
  private var ioPubActor: ActorRef                    = _

  protected[spark] var interpreter: Interpreter       = _
  protected[spark] var sparkContext: SparkContext     = _
  protected[spark] var dependencyDownloader: DependencyDownloader = _
  private var dependencyMap: DependencyMap            = _
  private var builtinLoader: BuiltinLoader            = _
  private var magicLoader: MagicLoader                = _

  private var actorSystem: ActorSystem                = _
  private var actorLoader: ActorLoader                = _
  private var interpreterActor: ActorRef              = _
  private var kernelMessageRelayActor: ActorRef       = _
  private var executeRequestRelayActor: ActorRef      = _
  private var signatureManagerActor: ActorRef         = _
  private var magicManagerActor: ActorRef             = _
  private var statusDispatch: ActorRef                = _

  /**
   * Initializes all kernel systems.
   */
  def initialize() = {
    // TODO: Investigate potential to initialize System out/err/in to capture
    //       Console DynamicVariable initialization (since takes System fields)
    //       and redirect it to a workable location (like an actor) with the
    //       thread's current information attached
    //
    // E.G. System.setOut(customPrintStream) ... all new threads will have
    //      customPrintStream as their initial Console.out value
    //

    initializeBareComponents()
    createSockets()
    initializeKernelHandlers()
    publishStatus(KernelStatusType.Starting, Some(null))
    initializeInterpreter()
    initializeSparkContext()
    initializeDependencyDownloader()
    initializeMagicLoader()
    initializeSystemActors()
    registerInterruptHook()
    registerShutdownHook()

    logger.info("Initializing security manager")
    System.setSecurityManager(new KernelSecurityManager)

    logger.info("Marking relay as ready for receiving messages")
    kernelMessageRelayActor ! true

    this
  }

  /**
   * Shuts down all kernel systems.<commit_msg>Remove blasphemous Some(null)<commit_after>package com.ibm.spark

import akka.actor.{Props, ActorSystem, ActorRef}
import com.ibm.spark.dependencies.{IvyDependencyDownloader, DependencyDownloader}
import com.ibm.spark.interpreter._
import com.ibm.spark.kernel.protocol.v5.KernelStatusType._
import com.ibm.spark.kernel.protocol.v5.MessageType.MessageType
import com.ibm.spark.kernel.protocol.v5.SocketType.SocketType
import com.ibm.spark.kernel.protocol.v5._
import com.ibm.spark.kernel.protocol.v5.dispatch.StatusDispatch
import com.ibm.spark.kernel.protocol.v5.handler._
import com.ibm.spark.kernel.protocol.v5.interpreter.InterpreterActor
import com.ibm.spark.kernel.protocol.v5.interpreter.tasks.InterpreterTaskFactory
import com.ibm.spark.kernel.protocol.v5.magic.MagicManager
import com.ibm.spark.kernel.protocol.v5.relay.{ExecuteRequestRelay, KernelMessageRelay}
import com.ibm.spark.kernel.protocol.v5.security.SignatureManagerActor
import com.ibm.spark.kernel.protocol.v5.socket._
import com.ibm.spark.kernel.protocol.v5.stream.KernelMessageStream
import com.ibm.spark.magic.MagicLoader
import com.ibm.spark.magic.builtin.BuiltinLoader
import com.ibm.spark.magic.dependencies.DependencyMap
import com.ibm.spark.security.KernelSecurityManager
import com.ibm.spark.utils.{GlobalStreamState, LogLike}
import com.typesafe.config.Config
import org.apache.spark.{SparkConf, SparkContext}
import collection.JavaConverters._
import play.api.libs.json.Json

case class SparkKernelBootstrap(config: Config) extends LogLike {

  private val DefaultAppName                          = SparkKernelInfo.banner
  private val DefaultActorSystemName                  = "spark-kernel-actor-system"

  private var socketFactory: ServerSocketFactory            = _
  private var heartbeatActor: ActorRef                = _
  private var shellActor: ActorRef                    = _
  private var ioPubActor: ActorRef                    = _

  protected[spark] var interpreter: Interpreter       = _
  protected[spark] var sparkContext: SparkContext     = _
  protected[spark] var dependencyDownloader: DependencyDownloader = _
  private var dependencyMap: DependencyMap            = _
  private var builtinLoader: BuiltinLoader            = _
  private var magicLoader: MagicLoader                = _

  private var actorSystem: ActorSystem                = _
  private var actorLoader: ActorLoader                = _
  private var interpreterActor: ActorRef              = _
  private var kernelMessageRelayActor: ActorRef       = _
  private var executeRequestRelayActor: ActorRef      = _
  private var signatureManagerActor: ActorRef         = _
  private var magicManagerActor: ActorRef             = _
  private var statusDispatch: ActorRef                = _

  /**
   * Initializes all kernel systems.
   */
  def initialize() = {
    // TODO: Investigate potential to initialize System out/err/in to capture
    //       Console DynamicVariable initialization (since takes System fields)
    //       and redirect it to a workable location (like an actor) with the
    //       thread's current information attached
    //
    // E.G. System.setOut(customPrintStream) ... all new threads will have
    //      customPrintStream as their initial Console.out value
    //

    initializeBareComponents()
    createSockets()
    initializeKernelHandlers()
    publishStatus(KernelStatusType.Starting)
    initializeInterpreter()
    initializeSparkContext()
    initializeDependencyDownloader()
    initializeMagicLoader()
    initializeSystemActors()
    registerInterruptHook()
    registerShutdownHook()

    logger.info("Initializing security manager")
    System.setSecurityManager(new KernelSecurityManager)

    logger.info("Marking relay as ready for receiving messages")
    kernelMessageRelayActor ! true

    this
  }

  /**
   * Shuts down all kernel systems.
{'fraction_non_alphanumeric': 0.07755049197307094, 'fraction_numerical': 0.0033661315380631796, 'mean_word_length': 3.344769403824522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5562356', 'n_tokens_mistral': 2224, 'n_tokens_neox': 2076, 'n_words': 454}