<commit_before>package controllers

import java.util.UUID
import java.io.File
import java.net.URLDecoder

import scala.util.{Try, Success, Failure}
import scala.concurrent.{Promise, Future}
import scala.concurrent.duration._
import scala.reflect.ClassTag

import play.api._
import play.api.mvc._
import play.api.mvc.BodyParsers.parse._
import play.api.libs.functional.syntax._
import play.api.libs.iteratee._
import play.api.libs.iteratee.Concurrent.Channel
import play.api.libs.json._

import com.typesafe.config._

import akka.actor._
import akka.pattern.ask
import akka.util.Timeout

import notebook._
import notebook.server._
import notebook.kernel.remote._
import notebook.NBSerializer.Metadata

object AppUtils {
  import play.api.Play.current

  lazy val config               = NotebookConfig(current.configuration.getConfig("manager").get)
  lazy val nbm                  = new NotebookManager(config.projectName, config.notebooksDir)
  lazy val notebookServerConfig = current.configuration.getConfig("notebook-server").get.underlying
  lazy val clustersConf         = config.config.getConfig("clusters").get

  lazy val kernelSystem =  ActorSystem( "NotebookServer",
                                        notebookServerConfig,
                                        play.api.Play.classloader // this resolves the Play classloader problems w/ remoting
                                      )
}

case class Crumb(url:String="", name:String="")
case class Breadcrumbs(home:String="/", crumbs:List[Crumb] = Nil)


object Application extends Controller {

  lazy val config = AppUtils.config
  lazy val nbm = AppUtils.nbm
  lazy val notebookServerConfig = AppUtils.notebookServerConfig

  implicit def kernelSystem =  AppUtils.kernelSystem

  val kernelIdToCalcService = collection.mutable.Map[String, CalcWebSocketService]()

  val clustersActor = kernelSystem.actorOf(Props(NotebookClusters(AppUtils.clustersConf)))
  implicit val GetClustersTimeout = Timeout(60 seconds)

  val project = "Spark Notebook" //TODO from application.conf
  val base_project_url = "/"
  val base_kernel_url = "/"
  val base_observable_url = "observable" // TODO: Ugh...
  val read_only = false.toString
  val terminals_available = false.toString //TODO

  def configTree() = Action {
    Ok(Json.obj())
  }

  def configCommon() = Action {
    Ok(Json.obj())
  }

  def configNotebook() = Action {
    Ok(Json.obj())
  }

  def kernelSpecs() = Action {
    Ok(Json.parse(
        """
        |{
        |  "kernelspecs": {
        |    "spark": {
        |      "name": "spark",
        |      "resources": {},
        |      "spec" : {
        |        "language": "scala",
        |        "display_name": "Apache Spark",
        |
        |        "language_info": {
        |          "name" : "scala",
        |          "file_extension" : "scala",
        |          "codemirror_mode" : "text/x-scala"
        |        }
        |      }
        |    }
        |  }
        |}
        |""".stripMargin.trim
      )
    )
  }

  private [this] def newSession(kernelId:Option[String]=None, notebookPath:Option[String]=None) = {
    val kId = kernelId.getOrElse(UUID.randomUUID.toString)
    val compilerArgs = config.kernel.compilerArgs.toList
    val initScripts = config.kernel.initScripts.toList
    val kernel = new Kernel(config.kernel.config.underlying, kernelSystem)
    KernelManager.add(kId, kernel)

    val r = Reads.map[String]

    // Load the notebook → get the metadata
    val md:Option[Metadata] = for {
      p <- notebookPath
      n <- nbm.load(p)
      m <- n.metadata
    } yield m

    val customLocalRepo:Option[String] = md.flatMap(_.customLocalRepo)

    val customRepos:Option[List[String]] = md.flatMap(_.customRepos)

    val customDeps:Option[List[String]] = md.flatMap(_.customDeps)
<commit_msg>add versions in kernel display name<commit_after>package controllers

import java.util.UUID
import java.io.File
import java.net.URLDecoder

import scala.util.{Try, Success, Failure}
import scala.concurrent.{Promise, Future}
import scala.concurrent.duration._
import scala.reflect.ClassTag

import play.api._
import play.api.mvc._
import play.api.mvc.BodyParsers.parse._
import play.api.libs.functional.syntax._
import play.api.libs.iteratee._
import play.api.libs.iteratee.Concurrent.Channel
import play.api.libs.json._

import com.typesafe.config._

import akka.actor._
import akka.pattern.ask
import akka.util.Timeout

import notebook._
import notebook.server._
import notebook.kernel.remote._
import notebook.NBSerializer.Metadata

object AppUtils {
  import play.api.Play.current

  lazy val config               = NotebookConfig(current.configuration.getConfig("manager").get)
  lazy val nbm                  = new NotebookManager(config.projectName, config.notebooksDir)
  lazy val notebookServerConfig = current.configuration.getConfig("notebook-server").get.underlying
  lazy val clustersConf         = config.config.getConfig("clusters").get

  lazy val kernelSystem =  ActorSystem( "NotebookServer",
                                        notebookServerConfig,
                                        play.api.Play.classloader // this resolves the Play classloader problems w/ remoting
                                      )
}

case class Crumb(url:String="", name:String="")
case class Breadcrumbs(home:String="/", crumbs:List[Crumb] = Nil)


object Application extends Controller {

  lazy val config = AppUtils.config
  lazy val nbm = AppUtils.nbm
  lazy val notebookServerConfig = AppUtils.notebookServerConfig

  implicit def kernelSystem =  AppUtils.kernelSystem

  val kernelIdToCalcService = collection.mutable.Map[String, CalcWebSocketService]()

  val clustersActor = kernelSystem.actorOf(Props(NotebookClusters(AppUtils.clustersConf)))
  implicit val GetClustersTimeout = Timeout(60 seconds)

  val project = "Spark Notebook" //TODO from application.conf
  val base_project_url = "/"
  val base_kernel_url = "/"
  val base_observable_url = "observable" // TODO: Ugh...
  val read_only = false.toString
  val terminals_available = false.toString //TODO

  def configTree() = Action {
    Ok(Json.obj())
  }

  def configCommon() = Action {
    Ok(Json.obj())
  }

  def configNotebook() = Action {
    Ok(Json.obj())
  }

  val kernelDef = Json.parse(
        s"""
        |{
        |  "kernelspecs": {
        |    "spark": {
        |      "name": "spark",
        |      "resources": {},
        |      "spec" : {
        |        "language": "scala",
        |        "display_name": "Scala [${notebook.BuildInfo.scalaVersion}] Spark [${notebook.BuildInfo.xSparkVersion}] Hadoop [${notebook.BuildInfo.xHadoopVersion}]",
        |
        |        "language_info": {
        |          "name" : "scala",
        |          "file_extension" : "scala",
        |          "codemirror_mode" : "text/x-scala"
        |        }
        |      }
        |    }
        |  }
        |}
        |""".stripMargin.trim
      )

  def kernelSpecs() = Action { Ok(kernelDef) }

  private [this] def newSession(kernelId:Option[String]=None, notebookPath:Option[String]=None) = {
    val kId = kernelId.getOrElse(UUID.randomUUID.toString)
    val compilerArgs = config.kernel.compilerArgs.toList
    val initScripts = config.kernel.initScripts.toList
    val kernel = new Kernel(config.kernel.config.underlying, kernelSystem)
    KernelManager.add(kId, kernel)

    val r = Reads.map[String]

    // Load the notebook → get the metadata
    val md:Option[Metadata] = for {
      p <- notebookPath
      n <- nbm.load(p)
      m <- n.metadata
    } yield m

    val customLocalRepo:Option[String] = md.flatMap(_.customLocalRepo)

    val customRepos:Option[List[String]] = md.flatMap(_.customRepos)

    val customDeps:Option[List[String]] = md.flatMap(_.customDeps)

{'fraction_non_alphanumeric': 0.11209439528023599, 'fraction_numerical': 0.0005130178273695011, 'mean_word_length': 2.7168732125834127, 'pattern_counts': {'":': 14, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6180309', 'n_tokens_mistral': 2396, 'n_tokens_neox': 2331, 'n_words': 485}