<commit_before>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.container.standalone

import com.google.inject.{Key, AbstractModule, Injector, Inject}
import com.yahoo.config.application.api.{DeployLogger, RuleConfigDeriver, FileRegistry, ApplicationPackage}
import com.yahoo.config.provision.Zone
import com.yahoo.jdisc.application.Application
import com.yahoo.container.jdisc.ConfiguredApplication
import java.io.{IOException, File}
import com.yahoo.config.model.test.MockRoot
import com.yahoo.config.model.application.provider._
import com.yahoo.vespa.defaults.Defaults
import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ManhattanContainerModelBuilder, ContainerModelBuilder}
import org.w3c.dom.Element
import com.yahoo.config.model.builder.xml.XmlHelper
import com.yahoo.vespa.model.container.Container
import com.yahoo.collections.CollectionUtil.first
import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder
import com.yahoo.io.IOUtils
import com.yahoo.container.di.config.SubscriberFactory
import StandaloneContainerApplication._
import com.google.inject.name.Names
import scala.util.Try
import java.nio.file.{FileSystems, Path, Paths, Files}
import com.yahoo.config.model.{ConfigModelRepo, ApplicationConfigProducerRoot}
import scala.collection.JavaConversions._
import com.yahoo.text.XML
import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking

import java.lang.{ Boolean => JBoolean }
import Environment._
import com.yahoo.config.model.deploy.DeployState

/**
 * @author tonytv
 * @author gjoranv
 */
class StandaloneContainerApplication @Inject()(injector: Injector) extends Application {

  ConfiguredApplication.ensureVespaLoggingInitialized()

  val applicationPath: Path = injectedApplicationPath.getOrElse(yinstApplicationPath)

  val distributedFiles = new LocalFileDb(applicationPath)

  val configModelRepo = Try { injector.getInstance(Key.get(classOf[ConfigModelRepo], configModelRepoName))}.getOrElse(new ConfigModelRepo)

  val networkingOption = Try {
    injector.getInstance(Key.get(classOf[JBoolean], Names.named(disableNetworkingAnnotation)))
  }.map  {
    case JBoolean.TRUE => Networking.disable
    case JBoolean.FALSE => Networking.enable
  }.getOrElse(Networking.enable)

  val (modelRoot, container) = withTempDir(
    preprocessedApplicationDir => createContainerModel(applicationPath, distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo))

  val configuredApplication = createConfiguredApplication(container)

  def createConfiguredApplication(container: Container): Application = {
    val augmentedInjector = injector.createChildInjector(new AbstractModule {
      def configure() {
        bind(classOf[SubscriberFactory]).toInstance(new StandaloneSubscriberFactory(modelRoot))
      }
    })

    System.setProperty("config.id", container.getConfigId) //TODO: DRY
    augmentedInjector.getInstance(classOf[ConfiguredApplication])
  }

  def injectedApplicationPath = Try {
    injector.getInstance(Key.get(classOf[Path], applicationPathName))
  }.toOption

  def yinstApplicationPath = path(yinstVariable(applicationLocationYinstVariable))

  override def start() {
    try {
      com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles)
      configuredApplication.start()
    }
    catch {
      case e: Exception => { com.yahoo.container.Container.resetInstance(); throw e; }
    }
  }

  override def stop() {
    configuredApplication.stop()
  }

  override def destroy() {
    com.yahoo.container.Container.resetInstance()
    configuredApplication.destroy()
  }
}

object StandaloneContainerApplication {
  val packageName = "standalone_jdisc_container"
  val applicationLocationYinstVariable = s"$packageName.app_location"
  val deploymentProfileYinstVariable = s"$packageName.deployment_profile"
  val manhattanHttpPortYinstVariable = s"$packageName.manhattan_http_port"

  val applicationPathName = Names.named(applicationLocationYinstVariable)

  val disableNetworkingAnnotation = "JDisc.disableNetworking"
  val configModelRepoName = Names.named("ConfigModelRepo")
  val configDefinitionRepo = new StaticConfigDefinitionRepo()

  val defaultTmpBaseDir = Defaults.getDefaults().underVespaHome("tmp")
  val tmpDirName = "standalone_container"

  private def withTempDir[T](f: File => T): T = {
    val tmpDir = createTempDir()
    try {
      f(tmpDir)
    } finally {
      IOUtils.recursiveDeleteDir(tmpDir)
    }
  }

  private def createTempDir(): File = {
    def getBaseDir: Path = {
      val tmpBaseDir =
        if (new File(defaultTmpBaseDir).exists())
          defaultTmpBaseDir
        else
          System.getProperty("java.io.tmpdir")

      Paths.get(tmpBaseDir)
    }

    val basePath: Path = getBaseDir
    val tmpDir = Files.createTempDirectory(basePath, tmpDirName)
    tmpDir.toFile
  }

  private def validateApplication(applicationPackage: ApplicationPackage, logger: DeployLogger) = {
    try {
      applicationPackage.validateXML(logger)
    } catch {
      case e: IOException => throw new IllegalArgumentException(e)
    }
  }

  def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = {
    optionalYinstVariable(deploymentProfileYinstVariable) match {
      case None => new ContainerModelBuilder(true, networkingOption)
      case Some("manhattan") => new ManhattanContainerModelBuilder(manhattanHttpPort)
      case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigYinstVariables)
      case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'")
    }
  }

  def manhattanHttpPort: Int = {
    val port = yinstVariable(manhattanHttpPortYinstVariable)
    Try {
      Integer.parseInt(port)
    } filter( _ > 0) getOrElse {
      throw new RuntimeException(s"$manhattanHttpPortYinstVariable is not a valid port: '$port'")
    }
  }

  def createContainerModel(applicationPath: Path,
                           fileRegistry: FileRegistry,
                           preprocessedApplicationDir: File,
                           networkingOption: Networking,
                           configModelRepo: ConfigModelRepo = new ConfigModelRepo): (MockRoot, Container) = {
    val logger = new BaseDeployLogger
    val rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile).preprocessedDir(preprocessedApplicationDir).build()
    // TODO: Needed until we get rid of semantic rules
    val applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), new RuleConfigDeriver {
      override def derive(ruleBaseDir: String, outputDir: String): Unit = {}
    }, logger)
    validateApplication(applicationPackage, logger)
    val deployState = new DeployState.Builder().
      applicationPackage(applicationPackage).
      fileRegistry(fileRegistry).
      deployLogger(logger).
      configDefinitionRepo(configDefinitionRepo).
      build()

    val root = new MockRoot("", deployState)
    val vespaRoot = new ApplicationConfigProducerRoot(root,
      "vespa",
      deployState.getDocumentModel,
      deployState.getProperties.vespaVersion(),
      deployState.getProperties.applicationId())

    val spec = containerRootElement(applicationPackage)
    val containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec)
    containerModel.getCluster().prepare()
    containerModel.initialize(configModelRepo)
    val container = first(containerModel.getCluster().getContainers)<commit_msg>Allow source directories in standalone application.<commit_after>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.container.standalone

import com.google.inject.{Key, AbstractModule, Injector, Inject}
import com.yahoo.config.application.api.{DeployLogger, RuleConfigDeriver, FileRegistry, ApplicationPackage}
import com.yahoo.config.provision.Zone
import com.yahoo.jdisc.application.Application
import com.yahoo.container.jdisc.ConfiguredApplication
import java.io.{IOException, File}
import com.yahoo.config.model.test.MockRoot
import com.yahoo.config.model.application.provider._
import com.yahoo.vespa.defaults.Defaults
import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ManhattanContainerModelBuilder, ContainerModelBuilder}
import org.w3c.dom.Element
import com.yahoo.config.model.builder.xml.XmlHelper
import com.yahoo.vespa.model.container.Container
import com.yahoo.collections.CollectionUtil.first
import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder
import com.yahoo.io.IOUtils
import com.yahoo.container.di.config.SubscriberFactory
import StandaloneContainerApplication._
import com.google.inject.name.Names
import scala.util.Try
import java.nio.file.{FileSystems, Path, Paths, Files}
import com.yahoo.config.model.{ConfigModelRepo, ApplicationConfigProducerRoot}
import scala.collection.JavaConversions._
import com.yahoo.text.XML
import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking

import java.lang.{ Boolean => JBoolean }
import Environment._
import com.yahoo.config.model.deploy.DeployState

/**
 * @author tonytv
 * @author gjoranv
 */
class StandaloneContainerApplication @Inject()(injector: Injector) extends Application {

  ConfiguredApplication.ensureVespaLoggingInitialized()

  val applicationPath: Path = injectedApplicationPath.getOrElse(yinstApplicationPath)

  val distributedFiles = new LocalFileDb(applicationPath)

  val configModelRepo = Try { injector.getInstance(Key.get(classOf[ConfigModelRepo], configModelRepoName))}.getOrElse(new ConfigModelRepo)

  val networkingOption = Try {
    injector.getInstance(Key.get(classOf[JBoolean], Names.named(disableNetworkingAnnotation)))
  }.map  {
    case JBoolean.TRUE => Networking.disable
    case JBoolean.FALSE => Networking.enable
  }.getOrElse(Networking.enable)

  val (modelRoot, container) = withTempDir(
    preprocessedApplicationDir => createContainerModel(applicationPath, distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo))

  val configuredApplication = createConfiguredApplication(container)

  def createConfiguredApplication(container: Container): Application = {
    val augmentedInjector = injector.createChildInjector(new AbstractModule {
      def configure() {
        bind(classOf[SubscriberFactory]).toInstance(new StandaloneSubscriberFactory(modelRoot))
      }
    })

    System.setProperty("config.id", container.getConfigId) //TODO: DRY
    augmentedInjector.getInstance(classOf[ConfiguredApplication])
  }

  def injectedApplicationPath = Try {
    injector.getInstance(Key.get(classOf[Path], applicationPathName))
  }.toOption

  def yinstApplicationPath = path(yinstVariable(applicationLocationYinstVariable))

  override def start() {
    try {
      com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles)
      configuredApplication.start()
    }
    catch {
      case e: Exception => { com.yahoo.container.Container.resetInstance(); throw e; }
    }
  }

  override def stop() {
    configuredApplication.stop()
  }

  override def destroy() {
    com.yahoo.container.Container.resetInstance()
    configuredApplication.destroy()
  }
}

object StandaloneContainerApplication {
  val packageName = "standalone_jdisc_container"
  val applicationLocationYinstVariable = s"$packageName.app_location"
  val deploymentProfileYinstVariable = s"$packageName.deployment_profile"
  val manhattanHttpPortYinstVariable = s"$packageName.manhattan_http_port"

  val applicationPathName = Names.named(applicationLocationYinstVariable)

  val disableNetworkingAnnotation = "JDisc.disableNetworking"
  val configModelRepoName = Names.named("ConfigModelRepo")
  val configDefinitionRepo = new StaticConfigDefinitionRepo()

  val defaultTmpBaseDir = Defaults.getDefaults().underVespaHome("tmp")
  val tmpDirName = "standalone_container"

  private def withTempDir[T](f: File => T): T = {
    val tmpDir = createTempDir()
    try {
      f(tmpDir)
    } finally {
      IOUtils.recursiveDeleteDir(tmpDir)
    }
  }

  private def createTempDir(): File = {
    def getBaseDir: Path = {
      val tmpBaseDir =
        if (new File(defaultTmpBaseDir).exists())
          defaultTmpBaseDir
        else
          System.getProperty("java.io.tmpdir")

      Paths.get(tmpBaseDir)
    }

    val basePath: Path = getBaseDir
    val tmpDir = Files.createTempDirectory(basePath, tmpDirName)
    tmpDir.toFile
  }

  private def validateApplication(applicationPackage: ApplicationPackage, logger: DeployLogger) = {
    try {
      applicationPackage.validateXML(logger)
    } catch {
      case e: IOException => throw new IllegalArgumentException(e)
    }
  }

  def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = {
    optionalYinstVariable(deploymentProfileYinstVariable) match {
      case None => new ContainerModelBuilder(true, networkingOption)
      case Some("manhattan") => new ManhattanContainerModelBuilder(manhattanHttpPort)
      case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigYinstVariables)
      case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'")
    }
  }

  def manhattanHttpPort: Int = {
    val port = yinstVariable(manhattanHttpPortYinstVariable)
    Try {
      Integer.parseInt(port)
    } filter( _ > 0) getOrElse {
      throw new RuntimeException(s"$manhattanHttpPortYinstVariable is not a valid port: '$port'")
    }
  }

  def createContainerModel(applicationPath: Path,
                           fileRegistry: FileRegistry,
                           preprocessedApplicationDir: File,
                           networkingOption: Networking,
                           configModelRepo: ConfigModelRepo = new ConfigModelRepo): (MockRoot, Container) = {
    val logger = new BaseDeployLogger
    val rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile).includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build()
    // TODO: Needed until we get rid of semantic rules
    val applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), new RuleConfigDeriver {
      override def derive(ruleBaseDir: String, outputDir: String): Unit = {}
    }, logger)
    validateApplication(applicationPackage, logger)
    val deployState = new DeployState.Builder().
      applicationPackage(applicationPackage).
      fileRegistry(fileRegistry).
      deployLogger(logger).
      configDefinitionRepo(configDefinitionRepo).
      build()

    val root = new MockRoot("", deployState)
    val vespaRoot = new ApplicationConfigProducerRoot(root,
      "vespa",
      deployState.getDocumentModel,
      deployState.getProperties.vespaVersion(),
      deployState.getProperties.applicationId())

    val spec = containerRootElement(applicationPackage)
    val containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec)
    containerModel.getCluster().prepare()
    containerModel.initialize(configModelRepo)
    val container = first(containerModel.getCluster().getContainers)
{'fraction_non_alphanumeric': 0.08848287112561175, 'fraction_numerical': 0.0010440456769983688, 'mean_word_length': 4.638704930095659, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '382355', 'n_tokens_mistral': 4214, 'n_tokens_neox': 4104, 'n_words': 822}