<commit_before>//
// DistribTestCase.scala -- Scala class DistribTestCase
// Project OrcTests
//
// Created by jthywiss on Jul 29, 2017.
//
// Copyright (c) 2017 The University of Texas at Austin. All rights reserved.
//
// Use and redistribution of this file is governed by the license terms in
// the LICENSE file found in the project's top-level directory and also found at
// URL: http://orc.csres.utexas.edu/license.shtml .
//

package orc.test.proc

import java.io.{ File, FileOutputStream }
import java.net.{ InetAddress, InetSocketAddress, NetworkInterface, SocketException }

import scala.collection.JavaConverters.seqAsJavaListConverter

import orc.error.compiletime.{ CompilationException, FeatureNotSupportedException }
import orc.test.util.{ OsCommand, OsCommandResult, TestRunNumber, TestUtils }
import orc.test.util.TestUtils.OrcTestCase

import org.junit.Assume

/** JUnit test case for a distributed-Orc test.
  *
  * @author jthywiss
  */
class DistribTestCase extends OrcTestCase {

  @throws(classOf[Throwable])
  override def runTest() {
    val options = bindings.asInstanceOf[orc.Main.OrcCmdLineOptions]
    startFollowers()
    println("\n==== Starting " + orcFile + " ====")
    val leaderOpts = DistribTestConfig.expanded.getIterableFor("leaderOpts").getOrElse(Seq())
    val followerSockets = options.recognizedLongOpts("follower-sockets").getValue
    val outFilenamePrefix = orcFile.getName.stripSuffix(".orc")
    val runOutputDir = DistribTestConfig.expanded("runOutputDir")
    val leaderOutFile = new File(runOutputDir, s"${outFilenamePrefix}_0.out")
    val leaderErrFile = new File(runOutputDir, s"${outFilenamePrefix}_0.err")
    try {
      val actual = if (DistribTestCase.leaderSpec.isLocal) {
        //OrcForTesting.compileAndRun(orcFile.getPath(), 200 /*s*/, bindings)
        val result = OsCommand.getResultFrom(Seq(DistribTestCase.leaderSpec.javaCmd, "-cp", DistribTestCase.leaderSpec.classPath) ++ DistribTestCase.leaderSpec.jvmOptions ++ Seq(s"-Dorc.executionlog.fileprefix=${outFilenamePrefix}_", "-Dorc.executionlog.filesuffix=_0", DistribTestConfig.expanded("leaderClass")) ++ leaderOpts.toSeq ++ Seq(s"--follower-sockets=$followerSockets", orcFile.getPath), directory = new File(DistribTestCase.leaderSpec.workingDir), teeStdOutErr = true, stdoutTee = Seq(System.out, new FileOutputStream(leaderOutFile)), stderrTee = Seq(System.err, new FileOutputStream(leaderErrFile)))
        result.stdout
      } else {
        val result = OsCommand.getResultFrom(Seq("ssh", DistribTestCase.leaderSpec.hostname, s"cd '${DistribTestCase.leaderSpec.workingDir}'; { { '${DistribTestCase.leaderSpec.javaCmd}' -cp '${DistribTestCase.leaderSpec.classPath}' ${DistribTestCase.leaderSpec.jvmOptions.mkString(" ")} -Dorc.executionlog.fileprefix=${outFilenamePrefix}_ -Dorc.executionlog.filesuffix=_0 ${DistribTestConfig.expanded("leaderClass")} ${leaderOpts.mkString(" ")} --follower-sockets=$followerSockets '$orcFile' | tee '$leaderOutFile'; } 2>&1 1>&3 | tee '$leaderErrFile'; } 3>&1 1>&2"), teeStdOutErr = true)
        result.stdout
      }
      if (!expecteds.contains(actual)) {
        throw new AssertionError("Unexpected output:\n" + actual)
      }
    } catch {
      case fnse: FeatureNotSupportedException => Assume.assumeNoException(fnse)
      case ce: CompilationException => throw new AssertionError(ce.getMessageAndDiagnostics())
    } finally {
      println()
      Thread.sleep(500 /*ms*/)
      stopFollowers()
      Thread.sleep(500 /*ms*/)
      println()
    }
  }

  def startFollowers() {
    val outFilenamePrefix = orcFile.getName.stripSuffix(".orc")
    val runOutputDir = DistribTestConfig.expanded("runOutputDir")

    for (followerNumber <- 1 to DistribTestCase.followerSpecs.size) {
      val followerSpec = DistribTestCase.followerSpecs(followerNumber - 1)
      val followerOpts = DistribTestConfig.expanded.getIterableFor("followerOpts").getOrElse(Seq())
      val followerWorkingDir = new File(followerSpec.workingDir)
      val followerOutFile = new File(runOutputDir, s"${outFilenamePrefix}_$followerNumber.out")
      val followerErrFile = new File(runOutputDir, s"${outFilenamePrefix}_$followerNumber.err")

      if (followerSpec.isLocal) {
        println(s"Launching follower $followerNumber on port ${followerSpec.port}")
        val command = Seq(followerSpec.javaCmd, "-cp", followerSpec.classPath) ++ followerSpec.jvmOptions ++ Seq(DistribTestConfig.expanded("followerClass")) ++ followerOpts.toSeq ++ Seq(s"-Dorc.executionlog.fileprefix=${outFilenamePrefix}_", s"-Dorc.executionlog.filesuffix=_$followerNumber", followerNumber.toString, followerSpec.hostname+":"+followerSpec.port)
        OsCommand.runNoWait(command, directory = followerWorkingDir, stdout = followerOutFile, stderr = followerErrFile)
      } else {
        println(s"Launching follower $followerNumber on ${followerSpec.hostname}:${followerSpec.port}")
        /* FIXME: Escape strings for shell */
        val command = Seq("ssh", followerSpec.hostname, s"cd '${followerSpec.workingDir}'; '${followerSpec.javaCmd}' -cp '${followerSpec.classPath}' ${followerSpec.jvmOptions.mkString(" ")} -Dorc.executionlog.fileprefix=${outFilenamePrefix}_ -Dorc.executionlog.filesuffix=_$followerNumber ${DistribTestConfig.expanded("followerClass")} ${followerOpts.mkString(" ")} $followerNumber ${followerSpec.hostname}:${followerSpec.port} >'$followerOutFile' 2>'$followerErrFile'")<commit_msg>Fix command line syntax error when starting local followers<commit_after>//
// DistribTestCase.scala -- Scala class DistribTestCase
// Project OrcTests
//
// Created by jthywiss on Jul 29, 2017.
//
// Copyright (c) 2017 The University of Texas at Austin. All rights reserved.
//
// Use and redistribution of this file is governed by the license terms in
// the LICENSE file found in the project's top-level directory and also found at
// URL: http://orc.csres.utexas.edu/license.shtml .
//

package orc.test.proc

import java.io.{ File, FileOutputStream }
import java.net.{ InetAddress, InetSocketAddress, NetworkInterface, SocketException }

import scala.collection.JavaConverters.seqAsJavaListConverter

import orc.error.compiletime.{ CompilationException, FeatureNotSupportedException }
import orc.test.util.{ OsCommand, OsCommandResult, TestRunNumber, TestUtils }
import orc.test.util.TestUtils.OrcTestCase

import org.junit.Assume

/** JUnit test case for a distributed-Orc test.
  *
  * @author jthywiss
  */
class DistribTestCase extends OrcTestCase {

  @throws(classOf[Throwable])
  override def runTest() {
    val options = bindings.asInstanceOf[orc.Main.OrcCmdLineOptions]
    startFollowers()
    println("\n==== Starting " + orcFile + " ====")
    val leaderOpts = DistribTestConfig.expanded.getIterableFor("leaderOpts").getOrElse(Seq())
    val followerSockets = options.recognizedLongOpts("follower-sockets").getValue
    val outFilenamePrefix = orcFile.getName.stripSuffix(".orc")
    val runOutputDir = DistribTestConfig.expanded("runOutputDir")
    val leaderOutFile = new File(runOutputDir, s"${outFilenamePrefix}_0.out")
    val leaderErrFile = new File(runOutputDir, s"${outFilenamePrefix}_0.err")
    try {
      val actual = if (DistribTestCase.leaderSpec.isLocal) {
        //OrcForTesting.compileAndRun(orcFile.getPath(), 200 /*s*/, bindings)
        val result = OsCommand.getResultFrom(Seq(DistribTestCase.leaderSpec.javaCmd, "-cp", DistribTestCase.leaderSpec.classPath) ++ DistribTestCase.leaderSpec.jvmOptions ++ Seq(s"-Dorc.executionlog.fileprefix=${outFilenamePrefix}_", "-Dorc.executionlog.filesuffix=_0", DistribTestConfig.expanded("leaderClass")) ++ leaderOpts.toSeq ++ Seq(s"--follower-sockets=$followerSockets", orcFile.getPath), directory = new File(DistribTestCase.leaderSpec.workingDir), teeStdOutErr = true, stdoutTee = Seq(System.out, new FileOutputStream(leaderOutFile)), stderrTee = Seq(System.err, new FileOutputStream(leaderErrFile)))
        result.stdout
      } else {
        val result = OsCommand.getResultFrom(Seq("ssh", DistribTestCase.leaderSpec.hostname, s"cd '${DistribTestCase.leaderSpec.workingDir}'; { { '${DistribTestCase.leaderSpec.javaCmd}' -cp '${DistribTestCase.leaderSpec.classPath}' ${DistribTestCase.leaderSpec.jvmOptions.mkString(" ")} -Dorc.executionlog.fileprefix=${outFilenamePrefix}_ -Dorc.executionlog.filesuffix=_0 ${DistribTestConfig.expanded("leaderClass")} ${leaderOpts.mkString(" ")} --follower-sockets=$followerSockets '$orcFile' | tee '$leaderOutFile'; } 2>&1 1>&3 | tee '$leaderErrFile'; } 3>&1 1>&2"), teeStdOutErr = true)
        result.stdout
      }
      if (!expecteds.contains(actual)) {
        throw new AssertionError("Unexpected output:\n" + actual)
      }
    } catch {
      case fnse: FeatureNotSupportedException => Assume.assumeNoException(fnse)
      case ce: CompilationException => throw new AssertionError(ce.getMessageAndDiagnostics())
    } finally {
      println()
      Thread.sleep(500 /*ms*/)
      stopFollowers()
      Thread.sleep(500 /*ms*/)
      println()
    }
  }

  def startFollowers() {
    val outFilenamePrefix = orcFile.getName.stripSuffix(".orc")
    val runOutputDir = DistribTestConfig.expanded("runOutputDir")

    for (followerNumber <- 1 to DistribTestCase.followerSpecs.size) {
      val followerSpec = DistribTestCase.followerSpecs(followerNumber - 1)
      val followerOpts = DistribTestConfig.expanded.getIterableFor("followerOpts").getOrElse(Seq())
      val followerWorkingDir = new File(followerSpec.workingDir)
      val followerOutFile = new File(runOutputDir, s"${outFilenamePrefix}_$followerNumber.out")
      val followerErrFile = new File(runOutputDir, s"${outFilenamePrefix}_$followerNumber.err")

      if (followerSpec.isLocal) {
        println(s"Launching follower $followerNumber on port ${followerSpec.port}")
        val command = Seq(followerSpec.javaCmd, "-cp", followerSpec.classPath) ++ followerSpec.jvmOptions ++ Seq(s"-Dorc.executionlog.fileprefix=${outFilenamePrefix}_", s"-Dorc.executionlog.filesuffix=_$followerNumber", DistribTestConfig.expanded("followerClass")) ++ followerOpts.toSeq ++ Seq(followerNumber.toString, followerSpec.hostname+":"+followerSpec.port)
        OsCommand.runNoWait(command, directory = followerWorkingDir, stdout = followerOutFile, stderr = followerErrFile)
      } else {
        println(s"Launching follower $followerNumber on ${followerSpec.hostname}:${followerSpec.port}")
        /* FIXME: Escape strings for shell */
        val command = Seq("ssh", followerSpec.hostname, s"cd '${followerSpec.workingDir}'; '${followerSpec.javaCmd}' -cp '${followerSpec.classPath}' ${followerSpec.jvmOptions.mkString(" ")} -Dorc.executionlog.fileprefix=${outFilenamePrefix}_ -Dorc.executionlog.filesuffix=_$followerNumber ${DistribTestConfig.expanded("followerClass")} ${followerOpts.mkString(" ")} $followerNumber ${followerSpec.hostname}:${followerSpec.port} >'$followerOutFile' 2>'$followerErrFile'")
{'fraction_non_alphanumeric': 0.12937511453179404, 'fraction_numerical': 0.006230529595015576, 'mean_word_length': 5.00715465052284, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2133421', 'n_tokens_mistral': 3549, 'n_tokens_neox': 3267, 'n_words': 621}