<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark

import org.scalatest.{FunSuite, PrivateMethodTester}

import org.apache.spark.scheduler.{SchedulerBackend, TaskScheduler, TaskSchedulerImpl}
import org.apache.spark.scheduler.cluster.{SimrSchedulerBackend, SparkDeploySchedulerBackend}
import org.apache.spark.scheduler.cluster.mesos.{CoarseMesosSchedulerBackend, MesosSchedulerBackend}
import org.apache.spark.scheduler.local.LocalBackend

class SparkContextSchedulerCreationSuite
  extends FunSuite with LocalSparkContext with PrivateMethodTester with Logging {

  def createTaskScheduler(master: String): TaskSchedulerImpl =
    createTaskScheduler(master, new SparkConf())

  def createTaskScheduler(master: String, conf: SparkConf): TaskSchedulerImpl = {
    // Create local SparkContext to setup a SparkEnv. We don't actually want to start() the
    // real schedulers, so we don't want to create a full SparkContext with the desired scheduler.
    sc = new SparkContext("local", "test", conf)
    val createTaskSchedulerMethod =
      PrivateMethod[Tuple2[SchedulerBackend, TaskScheduler]]('createTaskScheduler)
    val (_, sched) = SparkContext invokePrivate createTaskSchedulerMethod(sc, master)
    sched.asInstanceOf[TaskSchedulerImpl]
  }

  test("bad-master") {
    val e = intercept[SparkException] {
      createTaskScheduler("localhost:1234")
    }
    assert(e.getMessage.contains("Could not parse Master URL"))
  }

  test("local") {
    val sched = createTaskScheduler("local")
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === 1)
      case _ => fail()
    }
  }

  test("local-*") {
    val sched = createTaskScheduler("local[*]")
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === Runtime.getRuntime.availableProcessors())
      case _ => fail()
    }
  }

  test("local-n") {
    val sched = createTaskScheduler("local[5]")
    assert(sched.maxTaskFailures === 1)
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === 5)
      case _ => fail()
    }
  }

  test("local-*-n-failures") {
    val sched = createTaskScheduler("local[* ,2]")
    assert(sched.maxTaskFailures === 2)
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === Runtime.getRuntime.availableProcessors())
      case _ => fail()
    }
  }

  test("local-n-failures") {
    val sched = createTaskScheduler("local[4, 2]")
    assert(sched.maxTaskFailures === 2)
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === 4)
      case _ => fail()
    }
  }

  test("bad-local-n") {
    val e = intercept[SparkException] {
      createTaskScheduler("local[2*]")
    }
    assert(e.getMessage.contains("Could not parse Master URL"))
  }

  test("bad-local-n-failures") {
    val e = intercept[SparkException] {
      createTaskScheduler("local[2*,4]")
    }
    assert(e.getMessage.contains("Could not parse Master URL"))
  }

  test("local-default-parallelism") {
    val conf = new SparkConf().set("spark.default.parallelism", "16")
    val sched = createTaskScheduler("local", conf)

    sched.backend match {
      case s: LocalBackend => assert(s.defaultParallelism() === 16)
      case _ => fail()
    }
  }

  test("simr") {
    createTaskScheduler("simr://uri").backend match {
      case s: SimrSchedulerBackend => // OK
      case _ => fail()
    }
  }

  test("local-cluster") {
    createTaskScheduler("local-cluster[3, 14, 512]").backend match {
      case s: SparkDeploySchedulerBackend => // OK
      case _ => fail()
    }
  }

  def testYarn(master: String, expectedClassName: String) {
    try {
      val sched = createTaskScheduler(master)
      assert(sched.getClass === Class.forName(expectedClassName))
    } catch {
      case e: SparkException =>
        assert(e.getMessage.contains("YARN mode not available"))
        logWarning("YARN not available, could not test actual YARN scheduler creation")
      case e: Throwable => fail(e)
    }
  }

  test("yarn-cluster") {
    testYarn("yarn-cluster", "org.apache.spark.scheduler.cluster.YarnClusterScheduler")
  }

  test("yarn-standalone") {
    testYarn("yarn-standalone", "org.apache.spark.scheduler.cluster.YarnClusterScheduler")
  }

  test("yarn-client") {
    testYarn("yarn-client", "org.apache.spark.scheduler.cluster.YarnScheduler")
  }

  def testMesos(master: String, expectedClass: Class[_], coarse: Boolean) {
    val conf = new SparkConf().set("spark.mesos.coarse", coarse.toString)
    try {
      val sched = createTaskScheduler(master, conf)
      assert(sched.backend.getClass === expectedClass)
    } catch {
      case e: UnsatisfiedLinkError =>
        assert(e.getMessage.contains("no mesos in"))
        logWarning("Mesos not available, could not test actual Mesos scheduler creation")
      case e: Throwable => fail(e)
    }
  }

  test("mesos fine-grained") {
    testMesos("mesos://localhost:1234", classOf[MesosSchedulerBackend], coarse = false)
  }

  test("mesos coarse-grained") {
    testMesos("mesos://localhost:1234", classOf[CoarseMesosSchedulerBackend], coarse = true)
  }

  test("mesos with zookeeper") {
    testMesos("zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false)
  }
}<commit_msg>[MINOR] [CORE] Accept alternative mesos unsatisfied link error in test.<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark

import org.scalatest.{FunSuite, PrivateMethodTester}

import org.apache.spark.scheduler.{SchedulerBackend, TaskScheduler, TaskSchedulerImpl}
import org.apache.spark.scheduler.cluster.{SimrSchedulerBackend, SparkDeploySchedulerBackend}
import org.apache.spark.scheduler.cluster.mesos.{CoarseMesosSchedulerBackend, MesosSchedulerBackend}
import org.apache.spark.scheduler.local.LocalBackend

class SparkContextSchedulerCreationSuite
  extends FunSuite with LocalSparkContext with PrivateMethodTester with Logging {

  def createTaskScheduler(master: String): TaskSchedulerImpl =
    createTaskScheduler(master, new SparkConf())

  def createTaskScheduler(master: String, conf: SparkConf): TaskSchedulerImpl = {
    // Create local SparkContext to setup a SparkEnv. We don't actually want to start() the
    // real schedulers, so we don't want to create a full SparkContext with the desired scheduler.
    sc = new SparkContext("local", "test", conf)
    val createTaskSchedulerMethod =
      PrivateMethod[Tuple2[SchedulerBackend, TaskScheduler]]('createTaskScheduler)
    val (_, sched) = SparkContext invokePrivate createTaskSchedulerMethod(sc, master)
    sched.asInstanceOf[TaskSchedulerImpl]
  }

  test("bad-master") {
    val e = intercept[SparkException] {
      createTaskScheduler("localhost:1234")
    }
    assert(e.getMessage.contains("Could not parse Master URL"))
  }

  test("local") {
    val sched = createTaskScheduler("local")
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === 1)
      case _ => fail()
    }
  }

  test("local-*") {
    val sched = createTaskScheduler("local[*]")
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === Runtime.getRuntime.availableProcessors())
      case _ => fail()
    }
  }

  test("local-n") {
    val sched = createTaskScheduler("local[5]")
    assert(sched.maxTaskFailures === 1)
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === 5)
      case _ => fail()
    }
  }

  test("local-*-n-failures") {
    val sched = createTaskScheduler("local[* ,2]")
    assert(sched.maxTaskFailures === 2)
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === Runtime.getRuntime.availableProcessors())
      case _ => fail()
    }
  }

  test("local-n-failures") {
    val sched = createTaskScheduler("local[4, 2]")
    assert(sched.maxTaskFailures === 2)
    sched.backend match {
      case s: LocalBackend => assert(s.totalCores === 4)
      case _ => fail()
    }
  }

  test("bad-local-n") {
    val e = intercept[SparkException] {
      createTaskScheduler("local[2*]")
    }
    assert(e.getMessage.contains("Could not parse Master URL"))
  }

  test("bad-local-n-failures") {
    val e = intercept[SparkException] {
      createTaskScheduler("local[2*,4]")
    }
    assert(e.getMessage.contains("Could not parse Master URL"))
  }

  test("local-default-parallelism") {
    val conf = new SparkConf().set("spark.default.parallelism", "16")
    val sched = createTaskScheduler("local", conf)

    sched.backend match {
      case s: LocalBackend => assert(s.defaultParallelism() === 16)
      case _ => fail()
    }
  }

  test("simr") {
    createTaskScheduler("simr://uri").backend match {
      case s: SimrSchedulerBackend => // OK
      case _ => fail()
    }
  }

  test("local-cluster") {
    createTaskScheduler("local-cluster[3, 14, 512]").backend match {
      case s: SparkDeploySchedulerBackend => // OK
      case _ => fail()
    }
  }

  def testYarn(master: String, expectedClassName: String) {
    try {
      val sched = createTaskScheduler(master)
      assert(sched.getClass === Class.forName(expectedClassName))
    } catch {
      case e: SparkException =>
        assert(e.getMessage.contains("YARN mode not available"))
        logWarning("YARN not available, could not test actual YARN scheduler creation")
      case e: Throwable => fail(e)
    }
  }

  test("yarn-cluster") {
    testYarn("yarn-cluster", "org.apache.spark.scheduler.cluster.YarnClusterScheduler")
  }

  test("yarn-standalone") {
    testYarn("yarn-standalone", "org.apache.spark.scheduler.cluster.YarnClusterScheduler")
  }

  test("yarn-client") {
    testYarn("yarn-client", "org.apache.spark.scheduler.cluster.YarnScheduler")
  }

  def testMesos(master: String, expectedClass: Class[_], coarse: Boolean) {
    val conf = new SparkConf().set("spark.mesos.coarse", coarse.toString)
    try {
      val sched = createTaskScheduler(master, conf)
      assert(sched.backend.getClass === expectedClass)
    } catch {
      case e: UnsatisfiedLinkError =>
        assert(e.getMessage.contains("mesos"))
        logWarning("Mesos not available, could not test actual Mesos scheduler creation")
      case e: Throwable => fail(e)
    }
  }

  test("mesos fine-grained") {
    testMesos("mesos://localhost:1234", classOf[MesosSchedulerBackend], coarse = false)
  }

  test("mesos coarse-grained") {
    testMesos("mesos://localhost:1234", classOf[CoarseMesosSchedulerBackend], coarse = true)
  }

  test("mesos with zookeeper") {
    testMesos("zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false)
  }
}
{'fraction_non_alphanumeric': 0.12114972821610938, 'fraction_numerical': 0.007906440454620326, 'mean_word_length': 3.715728155339806, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6274977', 'n_tokens_mistral': 3770, 'n_tokens_neox': 3638, 'n_words': 889}