<commit_before>/*
 * Copyright 2014 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.ibm.spark.interpreter

import java.io.{BufferedReader, InputStreamReader, PrintStream, ByteArrayOutputStream}
import java.net.{URL, URLClassLoader}
import java.nio.charset.Charset
import java.util.concurrent.ExecutionException

import com.ibm.spark.interpreter.imports.printers.{WrapperConsole, WrapperSystem}
import com.ibm.spark.utils.{TaskManager, MultiOutputStream}
import org.apache.spark.repl.{SparkJLineCompletion, SparkIMain}
import org.slf4j.LoggerFactory
import scala.concurrent.{Await, Future}
import scala.tools.nsc.{Global, Settings, io}
import scala.tools.nsc.backend.JavaPlatform
import scala.tools.nsc.interpreter._
import scala.tools.nsc.io.AbstractFile
import scala.tools.nsc.util.MergedClassPath

import scala.util.{Try => UtilTry}

import scala.language.reflectiveCalls

class ScalaInterpreter(
  args: List[String],
  out: OutputStream
) extends Interpreter {
  this: SparkIMainProducerLike
    with TaskManagerProducerLike
    with SettingsProducerLike =>

  protected val logger = LoggerFactory.getLogger(this.getClass.getName)

  private val ExecutionExceptionName = "lastException"
  val settings: Settings = newSettings(args)

  private val _thisClassloader = this.getClass.getClassLoader
  protected val _runtimeClassloader =
    new URLClassLoader(Array(), _thisClassloader) {
      def addJar(url: URL) = this.addURL(url)
    }

  /* Add scala.runtime libraries to interpreter classpath */ {
    val urls = _thisClassloader match {
      case cl: java.net.URLClassLoader => cl.getURLs.toList
      case a => // TODO: Should we really be using sys.error here?
        sys.error("[SparkInterpreter] Unexpected class loader: " + a.getClass)
    }
    val classpath = urls.map(_.toString)

    settings.classpath.value =
      classpath.distinct.mkString(java.io.File.pathSeparator)
    settings.embeddedDefaults(_runtimeClassloader)
  }

  private val lastResultOut = new ByteArrayOutputStream()
  private val multiOutputStream = MultiOutputStream(List(out, lastResultOut))
  private var taskManager: TaskManager = _
  var sparkIMain: SparkIMain = _
  protected var jLineCompleter: SparkJLineCompletion = _

  /**
   * Adds jars to the runtime and compile time classpaths. Does not work with
   * directories or expanding star in a path.
   * @param jars The list of jar locations
   */
  override def addJars(jars: URL*): Unit = {
    // Enable Scala class support
    reinitializeSymbols()

    jars.foreach(_runtimeClassloader.addJar)
    updateCompilerClassPath(jars : _*)

    // Refresh all of our variables
    refreshDefinitions()
  }

  // TODO: Need to figure out a better way to compare the representation of
  //       an annotation (contained in AnnotationInfo) with various annotations
  //       like scala.transient
  protected def buildModifierList(termNameString: String) = {
    import scala.language.existentials
    val termSymbol = sparkIMain.symbolOfTerm(termNameString)

    if (termSymbol.hasAccessorFlag) {
      termSymbol.accessed.annotations map {
        case a if a.toString == "transient" => "@transient"
        case a =>
          logger.debug(s"Ignoring unknown accessor annotation: $a")
          ""
      } filterNot {
        _.isEmpty
      }
    } else {
      termSymbol.annotations map {
        case a if a.toString == "transient" => "@transient"
        case a =>
          logger.debug(s"Ignoring unknown annotation: $a")
          ""
      } filterNot {
        _.isEmpty
      }
    }
  }

  protected def refreshDefinitions(): Unit = {
    sparkIMain.definedTerms.foreach(termName => {
      val termNameString = termName.toString
      val termTypeString = sparkIMain.typeOfTerm(termNameString).toLongString
      sparkIMain.valueOfTerm(termNameString) match {
        case Some(termValue)  =>
          logger.debug(s"Rebinding of $termNameString as $termTypeString")
          UtilTry(sparkIMain.beSilentDuring {
            sparkIMain.bind(<commit_msg>Refactored annotation info conversion to its own method<commit_after>/*
 * Copyright 2014 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.ibm.spark.interpreter

import java.io.{BufferedReader, InputStreamReader, PrintStream, ByteArrayOutputStream}
import java.net.{URL, URLClassLoader}
import java.nio.charset.Charset
import java.util.concurrent.ExecutionException

import com.ibm.spark.interpreter.imports.printers.{WrapperConsole, WrapperSystem}
import com.ibm.spark.utils.{TaskManager, MultiOutputStream}
import org.apache.spark.repl.{SparkJLineCompletion, SparkIMain}
import org.slf4j.LoggerFactory
import scala.concurrent.{Await, Future}
import scala.tools.nsc.{Global, Settings, io}
import scala.tools.nsc.backend.JavaPlatform
import scala.tools.nsc.interpreter._
import scala.tools.nsc.io.AbstractFile
import scala.tools.nsc.util.MergedClassPath

import scala.util.{Try => UtilTry}

import scala.language.reflectiveCalls

class ScalaInterpreter(
  args: List[String],
  out: OutputStream
) extends Interpreter {
  this: SparkIMainProducerLike
    with TaskManagerProducerLike
    with SettingsProducerLike =>

  protected val logger = LoggerFactory.getLogger(this.getClass.getName)

  private val ExecutionExceptionName = "lastException"
  val settings: Settings = newSettings(args)

  private val _thisClassloader = this.getClass.getClassLoader
  protected val _runtimeClassloader =
    new URLClassLoader(Array(), _thisClassloader) {
      def addJar(url: URL) = this.addURL(url)
    }

  /* Add scala.runtime libraries to interpreter classpath */ {
    val urls = _thisClassloader match {
      case cl: java.net.URLClassLoader => cl.getURLs.toList
      case a => // TODO: Should we really be using sys.error here?
        sys.error("[SparkInterpreter] Unexpected class loader: " + a.getClass)
    }
    val classpath = urls.map(_.toString)

    settings.classpath.value =
      classpath.distinct.mkString(java.io.File.pathSeparator)
    settings.embeddedDefaults(_runtimeClassloader)
  }

  private val lastResultOut = new ByteArrayOutputStream()
  private val multiOutputStream = MultiOutputStream(List(out, lastResultOut))
  private var taskManager: TaskManager = _
  var sparkIMain: SparkIMain = _
  protected var jLineCompleter: SparkJLineCompletion = _

  /**
   * Adds jars to the runtime and compile time classpaths. Does not work with
   * directories or expanding star in a path.
   * @param jars The list of jar locations
   */
  override def addJars(jars: URL*): Unit = {
    // Enable Scala class support
    reinitializeSymbols()

    jars.foreach(_runtimeClassloader.addJar)
    updateCompilerClassPath(jars : _*)

    // Refresh all of our variables
    refreshDefinitions()
  }

  // TODO: Need to figure out a better way to compare the representation of
  //       an annotation (contained in AnnotationInfo) with various annotations
  //       like scala.transient
  protected def convertAnnotationsToModifiers(
    annotationInfos: List[Global#AnnotationInfo]
  ) = annotationInfos map {
    case a if a.toString == "transient" => "@transient"
    case a =>
      logger.debug(s"Ignoring unknown annotation: $a")
      ""
  } filterNot {
    _.isEmpty
  }

  protected def buildModifierList(termNameString: String) = {
    import scala.language.existentials
    val termSymbol = sparkIMain.symbolOfTerm(termNameString)

    convertAnnotationsToModifiers(
      if (termSymbol.hasAccessorFlag) termSymbol.accessed.annotations
      else termSymbol.annotations
    )
  }

  protected def refreshDefinitions(): Unit = {
    sparkIMain.definedTerms.foreach(termName => {
      val termNameString = termName.toString
      val termTypeString = sparkIMain.typeOfTerm(termNameString).toLongString
      sparkIMain.valueOfTerm(termNameString) match {
        case Some(termValue)  =>
          logger.debug(s"Rebinding of $termNameString as $termTypeString")
          UtilTry(sparkIMain.beSilentDuring {
            sparkIMain.bind(
{'fraction_non_alphanumeric': 0.0845771144278607, 'fraction_numerical': 0.001990049751243781, 'mean_word_length': 3.7560462670872767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3837398', 'n_tokens_mistral': 2621, 'n_tokens_neox': 2560, 'n_words': 715}