<commit_before>package sparklyr

import java.io.File
import java.util.Arrays

import org.apache.spark._
import org.apache.spark.sql._

import scala.collection.JavaConverters._

class Rscript(logger: Logger) {
  def getScratchDir(): File = {
    val sparkFiles = Class.forName("org.apache.spark.SparkFiles")
    val selectedMethods = sparkFiles.getMethods.filter(m => m.getName == "getRootDirectory")
    val rootDir = selectedMethods(0).invoke(sparkFiles).asInstanceOf[String]
    new File(rootDir)
  }

  def getSparkConf(): SparkConf = {
    // val sparkConf = SparkEnv.get.conf
    val sparkEnvRoot = Class.forName("org.apache.spark.SparkEnv")
    val selectedMethods = sparkEnvRoot.getMethods.filter(
      m => m.getName == "get" && m.getParameterTypes.length == 0
    )
    val sparkEnv: SparkEnv= selectedMethods(0).invoke(sparkEnvRoot).asInstanceOf[SparkEnv]

    val sparkEnvMethods = sparkEnv.getClass.getMethods.filter(
      m => m.getName == "conf" && m.getParameterTypes.length == 0
    )
    sparkEnvMethods(0).invoke(sparkEnv).asInstanceOf[SparkConf]
  }

  def getCommand(): String = {
    val sparkConf: SparkConf = getSparkConf()

    // val command: String = sparkConf.get("spark.r.command", "Rscript")
    val sparkConfMethods = sparkConf.getClass.getMethods.filter(
      m => m.getName == "get" && m.getParameterTypes.length == 2
    )
    sparkConfMethods(0).invoke(sparkConf, "spark.r.command", "Rscript").asInstanceOf[String]
  }

  def run(commands: String) = {
    val processBuilder: ProcessBuilder = new ProcessBuilder(commands.split(" ").toList.asJava)

    processBuilder.redirectErrorStream(true);
    processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
    processBuilder.directory(getScratchDir());

    logger.log("is running custom command: " + commands)
    val process: Process = processBuilder.start()<commit_msg>remove workaround for livy sources<commit_after>package sparklyr

import java.io.File
import java.util.Arrays

import org.apache.spark._
import org.apache.spark.sql._

import scala.collection.JavaConverters._

class Rscript(logger: Logger) {
  def getScratchDir(): File = {
    new File(SparkFiles.getRootDirectory)
  }

  def getCommand(): String = {
    SparkEnv.get.conf.get("spark.r.command", "Rscript")
  }

  def run(commands: String) = {
    val processBuilder: ProcessBuilder = new ProcessBuilder(commands.split(" ").toList.asJava)

    processBuilder.redirectErrorStream(true);
    processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
    processBuilder.directory(getScratchDir());

    logger.log("is running custom command: " + commands)
    val process: Process = processBuilder.start()
{'fraction_non_alphanumeric': 0.12303664921465969, 'fraction_numerical': 0.002617801047120419, 'mean_word_length': 4.371485943775101, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1982077', 'n_tokens_mistral': 838, 'n_tokens_neox': 809, 'n_words': 154}