<commit_before>import Classification._
import Regression._
import org.apache.log4j.{Level, Logger}
import utils.Utils
import org.rogach.scallop._
import org.apache.spark._
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.rdd.RDD
import java.io._

import utils.Functions.ElasticNet

class RunParser(arguments: Seq[String]) extends org.rogach.scallop.ScallopConf(arguments) {
  val dir = opt[String](required = true, default = Some("../results/"), short = 'w', descr = "working directory where results " +
    "are stored. Default is \"../results\". ")
  val optimizers = trailArg[List[String]](descr = "List of optimizers to be used. At least one is required")
  verify()
}

object RUN {

  def main(args: Array[String]) {
    //Spark conf
    val a: Map[String, Object] = null
    val conf = new SparkConf().setAppName("Distributed Machine Learning").setMaster("local[*]")
    val sc = new SparkContext(conf)
    val sqlContext = new org.apache.spark.sql.SQLContext(sc)
    //Turn off logs
    val rootLogger = Logger.getRootLogger()
    rootLogger.setLevel(Level.ERROR)
    //Parse arguments
    val parser = new RunParser(args)
    val optimizers: List[String] = parser.optimizers()
    assert(optimizers.length > 0)
    val workingDir = parser.dir()

    val train: RDD[LabeledPoint] = Utils.loadLibSVMFromDir(workingDir + "train/", sc)
    val test: RDD[LabeledPoint] = Utils.loadLibSVMFromDir(workingDir + "test/", sc)

    val gdparams = Utils.readGDParameters(workingDir)
    val sgdparams = Utils.readSGDParameters(workingDir)
    val lbfgsparams = Utils.readLBFGSParameters(workingDir)
    val cocoaparams = Utils.readCocoaParameters(workingDir, train, test)
    val elasticparams = Utils.readElasticProxCocoaParameters(workingDir, train, test)
    val l1cocoaparams = Utils.readL1ProxCocoaParameters(workingDir, train, test)
    val lambda = Utils.readRegParameters(workingDir)

    val output = new File(workingDir + "res.out")
    val bw = new BufferedWriter(new FileWriter(output))<commit_msg>performance bug for cscs fixed<commit_after>import Classification._
import Regression._
import org.apache.log4j.{Level, Logger}
import utils.Utils
import org.rogach.scallop._
import org.apache.spark._
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.rdd.RDD
import java.io._

import utils.Functions.ElasticNet

class RunParser(arguments: Seq[String]) extends org.rogach.scallop.ScallopConf(arguments) {
  val dir = opt[String](required = true, default = Some("../results/"), short = 'w', descr = "working directory where results " +
    "are stored. Default is \"../results\". ")
  val optimizers = trailArg[List[String]](descr = "List of optimizers to be used. At least one is required")
  verify()
}

object RUN {

  def main(args: Array[String]) {
    //Spark conf
    val a: Map[String, Object] = null
    val conf = new SparkConf().setAppName("Distributed Machine Learning")
    val sc = new SparkContext(conf)
    val sqlContext = new org.apache.spark.sql.SQLContext(sc)
    //Turn off logs
    val rootLogger = Logger.getRootLogger()
    rootLogger.setLevel(Level.ERROR)
    //Parse arguments
    val parser = new RunParser(args)
    val optimizers: List[String] = parser.optimizers()
    assert(optimizers.length > 0)
    val workingDir = parser.dir()

    val train: RDD[LabeledPoint] = Utils.loadLibSVMFromDir(workingDir + "train/", sc)
    val test: RDD[LabeledPoint] = Utils.loadLibSVMFromDir(workingDir + "test/", sc)

    val gdparams = Utils.readGDParameters(workingDir)
    val sgdparams = Utils.readSGDParameters(workingDir)
    val lbfgsparams = Utils.readLBFGSParameters(workingDir)
    val cocoaparams = Utils.readCocoaParameters(workingDir, train, test)
    val elasticparams = Utils.readElasticProxCocoaParameters(workingDir, train, test)
    val l1cocoaparams = Utils.readL1ProxCocoaParameters(workingDir, train, test)
    val lambda = Utils.readRegParameters(workingDir)

    val output = new File(workingDir + "res.out")
    val bw = new BufferedWriter(new FileWriter(output))
{'fraction_non_alphanumeric': 0.1121771217712177, 'fraction_numerical': 0.0019680196801968018, 'mean_word_length': 4.364116094986807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6120748', 'n_tokens_mistral': 1320, 'n_tokens_neox': 1293, 'n_words': 315}