<commit_before>package utils

import Functions.{CocoaLabeledPoint, ProxCocoaDataMatrix, ProxCocoaLabeledPoint}
import breeze.linalg.{DenseVector, SparseVector}
import l1distopt.utils.{DebugParams, Params}
import org.apache.spark.SparkContext
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.util.MLUtils
import org.apache.spark.rdd.RDD

import scala.xml.XML

/**
  * Created by amirreza on 12/04/16.
  */
object Utils {
  def loadRawDataset(dataset: String, sc: SparkContext): RDD[LabeledPoint] = {
    // get projectPath from config
    val xml = XML.loadFile("configs.xml")
    val projectPath = (xml \\ "config" \\ "projectpath") text
    val datasetPath = projectPath + (if (projectPath.endsWith("/")) "" else "/") + "datasets/" + dataset
    //Load data
    val data: RDD[LabeledPoint] = MLUtils.loadLibSVMFile(sc, datasetPath)<commit_msg>loadLibSVMFrom dir implemented<commit_after>package utils

import java.io.File

import Functions.{CocoaLabeledPoint, ProxCocoaDataMatrix, ProxCocoaLabeledPoint}
import breeze.linalg.{DenseVector, SparseVector}
import l1distopt.utils.{DebugParams, Params}
import org.apache.spark.SparkContext
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.util.MLUtils
import org.apache.spark.rdd.RDD

import scala.xml.XML

/**
  * Created by amirreza on 12/04/16.
  */
object Utils {
  def loadLibSVMFromDir(dir: String, sc: SparkContext): RDD[LabeledPoint] = {
    val files = (new File(dir)).listFiles.filter(_.getName.startsWith("part")).map(_.getName)
    return files.map(part => MLUtils.loadLibSVMFile(sc, dir + part).repartition(1)).reduceLeft(_.union(_))
  }

  def loadRawDataset(dataset: String, sc: SparkContext): RDD[LabeledPoint] = {
    // get projectPath from config
    val xml = XML.loadFile("configs.xml")
    val projectPath = (xml \\ "config" \\ "projectpath") text
    val datasetPath = projectPath + (if (projectPath.endsWith("/")) "" else "/") + "datasets/" + dataset
    //Load data
    val data: RDD[LabeledPoint] = MLUtils.loadLibSVMFile(sc, datasetPath)
{'fraction_non_alphanumeric': 0.1293269230769231, 'fraction_numerical': 0.007211538461538462, 'mean_word_length': 4.670299727520436, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '967479', 'n_tokens_mistral': 720, 'n_tokens_neox': 701, 'n_words': 144}