<commit_before>package com.cloudera.ds.legallsa

import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Paths}

import org.apache.spark.SparkContext
import org.apache.spark.mllib.linalg.{Matrix, Vector}
import org.apache.spark.rdd.RDD

object DataIO {


  val raw_count_path = "hdfs:///user/juliet/legallsa/Word_stats_MACs_0708.csv"
  val tfidf_path = "hdfs:///user/juliet/legallsa/Word_stats_MACs_0708_TF-IDF.csv"


  def transformLine(line: Array[String]) = {

  }

  def read_marix_input(sc: SparkContext, path: String): Matrix = {
    val lines: RDD[String] = sc.textFile(path)
    val header = lines.first()
    lines.map{ line =>
      val arr =line.split(",")
      val word_arr = arr.slice(12, arr.length)
      word_arr.map()
    }
  }

<commit_msg>Creates RowMatrix from csvs.<commit_after>package com.cloudera.ds.legallsa

import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Paths}

import org.apache.spark.SparkContext
import org.apache.spark.mllib.linalg.distributed.RowMatrix
import org.apache.spark.mllib.linalg.{Vectors, Matrix, Vector}
import org.apache.spark.rdd.RDD

object DataIO {
  val raw_count_path = "hdfs:///user/juliet/legallsa/Word_stats_MACs_0708.csv"
  val tfidf_path = "hdfs:///user/juliet/legallsa/Word_stats_MACs_0708_TF-IDF.csv"

  def read_marix_input(sc: SparkContext, path: String): RowMatrix = {
    val lines: RDD[String] = sc.textFile(path)
    val header = lines.first()
    val vectors = lines.map{ line =>
      val arr =line.split(",")
      val word_arr = arr.slice(12, arr.length)

      val nonzeroElems: Seq[(Int, Double)] = word_arr.map(elem => elem.toDouble).zipWithIndex.filter(tuple =>
      tuple._1 != 0.0).map(tuple => (tuple._2, tuple._1)).toSeq
      Vectors.sparse(word_arr.length, nonzeroElems)
    }
    new RowMatrix(vectors)
  }


{'fraction_non_alphanumeric': 0.13424657534246576, 'fraction_numerical': 0.0136986301369863, 'mean_word_length': 3.9351351351351354, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3396710', 'n_tokens_mistral': 721, 'n_tokens_neox': 684, 'n_words': 114}