<commit_before>/* SimpleApp.java */
import org.apache.spark.api.java.*;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.stat.Statistics;
import org.apache.spark.mllib.linalg.*;

public class SimpleCorrelation {
  public static void main(String[] args) {
	if (args.length != 1) {
		throw new IllegalArgumentException("No input file");
	}
    String inputPath = args[0]; // Should be some file on your system
    
    SparkConf conf = new SparkConf().setAppName("Simple Application");
    JavaSparkContext sc = new JavaSparkContext(conf);
    JavaRDD<Vector> pairData = sc.textFile(inputPath)
    		.cache().
    		map(new Function<String, Vector>(){

				public Vector call(String pairLine) throws Exception {
					
					String[] values = pairLine.split(",");
					Vector outVec = Vectors.dense(Double.parseDouble(values[0]), Double.parseDouble(values[1]));
					return outVec;
				}
    			
    		});
    
    Matrix correlationVal = Statistics.corr(pairData.rdd(), "pearson");
    System.out.println("Correlation Matrix: \n" + correlationVal.toString());
  }
}<commit_msg>I made you some doubles.<commit_after>/* SimpleApp.java */
import org.apache.spark.api.java.*;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.stat.Statistics;
import org.apache.spark.mllib.linalg.*;

public class SimpleCorrelation {
  public static void main(String[] args) {
	if (args.length != 1) {
		throw new IllegalArgumentException("No input file");
	}
    String inputPath = args[0]; // Should be some file on your system
    
    SparkConf conf = new SparkConf().setAppName("Simple Application");
    JavaSparkContext sc = new JavaSparkContext(conf);
    JavaRDD<Vector> pairData = sc.textFile(inputPath)
    		.cache().
    		map(new Function<String, Vector>(){

				public Vector call(String pairLine) throws Exception {
					
					String[] stringValues = pairLine.split(",");
					double[] values = parseDoubles(stringValues);
					Vector outVec = Vectors.dense(values);
					return outVec;
				}
    			
    		});
    
    Matrix correlationVal = Statistics.corr(pairData.rdd(), "pearson");
    System.out.println("Correlation Matrix: \n" + correlationVal.toString());
  }
  
  private static double[] parseDoubles(String[] inVals) {
	double[] nums = new double[inVals.length];
	for (int i = 0; i < nums.length; i++) {
		nums[i] = Double.parseDouble(inVals[i]);
	}
	return nums;
  }
}
{'fraction_non_alphanumeric': 0.13457249070631971, 'fraction_numerical': 0.0026022304832713753, 'mean_word_length': 3.788256227758007, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '525807', 'n_tokens_mistral': 943, 'n_tokens_neox': 858, 'n_words': 188}