<commit_before>package com.github.samplett.skaluh


/**
 * a scala class for common clustering algorithms / utilities
 */
class Cluster {

  val sampleSet = Map(
    "Gothamist" -> Map("china" -> 0, "kids" -> 3, "music" -> 3, "yahoo" -> 0),
    "GigaOM" -> Map("china" -> 6, "kids" -> 0, "music" -> 0, "yahoo" -> 2),
    "Quick Online Tips" -> Map("china" -> 0, "kids" -> 2, "music" -> 2, "yahoo" -> 22)
  );

  def readClusterDatasetFromFile(filePath: String): (Array[String], Array[String], Iterable[Array[Double]]) = {
    val lines = scala.collection.mutable.ArrayBuffer[String]();
    for (line <- scala.io.Source.fromFile(filePath).getLines())
      lines += line;
    val colnames = lines(0).split("\t")
    var rownames = scala.collection.mutable.ArrayBuffer[String]();
    var data = new scala.collection.mutable.HashMap[String, Array[Double]];
    for (line <- lines.drop(1)) {
      // excluding the first row which holds the column names
      val p = line.split("\t");
      val floats = for (item <- p.drop(1)) yield Double.unbox(item);
      data += p(0) -> floats;
      rownames += p(0);
    }
    return (colnames, rownames.toArray, data.values);
  }

  def pearson(f1: Array[Double], f2: Array[Double]): Double = {
    var sum1 = 0;
    for (item <- f1) sum1 += item;

    var sum2 = 0;
    for (item <- f2) sum2 += item;

    var sumSq1 = 0;
    for (item <- f1) sumSq1 += math.pow(item, 2);

    var sumSq2 = 0;
    for (item <- f2) sumSq2 += math.pow(item, 2);

    var pSum = 0;
    for (i <- 0 until f1.length if f2(i)) pSum += f1(i) * f2(i);

    val num = pSum - (sum1 * sum2 / f1.length);
    val den = math.sqrt((sumSq1 - math.pow(sum1, 2) / f1.length) * (sumSq2 - math.pow(sum2, 2) / f2.length))

    return 1 - num / den;
  }

}<commit_msg>using fold instead of loop for summing items in an array<commit_after>package com.github.samplett.skaluh


/**
 * a scala class for common clustering algorithms / utilities
 */
class Cluster {

  val sampleSet = Map(
    "Gothamist" -> Map("china" -> 0, "kids" -> 3, "music" -> 3, "yahoo" -> 0),
    "GigaOM" -> Map("china" -> 6, "kids" -> 0, "music" -> 0, "yahoo" -> 2),
    "Quick Online Tips" -> Map("china" -> 0, "kids" -> 2, "music" -> 2, "yahoo" -> 22)
  );

  def readClusterDatasetFromFile(filePath: String): (Array[String], Array[String], Iterable[Array[Double]]) = {
    val lines = scala.collection.mutable.ArrayBuffer[String]();
    for (line <- scala.io.Source.fromFile(filePath).getLines())
      lines += line;
    val colnames = lines(0).split("\t")
    var rownames = scala.collection.mutable.ArrayBuffer[String]();
    var data = new scala.collection.mutable.HashMap[String, Array[Double]];
    for (line <- lines.drop(1)) {
      // excluding the first row which holds the column names
      val p = line.split("\t");
      val floats = for (item <- p.drop(1)) yield Double.unbox(item);
      data += p(0) -> floats;
      rownames += p(0);
    }
    return (colnames, rownames.toArray, data.values);
  }

  def pearson(f1: Array[Double], f2: Array[Double]): Double = {
    val sum1 = f1.foldLeft(0d) {
      (total, n) =>
        total + n
    };

    val sum2 = f2.foldLeft(0d) {
      (total, n) =>
        total + n
    };

    var sumSq1 = 0;
    for (item <- f1) sumSq1 += math.pow(item, 2);

    var sumSq2 = 0;
    for (item <- f2) sumSq2 += math.pow(item, 2);

    var pSum = 0;
    for (i <- 0 until f1.length if f2(i)) pSum += f1(i) * f2(i);

    val num = pSum - (sum1 * sum2 / f1.length);
    val den = math.sqrt((sumSq1 - math.pow(sum1, 2) / f1.length) * (sumSq2 - math.pow(sum2, 2) / f2.length))

    return 1 - num / den;
  }

}
{'fraction_non_alphanumeric': 0.1811175337186898, 'fraction_numerical': 0.030278007156619872, 'mean_word_length': 2.6744186046511627, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5106184', 'n_tokens_mistral': 1442, 'n_tokens_neox': 1335, 'n_words': 354}