<commit_before>module Core {

  /*
   Class to hold labels for data.  Has names in names
   */
  class LabelMatrix {
    var ldom: domain(1) = {1..0},
        dataDom: domain(2),
        data: [dataDom] real,
        names: [ldom] string;
    /*
    Loads a label file into a Matrix.  Labels should be binary indicators
    <record id: string> <category 1 indicator> ... <category L indicator>
     */
    proc readFromFile(fn: string, addDummy: bool = false) {
      var lFile = open(fn, iomode.r).reader(),
          x: [1..0] real,
          nFields: int,
          line: string,
          //ldom: domain(1),
          xline: [ldom] real,
          nRows: int = 1,
          firstLine: bool = true;
      for line in lFile.lines() {
         var fields = line.split("\t");
         if firstLine {
           nFields = fields.size;
           if addDummy {
             // Add a dummy column
             ldom = {1..nFields};
           } else {
             ldom = {1..nFields-1};
           }
           dataDom = {1..0, 1..nFields};
           firstLine = false;
         } else {
           if fields.size != nFields {
             halt("Unequal number of fields in label file");
           }
         }
         //names.push_back(fields[1]);
         writeln(ldom);
         ldom = {1..ldom.last+1};
         //names.push_back(fields[1]);
         names[ldom.last] = fields[1];
         dataDom = { 1..#nRows, ldom.dim(1)};
         var xline: [ldom] real;
         for v in 2..fields.size {
           xline = fields[v]:real;
         }
         data[dataDom.last(1), ..] = xline;
         nRows += 1;
      }
    }
  }

  /*
   Does the pairwise cosine distance between rows of X and rows of Y.
   Dimensions must be conformable
   */
  proc cosineDistance(X:[], Y:[], denseOutput=true) {
    if X.shape[2] != Y.shape[2] {
      halt(" dimension mismatch: X = (%n,%n)  Y = (%n,%n)".format(X.shape[1], X.shape[2], Y.shape[1], Y.shape[2]));
    }
    // Make magic happen

  }
}<commit_msg>Make some magic happen (cosineDistance)<commit_after>module Core {

  /*
   Class to hold labels for data.  Has names in names
   */
  class LabelMatrix {
    var ldom: domain(1) = {1..0},
        dataDom: domain(2),
        data: [dataDom] real,
        names: [ldom] string;
    /*
    Loads a label file into a Matrix.  Labels should be binary indicators
    <record id: string> <category 1 indicator> ... <category L indicator>
     */
    proc readFromFile(fn: string, addDummy: bool = false) {
      var lFile = open(fn, iomode.r).reader(),
          x: [1..0] real,
          nFields: int,
          line: string,
          //ldom: domain(1),
          xline: [ldom] real,
          nRows: int = 1,
          firstLine: bool = true;
      for line in lFile.lines() {
         var fields = line.split("\t");
         if firstLine {
           nFields = fields.size;
           if addDummy {
             // Add a dummy column
             ldom = {1..nFields};
           } else {
             ldom = {1..nFields-1};
           }
           dataDom = {1..0, 1..nFields};
           firstLine = false;
         } else {
           if fields.size != nFields {
             halt("Unequal number of fields in label file");
           }
         }
         //names.push_back(fields[1]);
         writeln(ldom);
         ldom = {1..ldom.last+1};
         //names.push_back(fields[1]);
         names[ldom.last] = fields[1];
         dataDom = { 1..#nRows, ldom.dim(1)};
         var xline: [ldom] real;
         for v in 2..fields.size {
           xline = fields[v]:real;
         }
         data[dataDom.last(1), ..] = xline;
         nRows += 1;
      }
    }
  }

  /*
   Does the pairwise cosine distance between rows of X.
   Dimensions must be conformable
   */
  proc cosineDistance(X:[?Xdom], denseOutput=true) {
    // TODO
    if !denseOutput then halt('denseOutput=false not yet supported');

    var cosDistDom: domain(2) = {Xdom.dim(1), Xdom.dim(1)},
        cosDist: [cosDistDom] real;

    // TODO: verbose output
    //writeln(" Got V: ", V.shape);
    //writeln(" cosDistDom.dims(1) ", cosDistDom.dims());

    // Pre-compute repeated cosim's
    var Xii: [Xdom.dim(1)] real;
    [i in Xdom.dim(1)] Xii[i] = dot(X[i,..], X[i,..]);

    forall i in Xdom.dim(1) {
      const x1 = Xii[i];
      for j in i+1..Xdom.dim(1).size {
        // Do cosim
        const x2 = Xii[j];
        const c = 1 - dot(X[i,..], X[j,..]) / (x1 * x2);
        cosDist[i,j] = c;
        cosDist[j,i] = c;
      }
    }
    return cosDist;
  }

  /*
   Does the pairwise cosine distance between rows of X and rows of Y.
   Dimensions must be conformable
   */
  proc cosineDistance(X:[?Xdom], Y:[?Ydom], denseOutput=true) {
    // TODO
    if !denseOutput then halt('denseOutput=false not yet supported');

    if X.shape[2] != Y.shape[2] {
      halt(" dimension mismatch: X = (%n,%n)  Y = (%n,%n)".format(X.shape[1], X.shape[2], Y.shape[1], Y.shape[2]));
    }

    var cosDistDom: domain(2) = {Xdom.dim(1), Ydom.dim(1)},
        cosDist: [cosDistDom] real;

    // TODO: verbose output
    //writeln(" Got V: ", V.shape);
    //writeln(" cosDistDom.dims(1) ", cosDistDom.dims());

    // Pre-compute repeated cosim's
    var XYii: [Xdom.dim(1)] real;
    [i in Xdom.dim(1)] XYii[i] = dot(X[i,..], Y[i,..]);

    forall i in Xdom.dim(1) {
      const x1 = Xii[i];
      for j in i+1..Xdom.dim(1).size {
        // Do cosim
        const x2 = XYii[j];
        const c = 1 - dot(X[i,..], Y[j,..]) / (x1 * x2);
        cosDist[i,j] = c;
        cosDist[j,i] = c;
      }
    }
    return cosDist;
  }
}
{'fraction_non_alphanumeric': 0.15721695582185655, 'fraction_numerical': 0.016097299230906813, 'mean_word_length': 1.717201166180758, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2586355', 'n_tokens_mistral': 1978, 'n_tokens_neox': 1923, 'n_words': 501}