<commit_before>import org.apache.spark.api.java.JavaRDD
import org.apache.spark.mllib.regression._
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.DoubleBuffer

class PythonMLLibAPI extends Serializable {
  def deserializeDoubleVector(bytes: Array[Byte]): Array[Double] = {
    val packetLength = bytes.length;
    if (packetLength < 16) {
      throw new IllegalArgumentException("Byte array too short.");
    }
    val bb = ByteBuffer.wrap(bytes);
    bb.order(ByteOrder.nativeOrder());
    val magic = bb.getLong();
    if (magic != 1) {
      throw new IllegalArgumentException("Magic " + magic + " is wrong.");
    }
    val length = bb.getLong();
    if (packetLength != 16 + 8 * length) {
      throw new IllegalArgumentException("Length " + length + "is wrong.");
    }
    val db = bb.asDoubleBuffer();
    val ans = new Array[Double](length.toInt);
    db.get(ans);
    return ans;
  }

  def serializeDoubleVector(doubles: Array[Double]): Array[Byte] = {
    val len = doubles.length;
    val bytes = new Array[Byte](16 + 8 * len);
    val bb = ByteBuffer.wrap(bytes);
    bb.order(ByteOrder.nativeOrder());
    bb.putLong(1);
    bb.putLong(len);
    val db = bb.asDoubleBuffer();
    db.put(doubles);
    return bytes;
  }

  def trainLinearRegressionModel(dataBytesJRDD: JavaRDD[Array[Byte]]):
      java.util.List[java.lang.Object] = {
    val data = dataBytesJRDD.rdd.map(x => deserializeDoubleVector(x))
        .map(v => LabeledPoint(v(0), v.slice(1, v.length)));
    val model = LinearRegressionWithSGD.train(data, 222);
    val ret = new java.util.LinkedList[java.lang.Object]();
    ret.add(serializeDoubleVector(model.weights));
    ret.add(model.intercept: java.lang.Double);
    return ret;
  }
}<commit_msg>Un-semicolon PythonMLLibAPI.<commit_after>import org.apache.spark.api.java.JavaRDD
import org.apache.spark.mllib.regression._
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.DoubleBuffer

class PythonMLLibAPI extends Serializable {
  def deserializeDoubleVector(bytes: Array[Byte]): Array[Double] = {
    val packetLength = bytes.length
    if (packetLength < 16) {
      throw new IllegalArgumentException("Byte array too short.")
    }
    val bb = ByteBuffer.wrap(bytes)
    bb.order(ByteOrder.nativeOrder())
    val magic = bb.getLong()
    if (magic != 1) {
      throw new IllegalArgumentException("Magic " + magic + " is wrong.")
    }
    val length = bb.getLong()
    if (packetLength != 16 + 8 * length) {
      throw new IllegalArgumentException("Length " + length + "is wrong.")
    }
    val db = bb.asDoubleBuffer()
    val ans = new Array[Double](length.toInt)
    db.get(ans)
    return ans
  }

  def serializeDoubleVector(doubles: Array[Double]): Array[Byte] = {
    val len = doubles.length
    val bytes = new Array[Byte](16 + 8 * len)
    val bb = ByteBuffer.wrap(bytes)
    bb.order(ByteOrder.nativeOrder())
    bb.putLong(1)
    bb.putLong(len)
    val db = bb.asDoubleBuffer()
    db.put(doubles)
    return bytes
  }

  def trainLinearRegressionModel(dataBytesJRDD: JavaRDD[Array[Byte]]):
      java.util.List[java.lang.Object] = {
    val data = dataBytesJRDD.rdd.map(x => deserializeDoubleVector(x))
        .map(v => LabeledPoint(v(0), v.slice(1, v.length)))
    val model = LinearRegressionWithSGD.train(data, 222)
    val ret = new java.util.LinkedList[java.lang.Object]()
    ret.add(serializeDoubleVector(model.weights))
    ret.add(model.intercept: java.lang.Double)
    return ret
  }
}
{'fraction_non_alphanumeric': 0.13308528906697195, 'fraction_numerical': 0.008586147681740127, 'mean_word_length': 3.480769230769231, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1089178', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1119, 'n_words': 241}