<commit_before>package shark

import com.esotericsoftware.kryo.Kryo
import de.javakaffee.kryoserializers.ArraysAsListSerializer
import java.util.Arrays

class KryoRegistrator extends spark.KryoRegistrator {
  def registerClasses(kryo: Kryo) {

    kryo.register(classOf[exec.ReduceKey])

    // Java Arrays.asList returns an internal class that cannot be serialized
    // by default Kryo. This provides a workaround.
    kryo.register(Arrays.asList().getClass, new ArraysAsListSerializer(kryo))
  }
}<commit_msg>Added custom byte array serializer to Kryo. Fixes #22<commit_after>package shark

import com.esotericsoftware.kryo.Kryo
import de.javakaffee.kryoserializers.ArraysAsListSerializer
import java.util.Arrays
import java.nio.ByteBuffer
import com.esotericsoftware.kryo.serialize.{IntSerializer, SimpleSerializer}

class KryoRegistrator extends spark.KryoRegistrator {
  def registerClasses(kryo: Kryo) {

    kryo.register(classOf[exec.ReduceKey])

    // Java Arrays.asList returns an internal class that cannot be serialized
    // by default Kryo. This provides a workaround.
    kryo.register(Arrays.asList().getClass, new ArraysAsListSerializer(kryo))
    
    // This is a work-around for Kryo's byte-by-byte serialization of byte
    // arrays. Note that it does not handle nulls because null rows are already
    // encoded using Hive's SerDe.
    kryo.register(classOf[Array[Byte]], new SimpleSerializer[Array[Byte]]() {
      def write (buffer: ByteBuffer, arr: Array[Byte]) {
        IntSerializer.put(buffer, arr.length, true)
        buffer.put(arr)
      }
      def read (buffer: ByteBuffer): Array[Byte] =  {
        val len = IntSerializer.get(buffer, true)
        val arr = new Array[Byte](len)
        buffer.get(arr)
        arr
      }
    })
  }
}
{'fraction_non_alphanumeric': 0.0922384701912261, 'fraction_numerical': 0.0011248593925759281, 'mean_word_length': 3.744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1357941', 'n_tokens_mistral': 573, 'n_tokens_neox': 566, 'n_words': 142}