<commit_before>#include <stdbool.h>
#include <stdio.h>
#include <string.h>

#include "ruby.h"

VALUE thaw(VALUE, VALUE);
static VALUE read_object();
static int read_extended_size();
static int read_compact_size();
static void read_n_hash_pairs(VALUE, int);
static void read_n_array_entries(VALUE, int);
static VALUE read_string(bool);
static void read_magic_numbers();

enum states
{
  READ_EXTENDED_SIZE,
  READ_COMPACT_SIZE,
  READ_NEW_OBJECT
};

enum perl_types
{
  PT_HASH_KEY   = 1,
  PT_ARRAY      = 2,
  PT_HASH       = 3,
  PT_VECTOR     = 4,
  PT_UNDEF      = 5,
  PT_STRING     = 10,
  PT_STRING_ALT = 23
};

// Used globally. Raptors. I know.
static unsigned char *serialized;
static unsigned char *serialized_end;

static void
check_pointer(unsigned char *ptr)
{
  extern unsigned char *serialized_end;
  if (ptr > serialized_end) {
    rb_raise(rb_eRangeError, "malformed data");
  }
}

VALUE
thaw(VALUE self, VALUE str)
{
  Check_Type(str, T_STRING);
  extern unsigned char *serialized, *serialized_end;

  serialized = RSTRING_PTR(str);
  serialized_end = serialized + RSTRING_LEN(str);
  
  read_magic_numbers();
  
  return read_object();
}

static void
read_magic_numbers()
{
  serialized++;
  serialized++;
}

static VALUE
read_object()
{
  extern unsigned char *serialized;
  check_pointer(serialized);
  int type = *serialized++;
  int size = read_extended_size();

  VALUE object;

  switch(type) {
  case PT_HASH:
    object = rb_hash_new();
    read_n_hash_pairs(object, size);
    break;
  case PT_ARRAY:
    object = rb_ary_new();
    read_n_array_entries(object, size);
    break;
  case PT_UNDEF:
    object = Qnil;
    break;
  }
  
  return object;
}

  
static void
read_n_hash_pairs(VALUE hash, int num)
{
  extern unsigned char *serialized;
  check_pointer(serialized);
  
  if (num == 0) {
    return;
  }
  
  int type = *serialized++;
  VALUE temp;
  VALUE str;
  
  switch(type) {
  case PT_UNDEF:
    rb_hash_aset(hash, read_string(true), Qnil);
    break;
  case PT_VECTOR:
    temp = read_object();
    str = read_string(true);
    rb_hash_aset(hash, str, temp);
    break;
  case PT_STRING:
  case PT_STRING_ALT:
    temp = read_string(false);
    rb_hash_aset(hash, read_string(true), temp);
    break;
  }

  read_n_hash_pairs(hash, num-1);
}

static void
read_n_array_entries(VALUE array, int num)
{
  extern unsigned char *serialized;
  check_pointer(serialized);
  
  if (num == 0) {
    return;
  }
  
  VALUE item = Qnil;
  int type = *serialized++;

  switch(type) {<commit_msg>A little more refactoring, a bit of documentation<commit_after>#include <stdbool.h>
#include <stdio.h>
#include <string.h>

#include "ruby.h"

VALUE thaw(VALUE, VALUE);
static VALUE read_object();
static int read_extended_size();
static int read_compact_size();
static void read_n_hash_pairs(VALUE, int);
static void read_n_array_entries(VALUE, int);
static VALUE read_string(bool);
static void read_magic_numbers();
static void check_pointer(unsigned char*);

enum perl_types
{
  PT_HASH_KEY   = 1,
  PT_ARRAY      = 2,
  PT_HASH       = 3,
  PT_VECTOR     = 4,
  PT_UNDEF      = 5,
  PT_STRING     = 10,
  PT_STRING_ALT = 23
};

// Used globally. Raptors. I know.
static unsigned char *serialized;
static unsigned char *serialized_end;

/*
 * Given a perl Storable frozen blob, decode it into a ruby data structure.
 */
VALUE
thaw(VALUE self, VALUE str)
{
  Check_Type(str, T_STRING);
  extern unsigned char *serialized, *serialized_end;

  serialized = RSTRING_PTR(str);
  serialized_end = serialized + RSTRING_LEN(str);
  
  read_magic_numbers();
  
  return read_object();
}

/*
 * Malformed strings can theoretically cause segfaults. Segfaults are bad.
 * We'll check pretty much everything we do against the pre-computed end-of-string.
 */
static void
check_pointer(unsigned char *ptr)
{
  extern unsigned char *serialized_end;
  if (ptr > serialized_end) {
    rb_raise(rb_eRangeError, "malformed data");
  }
}

/*
 * Certain test cases start with \005\006. Other ones don't.
 * This will need to be fixed eventually.
 */
static void
read_magic_numbers()
{
  serialized++;
  serialized++;
}

/*
 * Figure out what type of object is at the front of serialized, read it in, potentially
 * recursively creating several other sub-objects in the process, and return it.
 */
static VALUE
read_object()
{
  extern unsigned char *serialized;
  check_pointer(serialized);
  int type = *serialized++;
  int size = read_extended_size();

  VALUE object;

  switch(type) {
  case PT_HASH:
    object = rb_hash_new();
    read_n_hash_pairs(object, size);
    break;
  case PT_ARRAY:
    object = rb_ary_new();
    read_n_array_entries(object, size);
    break;
  case PT_UNDEF:
    object = Qnil;
    break;
  }
  
  return object;
}

  
/*
 * We've already created a hash, and read the size of it.
 * Now we need to read in n items, and add them to the hash.
 */
static void
read_n_hash_pairs(VALUE hash, int num)
{
  extern unsigned char *serialized;
  check_pointer(serialized);
  
  if (num == 0) {
    return;
  }
  
  int type = *serialized++;
  VALUE temp;
  VALUE str;
  
  switch(type) {
  case PT_UNDEF:
    rb_hash_aset(hash, read_string(true), Qnil);
    break;
  case PT_VECTOR:
    temp = read_object();
    str = read_string(true);
    rb_hash_aset(hash, str, temp);
    break;
  case PT_STRING:
  case PT_STRING_ALT:
    temp = read_string(false);
    rb_hash_aset(hash, read_string(true), temp);
    break;
  }

  read_n_hash_pairs(hash, num-1);
}

/*
 * We've already created an array, and read the size of it.
 * Now we need to read in n items, and add them to the array.
 */
static void
read_n_array_entries(VALUE array, int num)
{
  extern unsigned char *serialized;
  check_pointer(serialized);
  
  if (num == 0) {
    return;
  }
  
  VALUE item = Qnil;
  int type = *serialized++;

  switch(type) {
{'fraction_non_alphanumeric': 0.10590248075278015, 'fraction_numerical': 0.0051325919589392645, 'mean_word_length': 3.6323296354992074, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1497560', 'n_tokens_mistral': 2144, 'n_tokens_neox': 2072, 'n_words': 570}