<commit_before>//- Copyright 2013 the Neutrino authors (see AUTHORS).
//- Licensed under the Apache License, Version 2.0 (see LICENSE).

#include "alloc.h"
#include "freeze.h"
#include "runtime.h"
#include "safe-inl.h"
#include "serialize.h"
#include "try-inl.h"
#include "value-inl.h"
#include "plankton.h"


// --- B y t e   s t r e a m ---

// A stream that allows bytes to be read one at a time from a blob.
typedef struct {
  blob_t blob;
  size_t cursor;
} byte_stream_t;

// Initializes a stream to read from the beginning of the given blob.
static void byte_stream_init(byte_stream_t *stream, blob_t blob) {
  stream->blob = blob;
  stream->cursor = 0;
}

// Returns true iff data can be read from this stream.
bool byte_stream_has_more(byte_stream_t *stream) {
  return stream->cursor < blob_byte_length(stream->blob);
}

// Returns the next byte from the given byte stream.
byte_t byte_stream_read(byte_stream_t *stream) {
  CHECK_TRUE("byte stream empty", byte_stream_has_more(stream));
  byte_t result = blob_byte_at(stream->blob, stream->cursor);
  stream->cursor++;
  return result;
}


// --- S e r i a l i z e ---

// Collection of state used when serializing data.
struct serialize_state_t {
  // The plankton assembler.
  pton_assembler_t *assm;
  // Map from objects we've seen to their index.
  value_t ref_map;
  // The offset of the next object we're going to write.
  size_t object_offset;
  // The runtime to use for heap allocation.
  runtime_t *runtime;
};

object_factory_t new_object_factory(new_empty_object_t *new_empty_object,
    set_object_fields_t *set_object_fields, void *data) {
  object_factory_t result = {new_empty_object, set_object_fields, data};
  return result;
}

value_t value_mapping_apply(value_mapping_t *mapping, value_t value, runtime_t *runtime) {
  return (mapping->function)(value, runtime, mapping->data);
}

// Initialize serialization state.
static value_t serialize_state_init(serialize_state_t *state, runtime_t *runtime,
    pton_assembler_t *assm) {
  state->assm = assm;
  state->object_offset = 0;
  state->runtime = runtime;
  TRY_SET(state->ref_map, new_heap_id_hash_map(runtime, 16));
  return success();
}

// Serialize any (non-condition) value on the given buffer.
static value_t value_serialize(value_t value, serialize_state_t *state);

static value_t integer_serialize(value_t value, pton_assembler_t *assm) {
  CHECK_DOMAIN(vdInteger, value);
  int64_t int_value = get_integer_value(value);
  pton_assembler_emit_int64(assm, int_value);
  return success();
}

value_t serialize_array(value_t value, serialize_state_t *state) {
  CHECK_FAMILY(ofArray, value);
  int64_t length = get_array_length(value);
  pton_assembler_begin_array(state->assm, (uint32_t) length);
  for (int64_t i = 0; i < length; i++) {
    TRY(value_serialize(get_array_at(value, i), state));
  }
  return success();
}

static value_t map_contents_serialize(uint32_t entry_count, id_hash_map_iter_t *iter,
    serialize_state_t *state) {
  uint32_t entries_written = 0;
  while (id_hash_map_iter_advance(iter)) {
    value_t key;
    value_t value;
    id_hash_map_iter_get_current(iter, &key, &value);
    TRY(value_serialize(key, state));
    TRY(value_serialize(value, state));
    entries_written++;
  }
  CHECK_EQ("serialized map length", entry_count, entries_written);
  return success();
}

value_t serialize_id_hash_map(value_t value, serialize_state_t *state) {
  CHECK_FAMILY(ofIdHashMap, value);
  uint32_t entry_count = (uint32_t) get_id_hash_map_size(value);
  pton_assembler_begin_map(state->assm, entry_count);
  id_hash_map_iter_t iter;
  id_hash_map_iter_init(&iter, value);
  return map_contents_serialize(entry_count, &iter, state);
}

static bool skip_reified_tag(value_t tag) {
  return in_family(ofKey, tag);
}

value_t serialize_reified_arguments(value_t value, serialize_state_t *state) {
  CHECK_FAMILY(ofReifiedArguments, value);
  value_t values = get_reified_arguments_values(value);
  value_t tags = get_reified_arguments_tags(value);
  uint32_t raw_argc = (uint32_t) get_call_tags_entry_count(tags);
  uint32_t argc = 0;
  // First count how may tags are not going to be skipped.
  for (uint32_t i = 0; i < raw_argc; i++) {
    value_t tag = get_call_tags_tag_at(tags, i);
    if (!skip_reified_tag(tag))
      argc++;
  }<commit_msg>Self-review<commit_after>//- Copyright 2013 the Neutrino authors (see AUTHORS).
//- Licensed under the Apache License, Version 2.0 (see LICENSE).

#include "alloc.h"
#include "freeze.h"
#include "runtime.h"
#include "safe-inl.h"
#include "serialize.h"
#include "try-inl.h"
#include "value-inl.h"
#include "plankton.h"


// --- B y t e   s t r e a m ---

// A stream that allows bytes to be read one at a time from a blob.
typedef struct {
  blob_t blob;
  size_t cursor;
} byte_stream_t;

// Initializes a stream to read from the beginning of the given blob.
static void byte_stream_init(byte_stream_t *stream, blob_t blob) {
  stream->blob = blob;
  stream->cursor = 0;
}

// Returns true iff data can be read from this stream.
bool byte_stream_has_more(byte_stream_t *stream) {
  return stream->cursor < blob_byte_length(stream->blob);
}

// Returns the next byte from the given byte stream.
byte_t byte_stream_read(byte_stream_t *stream) {
  CHECK_TRUE("byte stream empty", byte_stream_has_more(stream));
  byte_t result = blob_byte_at(stream->blob, stream->cursor);
  stream->cursor++;
  return result;
}


// --- S e r i a l i z e ---

// Collection of state used when serializing data.
struct serialize_state_t {
  // The plankton assembler.
  pton_assembler_t *assm;
  // Map from objects we've seen to their index.
  value_t ref_map;
  // The offset of the next object we're going to write.
  size_t object_offset;
  // The runtime to use for heap allocation.
  runtime_t *runtime;
};

object_factory_t new_object_factory(new_empty_object_t *new_empty_object,
    set_object_fields_t *set_object_fields, void *data) {
  object_factory_t result = {new_empty_object, set_object_fields, data};
  return result;
}

value_t value_mapping_apply(value_mapping_t *mapping, value_t value, runtime_t *runtime) {
  return (mapping->function)(value, runtime, mapping->data);
}

// Initialize serialization state.
static value_t serialize_state_init(serialize_state_t *state, runtime_t *runtime,
    pton_assembler_t *assm) {
  state->assm = assm;
  state->object_offset = 0;
  state->runtime = runtime;
  TRY_SET(state->ref_map, new_heap_id_hash_map(runtime, 16));
  return success();
}

// Serialize any (non-condition) value on the given buffer.
static value_t value_serialize(value_t value, serialize_state_t *state);

static value_t integer_serialize(value_t value, pton_assembler_t *assm) {
  CHECK_DOMAIN(vdInteger, value);
  int64_t int_value = get_integer_value(value);
  pton_assembler_emit_int64(assm, int_value);
  return success();
}

value_t serialize_array(value_t value, serialize_state_t *state) {
  CHECK_FAMILY(ofArray, value);
  int64_t length = get_array_length(value);
  pton_assembler_begin_array(state->assm, (uint32_t) length);
  for (int64_t i = 0; i < length; i++) {
    TRY(value_serialize(get_array_at(value, i), state));
  }
  return success();
}

static value_t map_contents_serialize(uint32_t entry_count, id_hash_map_iter_t *iter,
    serialize_state_t *state) {
  uint32_t entries_written = 0;
  while (id_hash_map_iter_advance(iter)) {
    value_t key;
    value_t value;
    id_hash_map_iter_get_current(iter, &key, &value);
    TRY(value_serialize(key, state));
    TRY(value_serialize(value, state));
    entries_written++;
  }
  CHECK_EQ("serialized map length", entry_count, entries_written);
  return success();
}

value_t serialize_id_hash_map(value_t value, serialize_state_t *state) {
  CHECK_FAMILY(ofIdHashMap, value);
  uint32_t entry_count = (uint32_t) get_id_hash_map_size(value);
  pton_assembler_begin_map(state->assm, entry_count);
  id_hash_map_iter_t iter;
  id_hash_map_iter_init(&iter, value);
  return map_contents_serialize(entry_count, &iter, state);
}

// Skip this tag when serializing a reified arguments set?
static bool skip_reified_tag(value_t tag) {
  return in_family(ofKey, tag);
}

value_t serialize_reified_arguments(value_t value, serialize_state_t *state) {
  CHECK_FAMILY(ofReifiedArguments, value);
  value_t values = get_reified_arguments_values(value);
  value_t tags = get_reified_arguments_tags(value);
  uint32_t raw_argc = (uint32_t) get_call_tags_entry_count(tags);
  uint32_t argc = 0;
  // First count how may tags are not going to be skipped.
  for (uint32_t i = 0; i < raw_argc; i++) {
    value_t tag = get_call_tags_tag_at(tags, i);
    if (!skip_reified_tag(tag))
      argc++;
  }
{'fraction_non_alphanumeric': 0.10563948794833353, 'fraction_numerical': 0.009226156152692884, 'mean_word_length': 4.867388362652233, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3171519', 'n_tokens_mistral': 3147, 'n_tokens_neox': 3063, 'n_words': 781}