<commit_before>#include <stdlib.h>
#include <string.h>

#include "shotgun/lib/shotgun.h"
#include "shotgun/lib/heap.h"
#include "shotgun/lib/cpu.h"
#include "shotgun/lib/methctx.h"
#include "shotgun/lib/bytearray.h"
#include "shotgun/lib/baker.h"
#include "shotgun/lib/tuple.h"

/* how many times object should be traced before tenuring */
#define DEFAULT_TENURE_AGE 6

/*
creates and initializes new copying GC instance
it start with space A as "from" space and B as "to" space
*/
baker_gc baker_gc_new(int size) {
  baker_gc g;
  g = (baker_gc)calloc(1, sizeof(struct baker_gc_struct));
  g->remember_set = ptr_array_new(8);
  g->seen_weak_refs = ptr_array_new(8);
  g->tenured_objects = ptr_array_new(8);

  g->space_a = heap_new(size);
  g->space_b = heap_new(size);
  g->current = g->space_a;
  g->next =    g->space_b;
  g->used =    0;
  g->tenure_age = DEFAULT_TENURE_AGE;
  g->become_from = Qnil;
  g->become_to = Qnil;
  return g;
}

/* prints various garbage collector information */
void baker_gc_describe(baker_gc g) {
  printf("Size :    %x (%d)\n", (unsigned int)g->current->size, (int)g->current->size);
  printf("Current (\"from\") heap space address: %p => %p\n", (void*)g->current->address, (void*)g->current->last);
  printf("Next (\"to\") heap space address:    %p => %p\n", (void*)g->next->address, (void*)g->next->last);
  printf("Remember set size: %zd\n", ptr_array_length(g->remember_set));
}

/* baker GC uses two heap spaces of the same size */
int baker_gc_memory_allocated(baker_gc g) {
  return g->current->size * 2;
}

/* returns how much memory is used by "from" heap space */
int baker_gc_memory_in_use(baker_gc g) {
  return g->current->current - g->current->address;
}

/* enlarges "to" space of the heap (where new objects are allocated) by value of sz */
int baker_gc_enlarge_next(baker_gc g, size_t sz) {
  rheap h;
  h = heap_new(sz);
  return baker_gc_set_next(g, h);
}

/* sets "to" heap space, always returns TRUE */
int baker_gc_set_next(baker_gc g, rheap h) {
  if(g->next == g->space_a) {
    g->space_a = h;
  } else {
    g->space_b = h;
  }
  g->next = h;
  return TRUE;
}

/* gets start address of current "from" heap space */
address baker_gc_start_address(baker_gc g) {
  return g->current->address;
}

/* total baker GC memory usage */
size_t baker_gc_used(baker_gc g) {
  return g->used;
}

/* reset memory usage info */
void baker_gc_reset_used(baker_gc g) {
  g->used = 0;
}

/* performs flip operation on heap spaces */
int baker_gc_swap(baker_gc g) {
  rheap tmp;
  tmp = g->current;
  g->current = g->next;
  g->next = tmp;

  g->last_start = (char*)tmp->address;
  g->last_end =   (char*)tmp->current;

  heap_reset(tmp);
  return TRUE;
}

int baker_gc_destroy(baker_gc g) {
  heap_deallocate(g->space_a);
  heap_deallocate(g->space_b);
  free(g);
  return TRUE;
}

/* sets forwarding pointer on object */
void baker_gc_set_forwarding_address(OBJECT obj, OBJECT dest) {
  SET_FORWARDED(obj);
  obj->klass = dest;
}

OBJECT baker_gc_forwarded_object(OBJECT obj) {
  OBJECT out = obj->klass;
  CHECK_PTR(out);
  return out;
}

#define baker_gc_maybe_mutate(st, g, iobj) ({     \
  OBJECT ret;                                 \
  if(baker_gc_forwarded_p(iobj)) {            \
    ret = baker_gc_forwarded_object(iobj);    \
  } else if(baker_gc_contains_p(g, iobj) || heap_contains_p(state->om->contexts, iobj)) {   \
    assert(((OBJECT)iobj)->klass); \
    ret = baker_gc_mutate_object(st, g, iobj);    \
  } else {                                    \
    ret = iobj;                               \
  }                                           \
  ret; })


int _object_stores_bytes(OBJECT self);
static int depth = 0;
static ptr_array _track_refs = NULL;

void track_ref(OBJECT obj) {
  if(!_track_refs) {
    _track_refs = ptr_array_new(8);
  }
  ptr_array_append(_track_refs, (xpointer)obj);
}

void untrack_ref(OBJECT obj) {
  if(!_track_refs) return;
  ptr_array_remove_fast(_track_refs, (xpointer)obj);
}

static void _mutate_references(STATE, baker_gc g, OBJECT iobj) {
  OBJECT cls, tmp, mut;
  int i, fields;

  void *om = g->om;

#if 0
  if(_track_refs) {
    if (ptr_array_contains(_track_refs, (xpointer)iobj))
        printf("Found2 %p!\n", (void*)iobj);
  }
#endif

  //printf("%d: Mutating class of %p\n", ++depth, iobj);

  /* CLASS_OBJECT() gets the object directly in the klass field
   * of iobj, which is very likely it's metaclass, not it's
   * formal class. FYI. */
  cls = CLASS_OBJECT(iobj);
  if(REFERENCE_P(cls)) {
    cls = baker_gc_maybe_mutate(state, g, cls);
  }

  SET_CLASS(iobj, cls);

  xassert(!FORWARDED_P(iobj));

  if(iobj->RefsAreWeak) {
    // printf("%p has weak refs.\n", (void*)iobj);
    ptr_array_append(g->seen_weak_refs, (xpointer)iobj);
    depth--;
    return;
  }

  //printf("%d: Mutating references of %p\n", depth, iobj);

  if(!_object_stores_bytes(iobj)) {
		/* follow object field references and mutate them */
    fields = NUM_FIELDS(iobj);
    for(i = 0; i < fields; i++) {
      tmp = NTH_FIELD(iobj, i);
      if(!REFERENCE_P(tmp)) continue;
			/* TODO: duplicated in other places: extract to separate function? */
      if(FORWARDED_P(tmp)) {
        mut = tmp->klass;
      } else if(heap_contains_p(g->current, tmp) || heap_contains_p(state->om->contexts, tmp)) {
        mut = baker_gc_mutate_object(state, g, tmp);
      } else {
        mut = tmp;<commit_msg>Fixing gc when using Data_Wrap_Struct with null mark and free functions.<commit_after>#include <stdlib.h>
#include <string.h>

#include "shotgun/lib/shotgun.h"
#include "shotgun/lib/heap.h"
#include "shotgun/lib/cpu.h"
#include "shotgun/lib/methctx.h"
#include "shotgun/lib/bytearray.h"
#include "shotgun/lib/baker.h"
#include "shotgun/lib/tuple.h"

/* how many times object should be traced before tenuring */
#define DEFAULT_TENURE_AGE 6

/*
creates and initializes new copying GC instance
it start with space A as "from" space and B as "to" space
*/
baker_gc baker_gc_new(int size) {
  baker_gc g;
  g = (baker_gc)calloc(1, sizeof(struct baker_gc_struct));
  g->remember_set = ptr_array_new(8);
  g->seen_weak_refs = ptr_array_new(8);
  g->tenured_objects = ptr_array_new(8);

  g->space_a = heap_new(size);
  g->space_b = heap_new(size);
  g->current = g->space_a;
  g->next =    g->space_b;
  g->used =    0;
  g->tenure_age = DEFAULT_TENURE_AGE;
  g->become_from = Qnil;
  g->become_to = Qnil;
  return g;
}

/* prints various garbage collector information */
void baker_gc_describe(baker_gc g) {
  printf("Size :    %x (%d)\n", (unsigned int)g->current->size, (int)g->current->size);
  printf("Current (\"from\") heap space address: %p => %p\n", (void*)g->current->address, (void*)g->current->last);
  printf("Next (\"to\") heap space address:    %p => %p\n", (void*)g->next->address, (void*)g->next->last);
  printf("Remember set size: %zd\n", ptr_array_length(g->remember_set));
}

/* baker GC uses two heap spaces of the same size */
int baker_gc_memory_allocated(baker_gc g) {
  return g->current->size * 2;
}

/* returns how much memory is used by "from" heap space */
int baker_gc_memory_in_use(baker_gc g) {
  return g->current->current - g->current->address;
}

/* enlarges "to" space of the heap (where new objects are allocated) by value of sz */
int baker_gc_enlarge_next(baker_gc g, size_t sz) {
  rheap h;
  h = heap_new(sz);
  return baker_gc_set_next(g, h);
}

/* sets "to" heap space, always returns TRUE */
int baker_gc_set_next(baker_gc g, rheap h) {
  if(g->next == g->space_a) {
    g->space_a = h;
  } else {
    g->space_b = h;
  }
  g->next = h;
  return TRUE;
}

/* gets start address of current "from" heap space */
address baker_gc_start_address(baker_gc g) {
  return g->current->address;
}

/* total baker GC memory usage */
size_t baker_gc_used(baker_gc g) {
  return g->used;
}

/* reset memory usage info */
void baker_gc_reset_used(baker_gc g) {
  g->used = 0;
}

/* performs flip operation on heap spaces */
int baker_gc_swap(baker_gc g) {
  rheap tmp;
  tmp = g->current;
  g->current = g->next;
  g->next = tmp;

  g->last_start = (char*)tmp->address;
  g->last_end =   (char*)tmp->current;

  heap_reset(tmp);
  return TRUE;
}

int baker_gc_destroy(baker_gc g) {
  heap_deallocate(g->space_a);
  heap_deallocate(g->space_b);
  free(g);
  return TRUE;
}

/* sets forwarding pointer on object */
void baker_gc_set_forwarding_address(OBJECT obj, OBJECT dest) {
  SET_FORWARDED(obj);
  obj->klass = dest;
}

OBJECT baker_gc_forwarded_object(OBJECT obj) {
  OBJECT out = obj->klass;
  CHECK_PTR(out);
  return out;
}

#define baker_gc_maybe_mutate(st, g, iobj) ({     \
  OBJECT ret;                                 \
  if(baker_gc_forwarded_p(iobj)) {            \
    ret = baker_gc_forwarded_object(iobj);    \
  } else if(baker_gc_contains_p(g, iobj) || heap_contains_p(state->om->contexts, iobj)) {   \
    assert(((OBJECT)iobj)->klass); \
    ret = baker_gc_mutate_object(st, g, iobj);    \
  } else {                                    \
    ret = iobj;                               \
  }                                           \
  ret; })


int _object_stores_bytes(OBJECT self);
static int depth = 0;
static ptr_array _track_refs = NULL;

void track_ref(OBJECT obj) {
  if(!_track_refs) {
    _track_refs = ptr_array_new(8);
  }
  ptr_array_append(_track_refs, (xpointer)obj);
}

void untrack_ref(OBJECT obj) {
  if(!_track_refs) return;
  ptr_array_remove_fast(_track_refs, (xpointer)obj);
}

static void _mutate_references(STATE, baker_gc g, OBJECT iobj) {
  OBJECT cls, tmp, mut;
  int i, fields;

  void *om = g->om;

#if 0
  if(_track_refs) {
    if (ptr_array_contains(_track_refs, (xpointer)iobj))
        printf("Found2 %p!\n", (void*)iobj);
  }
#endif

  //printf("%d: Mutating class of %p\n", ++depth, iobj);

  /* CLASS_OBJECT() gets the object directly in the klass field
   * of iobj, which is very likely it's metaclass, not it's
   * formal class. FYI. */
  cls = CLASS_OBJECT(iobj);
  if(REFERENCE_P(cls)) {
    cls = baker_gc_maybe_mutate(state, g, cls);
  }

  SET_CLASS(iobj, cls);

  xassert(!FORWARDED_P(iobj));

  if(iobj->RefsAreWeak) {
    // printf("%p has weak refs.\n", (void*)iobj);
    ptr_array_append(g->seen_weak_refs, (xpointer)iobj);
    depth--;
    return;
  }

  //printf("%d: Mutating references of %p\n", depth, iobj);

  if(!_object_stores_bytes(iobj)) {
		/* follow object field references and mutate them */
    fields = NUM_FIELDS(iobj);
    for(i = 0; i < fields; i++) {
      tmp = NTH_FIELD(iobj, i);
      if(!tmp || !REFERENCE_P(tmp)) continue;
			/* TODO: duplicated in other places: extract to separate function? */
      if(FORWARDED_P(tmp)) {
        mut = tmp->klass;
      } else if(heap_contains_p(g->current, tmp) || heap_contains_p(state->om->contexts, tmp)) {
        mut = baker_gc_mutate_object(state, g, tmp);
      } else {
        mut = tmp;
{'fraction_non_alphanumeric': 0.1473761874020105, 'fraction_numerical': 0.0023978603707461034, 'mean_word_length': 3.2997620935765264, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 137, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6231740', 'n_tokens_mistral': 4190, 'n_tokens_neox': 4092, 'n_words': 938}