<commit_before>#include <stdlib.h>
#include <iostream>
#include <sys/time.h>

#include "config.h"
#include "vm.hpp"
#include "objectmemory.hpp"
#include "gc/marksweep.hpp"
#include "gc/baker.hpp"
#include "gc/immix.hpp"
#include "gc/inflated_headers.hpp"
#include "gc/walker.hpp"
#include "on_stack.hpp"

#include "config_parser.hpp"

#include "builtin/class.hpp"
#include "builtin/fixnum.hpp"
#include "builtin/tuple.hpp"
#include "builtin/io.hpp"
#include "builtin/fiber.hpp"
#include "builtin/string.hpp"
#include "builtin/lookuptable.hpp"
#include "builtin/ffi_pointer.hpp"
#include "builtin/data.hpp"
#include "builtin/dir.hpp"
#include "builtin/array.hpp"
#include "builtin/thread.hpp"

#include "capi/handle.hpp"
#include "configuration.hpp"

#include "global_cache.hpp"

#include "instruments/tooling.hpp"

// Used by XMALLOC at the bottom
static long gc_malloc_threshold = 0;
static long bytes_until_collection = 0;

namespace rubinius {

  Object* object_watch = 0;

  /* ObjectMemory methods */
  ObjectMemory::ObjectMemory(STATE, Configuration& config)
    : young_(new BakerGC(this, config.gc_bytes))
    , mark_sweep_(new MarkSweepGC(this, config))
    , immix_(new ImmixGC(this))
    , inflated_headers_(new InflatedHeaders)
    , mark_(1)
    , code_manager_(&state->shared)
    , allow_gc_(true)
    , slab_size_(4096)
    , running_finalizers_(false)
    , added_finalizers_(false)
    , finalizer_thread_(state, nil<Thread>())

    , collect_young_now(false)
    , collect_mature_now(false)
    , root_state_(state)
  {
    // TODO Not sure where this code should be...
    if(char* num = getenv("RBX_WATCH")) {
      object_watch = (Object*)strtol(num, NULL, 10);
      std::cout << "Watching for " << object_watch << "\n";
    }

    collect_mature_now = false;
    last_object_id = 0;

    large_object_threshold = config.gc_large_object;
    young_->set_lifetime(config.gc_lifetime);

    if(config.gc_autotune) young_->set_autotune();

    for(size_t i = 0; i < LastObjectType; i++) {
      type_info[i] = NULL;
    }

    TypeInfo::init(this);

    gc_malloc_threshold = config.gc_malloc_threshold;
    bytes_until_collection = gc_malloc_threshold;
  }

  ObjectMemory::~ObjectMemory() {
    mark_sweep_->free_objects();

    // @todo free immix data

    for(size_t i = 0; i < LastObjectType; i++) {
      if(type_info[i]) delete type_info[i];
    }

    delete immix_;
    delete mark_sweep_;
    delete young_;

    // Must be last
    // delete inflated_headers_;

  }

  void ObjectMemory::on_fork() {
    finalizer_lock_.init();
    finalizer_var_.init();
    finalizer_thread_.set(nil<Thread>());
  }

  void ObjectMemory::assign_object_id(STATE, Object* obj) {
    SYNC(state);

    // Double check we've got no id still after the lock.
    if(obj->object_id() > 0) return;

    obj->set_object_id(state, state->om, ++last_object_id);
  }

  Integer* ObjectMemory::assign_object_id_ivar(STATE, Object* obj) {
    SYNC(state);
    Object* id = obj->get_ivar(state, G(sym_object_id));
    if(id->nil_p()) {
      /* All references have an even object_id. last_object_id starts out at 0
       * but we don't want to use 0 as an object_id, so we just add before using */
      id = Integer::from(state, ++state->om->last_object_id << TAG_REF_WIDTH);
      obj->set_ivar(state, G(sym_object_id), id);
    }
    return as<Integer>(id);
  }

  bool ObjectMemory::inflate_lock_count_overflow(STATE, ObjectHeader* obj,
                                                 int count)
  {
    SYNC(state);

    // Inflation always happens with the ObjectMemory lock held, so we don't
    // need to worry about another thread concurrently inflating it.
    //
    // But we do need to check that it's not already inflated.
    if(obj->inflated_header_p()) return false;

    InflatedHeader* ih = inflated_headers_->allocate(obj);
    ih->initialize_mutex(state->thread_id(), count);

    if(!obj->set_inflated_header(ih)) {
      if(obj->inflated_header_p()) return false;

      // Now things are really in a weird state, just abort.
      rubinius::bug("Massive header state confusion detected. Call a doctor.");
    }

    return true;
  }

  LockStatus ObjectMemory::contend_for_lock(STATE, ObjectHeader* obj,
                                            bool* error, size_t us)
  {
    bool timed = false;
    bool timeout = false;
    struct timespec ts = {0,0};

    {
      thread::Mutex::LockGuard lg(contention_lock_);

      // We want to lock obj, but someone else has it locked.
      //
      // If the lock is already inflated, no problem, just lock it!

      // Be sure obj is updated by the GC while we're waiting for it
      OnStack<1> os(state, obj);

step1:
      // Only contend if the header is thin locked.
      // Ok, the header is not inflated, but we can't inflate it and take
      // the lock because the locking thread needs to do that, so indicate
      // that the object is being contended for and then wait on the
      // contention condvar until the object is unlocked.

      HeaderWord orig = obj->header;
      HeaderWord new_val = orig;

      orig.f.meaning = eAuxWordLock;

      new_val.f.LockContended = 1;

      if(!obj->header.atomic_set(orig, new_val)) {
        if(new_val.f.meaning != eAuxWordLock) {
          if(cDebugThreading) {
            std::cerr << "[LOCK " << state->thread_id()
              << " contend_for_lock error: not thin locked.]\n";
          }
          *error = true;
          return eLockError;
        }

        // Something changed since we started to down this path,
        // start over.
        goto step1;
      }

      // Ok, we've registered the lock contention, now spin and wait
      // for the us to be told to retry.

      if(cDebugThreading) {
        std::cerr << "[LOCK " << state->thread_id() << " waiting on contention]\n";
      }

      if(us > 0) {
        timed = true;
        struct timeval tv;
        gettimeofday(&tv, NULL);

        ts.tv_sec = tv.tv_sec + (us / 1000000);
        ts.tv_nsec = (us % 1000000) * 1000;
      }

      state->set_sleeping();

      while(!obj->inflated_header_p()) {
        GCIndependent gc_guard(state);

        if(timed) {
          timeout = (contention_var_.wait_until(contention_lock_, &ts) == thread::cTimedOut);
          if(timeout) break;
        } else {
          contention_var_.wait(contention_lock_);
        }

        if(cDebugThreading) {
          std::cerr << "[LOCK " << state->thread_id() << " notified of contention breakage]\n";
        }

        // Someone is interrupting us trying to lock.
        if(state->check_local_interrupts) {
          state->check_local_interrupts = false;

          if(!state->interrupted_exception()->nil_p()) {
            if(cDebugThreading) {
              std::cerr << "[LOCK " << state->thread_id() << " detected interrupt]\n";
            }

            state->clear_sleeping();
            return eLockInterrupted;
          }
        }
      }

      state->clear_sleeping();

      if(cDebugThreading) {
        std::cerr << "[LOCK " << state->thread_id() << " contention broken]\n";
      }

      if(timeout) {
        if(cDebugThreading) {
          std::cerr << "[LOCK " << state->thread_id() << " contention timed out]\n";
        }

        return eLockTimeout;
      }
    } // contention_lock_ guard

    // We lock the InflatedHeader here rather than returning
    // and letting ObjectHeader::lock because the GC might have run
    // and we've used OnStack<> specificly to deal with that.
    //
    // ObjectHeader::lock doesn't use OnStack<>, it just is sure to
    // not access this if there is chance that a call blocked and GC'd
    // (which is true in the case of this function).

    InflatedHeader* ih = obj->inflated_header();

    if(timed) {
      return ih->lock_mutex_timed(state, &ts);
    } else {
      return ih->lock_mutex(state);
    }
  }

  void ObjectMemory::release_contention(STATE) {
    thread::Mutex::LockGuard lg(contention_lock_);
    contention_var_.broadcast();
  }

  bool ObjectMemory::inflate_and_lock(STATE, ObjectHeader* obj) {
    SYNC(state);

    InflatedHeader* ih = 0;
    int initial_count = 0;

    HeaderWord orig = obj->header;
    HeaderWord tmp = orig;

    switch(tmp.f.meaning) {
    case eAuxWordEmpty:
      // ERROR, we can not be here because it's empty. This is only to
      // be called when the header is already in use.
      return false;
    case eAuxWordInflated:
      // Already inflated. ERROR, let the caller sort it out.
      return false;
    case eAuxWordObjID:
      // We could be have made a header before trying again, so
      // keep using the original one.
      ih = inflated_headers_->allocate(obj);
      ih->set_object_id(tmp.f.aux_word);
      break;
    case eAuxWordLock:
      // We have to locking the object to inflate it, thats the law.
      if(tmp.f.aux_word >> cAuxLockTIDShift != state->thread_id()) {
        return false;
      }

      ih = inflated_headers_->allocate(obj);
      initial_count = orig.f.aux_word & cAuxLockRecCountMask;
    }

    ih->initialize_mutex(state->thread_id(), initial_count + 1);

    tmp.all_flags = ih;
    tmp.f.meaning = eAuxWordInflated;

    while(!obj->header.atomic_set(orig, tmp)) {
      // The header can't have been inflated by another thread, the
      // inflation process holds the OM lock.
      //
      // So some other bits must have changed, so lets just spin and
      // keep trying to update it.

      // Sanity check that the meaning is still the same, if not, then
      // something is really wrong.
      orig = obj->header;
      if(orig.f.meaning != tmp.f.meaning) {
        if(cDebugThreading) {
          std::cerr << "[LOCK object header consistence error detected.]\n";
        }
        return false;
      }

      tmp.all_flags = ih;
      tmp.f.meaning = eAuxWordInflated;
    }

    return true;
  }

  bool ObjectMemory::inflate_for_contention(STATE, ObjectHeader* obj) {
    SYNC(state);

    for(;;) {
      HeaderWord orig = obj->header;
      HeaderWord new_val = orig;

      InflatedHeader* ih = 0;

      switch(orig.f.meaning) {
      case eAuxWordEmpty:
        ih = inflated_headers_->allocate(obj);
        break;
      case eAuxWordObjID:
        // We could be have made a header before trying again, so
        // keep using the original one.
        ih = inflated_headers_->allocate(obj);
        ih->set_object_id(orig.f.aux_word);
        break;
      case eAuxWordLock:
        // We have to be locking the object to inflate it, thats the law.
        if(new_val.f.aux_word >> cAuxLockTIDShift != state->thread_id()) {
          if(cDebugThreading) {
            std::cerr << "[LOCK " << state->thread_id() << " object locked by another thread while inflating for contention]\n";
          }
          return false;
        }
        if(cDebugThreading) {
          std::cerr << "[LOCK " << state->thread_id() << " being unlocked and inflated atomicly]\n";
        }

        ih = inflated_headers_->allocate(obj);
        ih->flags().meaning = eAuxWordEmpty;
        ih->flags().aux_word = 0;
        break;
      case eAuxWordInflated:
        if(cDebugThreading) {
          std::cerr << "[LOCK " << state->thread_id() << " asked to inflated already inflated lock]\n";
        }
        return false;
      }

      ih->flags().LockContended = 0;

      new_val.all_flags = ih;
      new_val.f.meaning = eAuxWordInflated;

      // Try it all over again if it fails.
      if(!obj->header.atomic_set(orig, new_val)) continue;

      if(cDebugThreading) {
        std::cerr << "[LOCK " << state->thread_id() << " inflated lock for contention.]\n";
      }

      // Now inflated but not locked, which is what we want.
      return true;
    }
  }

  // WARNING: This returns an object who's body may not have been initialized.
  // It is the callers duty to initialize it.
  Object* ObjectMemory::new_object_fast(STATE, Class* cls, size_t bytes, object_type type) {
    SYNC(state);

    if(Object* obj = young_->raw_allocate(bytes, &collect_young_now)) {
      gc_stats.young_object_allocated(bytes);

      if(collect_young_now) root_state_->interrupts.set_perform_gc();
      obj->init_header(cls, YoungObjectZone, type);
      return obj;
    } else {
      UNSYNC;
      return new_object_typed(state, cls, bytes, type);
    }
  }

  bool ObjectMemory::refill_slab(STATE, gc::Slab& slab) {
    SYNC(state);

    Address addr = young_->allocate_for_slab(slab_size_);

    gc_stats.slab_allocated(slab.allocations(), slab.byte_used());

    if(addr) {
      slab.refill(addr, slab_size_);
      return true;
    } else {
      slab.refill(0, 0);
      return false;
    }
  }

  void ObjectMemory::set_young_lifetime(size_t age) {
    SYNC_TL;

    young_->set_lifetime(age);
  }

  void ObjectMemory::debug_marksweep(bool val) {
    SYNC_TL;

    if(val) {
      mark_sweep_->free_entries = false;
    } else {
      mark_sweep_->free_entries = true;
    }
  }

  bool ObjectMemory::valid_object_p(Object* obj) {
    if(obj->young_object_p()) {
      return young_->validate_object(obj) == cValid;
    } else if(obj->mature_object_p()) {
      return true;
    } else {
      return false;
    }
  }

  /* Garbage collection */

  Object* ObjectMemory::promote_object(Object* obj) {

    size_t sz = obj->size_in_bytes(root_state_);

    Object* copy = immix_->move_object(obj, sz);

    gc_stats.promoted_object_allocated(sz);
    if(unlikely(!copy)) {
      copy = mark_sweep_->move_object(obj, sz, &collect_mature_now);
    }

#ifdef ENABLE_OBJECT_WATCH
    if(watched_p(obj)) {
      std::cout << "detected object " << obj << " during promotion.\n";
    }
#endif

    return copy;
  }

  void ObjectMemory::collect(STATE, CallFrame* call_frame) {
    // Don't go any further unless we're allowed to GC.
    if(!can_gc()) return;

    SYNC(state);

    // If we were checkpointed, then someone else ran the GC, just return.
    if(state->shared.should_stop()) {
      UNSYNC;
      state->shared.checkpoint(state);
      return;
    }

    if(cDebugThreading) {
      std::cerr << "\n[ " << state << " WORLD beginning GC.]\n";
    }

    // Stops all other threads, so we're only here by ourselves.
    //
    // First, ask them to stop.
    state->shared.ask_for_stopage();

    // Now unlock ObjectMemory so that they can spin to any checkpoints.
    UNSYNC;

    // Wait for them all to check in.
    state->shared.stop_the_world(state);

    // Now we're alone, but we lock again just to safe.
    RESYNC;

    GCData gc_data(state);

    collect_young(gc_data);
    collect_mature(gc_data);

    // Ok, we're good. Get everyone going again.
    state->shared.restart_world(state);
    bool added = added_finalizers_;
    added_finalizers_ = false;

    UNSYNC;

    if(added) {
      if(finalizer_thread_.get() == Qnil) {
        start_finalizer_thread(state);
      }
      finalizer_var_.signal();
    }
  }

  void ObjectMemory::collect_maybe(STATE, CallFrame* call_frame) {
    // Don't go any further unless we're allowed to GC.
    if(!can_gc()) return;

    SYNC(state);

    // If we were checkpointed, then someone else ran the GC, just return.
    if(state->shared.should_stop()) {
      UNSYNC;
      state->shared.checkpoint(state);
      return;
    }

    if(cDebugThreading) {
      std::cerr << "\n[" << state << " WORLD beginning GC.]\n";
    }

    // Stops all other threads, so we're only here by ourselves.
    //
    // First, ask them to stop.
    state->shared.ask_for_stopage();

    // Now unlock ObjectMemory so that they can spin to any checkpoints.
    UNSYNC;

    // Wait for them all to check in.
    state->shared.stop_the_world(state);

    // Now we're alone, but we lock again just to safe.
    RESYNC;

    GCData gc_data(state);

    uint64_t start_time = 0;

    if(collect_young_now) {
      if(state->shared.config.gc_show) {
        start_time = get_current_time();
      }

      YoungCollectStats stats;

#ifdef RBX_PROFILER
      if(unlikely(state->tooling())) {
        tooling::GCEntry method(state, tooling::GCYoung);
        collect_young(gc_data, &stats);
      } else {
        collect_young(gc_data, &stats);
      }
#else
      collect_young(gc_data, &stats);
#endif

      if(state->shared.config.gc_show) {
        uint64_t fin_time = get_current_time();
        int diff = (fin_time - start_time) / 1000000;

        fprintf(stderr, "[GC %0.1f%% %d/%d %d %2dms]\n",
                  stats.percentage_used,
                  stats.promoted_objects,
                  stats.excess_objects,
                  stats.lifetime,
                  diff);
      }
    }

    if(collect_mature_now) {
      int before_kb = 0;

      if(state->shared.config.gc_show) {
        start_time = get_current_time();
        before_kb = mature_bytes_allocated() / 1024;
      }

#ifdef RBX_PROFILER
      if(unlikely(state->tooling())) {
        tooling::GCEntry method(state, tooling::GCMature);
        collect_mature(gc_data);
      } else {
        collect_mature(gc_data);
      }
#else
      collect_mature(gc_data);
#endif

      if(state->shared.config.gc_show) {
        uint64_t fin_time = get_current_time();
        int diff = (fin_time - start_time) / 1000000;
        int kb = mature_bytes_allocated() / 1024;
        fprintf(stderr, "[Full GC %dkB => %dkB %2dms]\n",
            before_kb,
            kb,
            diff);
      }

    }

    state->shared.restart_world(state);
    bool added = added_finalizers_;
    added_finalizers_ = false;

    UNSYNC;

    if(added) {
      if(finalizer_thread_.get() == Qnil) {
        start_finalizer_thread(state);
      }
      finalizer_var_.signal();
    }
  }

  void ObjectMemory::collect_young(GCData& data, YoungCollectStats* stats) {
    collect_young_now = false;

    timer::Running<1000000> timer(gc_stats.total_young_collection_time,
                                  gc_stats.last_young_collection_time);

    // validate_handles(data.handles());
    // validate_handles(data.cached_handles());

    young_->reset_stats();

    young_->collect(data, stats);

    prune_handles(data.handles(), true);
    prune_handles(data.cached_handles(), true);
    gc_stats.young_collection_count++;

    data.global_cache()->prune_young();

    if(data.threads()) {
      for(std::list<ManagedThread*>::iterator i = data.threads()->begin();
          i != data.threads()->end();
          ++i) {
        gc::Slab& slab = (*i)->local_slab();

        gc_stats.slab_allocated(slab.allocations(), slab.byte_used());

        void* addr = young_->allocate_for_slab(slab_size_);
        if(addr) {
          slab.refill(addr, slab_size_);
        } else {
          // We failed to allocate a slab, refill it to size 0.
          // It will fallback to ObjectMemory allocations later on.
          slab.refill(0, 0);
        }
      }
    }

  }

  void ObjectMemory::collect_mature(GCData& data) {

    // validate_handles(data.handles());
    // validate_handles(data.cached_handles());

    timer::Running<1000000> timer(gc_stats.total_full_collection_time,
                                  gc_stats.last_full_collection_time);

    collect_mature_now = false;

    code_manager_.clear_marks();

    immix_->reset_stats();

    immix_->collect(data);

    immix_->clean_weakrefs();

    code_manager_.sweep();<commit_msg>Lazy allocate thread slabs to prevent starvation<commit_after>#include <stdlib.h>
#include <iostream>
#include <sys/time.h>

#include "config.h"
#include "vm.hpp"
#include "objectmemory.hpp"
#include "gc/marksweep.hpp"
#include "gc/baker.hpp"
#include "gc/immix.hpp"
#include "gc/inflated_headers.hpp"
#include "gc/walker.hpp"
#include "on_stack.hpp"

#include "config_parser.hpp"

#include "builtin/class.hpp"
#include "builtin/fixnum.hpp"
#include "builtin/tuple.hpp"
#include "builtin/io.hpp"
#include "builtin/fiber.hpp"
#include "builtin/string.hpp"
#include "builtin/lookuptable.hpp"
#include "builtin/ffi_pointer.hpp"
#include "builtin/data.hpp"
#include "builtin/dir.hpp"
#include "builtin/array.hpp"
#include "builtin/thread.hpp"

#include "capi/handle.hpp"
#include "configuration.hpp"

#include "global_cache.hpp"

#include "instruments/tooling.hpp"

// Used by XMALLOC at the bottom
static long gc_malloc_threshold = 0;
static long bytes_until_collection = 0;

namespace rubinius {

  Object* object_watch = 0;

  /* ObjectMemory methods */
  ObjectMemory::ObjectMemory(STATE, Configuration& config)
    : young_(new BakerGC(this, config.gc_bytes))
    , mark_sweep_(new MarkSweepGC(this, config))
    , immix_(new ImmixGC(this))
    , inflated_headers_(new InflatedHeaders)
    , mark_(1)
    , code_manager_(&state->shared)
    , allow_gc_(true)
    , slab_size_(4096)
    , running_finalizers_(false)
    , added_finalizers_(false)
    , finalizer_thread_(state, nil<Thread>())

    , collect_young_now(false)
    , collect_mature_now(false)
    , root_state_(state)
  {
    // TODO Not sure where this code should be...
    if(char* num = getenv("RBX_WATCH")) {
      object_watch = (Object*)strtol(num, NULL, 10);
      std::cout << "Watching for " << object_watch << "\n";
    }

    collect_mature_now = false;
    last_object_id = 0;

    large_object_threshold = config.gc_large_object;
    young_->set_lifetime(config.gc_lifetime);

    if(config.gc_autotune) young_->set_autotune();

    for(size_t i = 0; i < LastObjectType; i++) {
      type_info[i] = NULL;
    }

    TypeInfo::init(this);

    gc_malloc_threshold = config.gc_malloc_threshold;
    bytes_until_collection = gc_malloc_threshold;
  }

  ObjectMemory::~ObjectMemory() {
    mark_sweep_->free_objects();

    // @todo free immix data

    for(size_t i = 0; i < LastObjectType; i++) {
      if(type_info[i]) delete type_info[i];
    }

    delete immix_;
    delete mark_sweep_;
    delete young_;

    // Must be last
    // delete inflated_headers_;

  }

  void ObjectMemory::on_fork() {
    finalizer_lock_.init();
    finalizer_var_.init();
    finalizer_thread_.set(nil<Thread>());
  }

  void ObjectMemory::assign_object_id(STATE, Object* obj) {
    SYNC(state);

    // Double check we've got no id still after the lock.
    if(obj->object_id() > 0) return;

    obj->set_object_id(state, state->om, ++last_object_id);
  }

  Integer* ObjectMemory::assign_object_id_ivar(STATE, Object* obj) {
    SYNC(state);
    Object* id = obj->get_ivar(state, G(sym_object_id));
    if(id->nil_p()) {
      /* All references have an even object_id. last_object_id starts out at 0
       * but we don't want to use 0 as an object_id, so we just add before using */
      id = Integer::from(state, ++state->om->last_object_id << TAG_REF_WIDTH);
      obj->set_ivar(state, G(sym_object_id), id);
    }
    return as<Integer>(id);
  }

  bool ObjectMemory::inflate_lock_count_overflow(STATE, ObjectHeader* obj,
                                                 int count)
  {
    SYNC(state);

    // Inflation always happens with the ObjectMemory lock held, so we don't
    // need to worry about another thread concurrently inflating it.
    //
    // But we do need to check that it's not already inflated.
    if(obj->inflated_header_p()) return false;

    InflatedHeader* ih = inflated_headers_->allocate(obj);
    ih->initialize_mutex(state->thread_id(), count);

    if(!obj->set_inflated_header(ih)) {
      if(obj->inflated_header_p()) return false;

      // Now things are really in a weird state, just abort.
      rubinius::bug("Massive header state confusion detected. Call a doctor.");
    }

    return true;
  }

  LockStatus ObjectMemory::contend_for_lock(STATE, ObjectHeader* obj,
                                            bool* error, size_t us)
  {
    bool timed = false;
    bool timeout = false;
    struct timespec ts = {0,0};

    {
      thread::Mutex::LockGuard lg(contention_lock_);

      // We want to lock obj, but someone else has it locked.
      //
      // If the lock is already inflated, no problem, just lock it!

      // Be sure obj is updated by the GC while we're waiting for it
      OnStack<1> os(state, obj);

step1:
      // Only contend if the header is thin locked.
      // Ok, the header is not inflated, but we can't inflate it and take
      // the lock because the locking thread needs to do that, so indicate
      // that the object is being contended for and then wait on the
      // contention condvar until the object is unlocked.

      HeaderWord orig = obj->header;
      HeaderWord new_val = orig;

      orig.f.meaning = eAuxWordLock;

      new_val.f.LockContended = 1;

      if(!obj->header.atomic_set(orig, new_val)) {
        if(new_val.f.meaning != eAuxWordLock) {
          if(cDebugThreading) {
            std::cerr << "[LOCK " << state->thread_id()
              << " contend_for_lock error: not thin locked.]\n";
          }
          *error = true;
          return eLockError;
        }

        // Something changed since we started to down this path,
        // start over.
        goto step1;
      }

      // Ok, we've registered the lock contention, now spin and wait
      // for the us to be told to retry.

      if(cDebugThreading) {
        std::cerr << "[LOCK " << state->thread_id() << " waiting on contention]\n";
      }

      if(us > 0) {
        timed = true;
        struct timeval tv;
        gettimeofday(&tv, NULL);

        ts.tv_sec = tv.tv_sec + (us / 1000000);
        ts.tv_nsec = (us % 1000000) * 1000;
      }

      state->set_sleeping();

      while(!obj->inflated_header_p()) {
        GCIndependent gc_guard(state);

        if(timed) {
          timeout = (contention_var_.wait_until(contention_lock_, &ts) == thread::cTimedOut);
          if(timeout) break;
        } else {
          contention_var_.wait(contention_lock_);
        }

        if(cDebugThreading) {
          std::cerr << "[LOCK " << state->thread_id() << " notified of contention breakage]\n";
        }

        // Someone is interrupting us trying to lock.
        if(state->check_local_interrupts) {
          state->check_local_interrupts = false;

          if(!state->interrupted_exception()->nil_p()) {
            if(cDebugThreading) {
              std::cerr << "[LOCK " << state->thread_id() << " detected interrupt]\n";
            }

            state->clear_sleeping();
            return eLockInterrupted;
          }
        }
      }

      state->clear_sleeping();

      if(cDebugThreading) {
        std::cerr << "[LOCK " << state->thread_id() << " contention broken]\n";
      }

      if(timeout) {
        if(cDebugThreading) {
          std::cerr << "[LOCK " << state->thread_id() << " contention timed out]\n";
        }

        return eLockTimeout;
      }
    } // contention_lock_ guard

    // We lock the InflatedHeader here rather than returning
    // and letting ObjectHeader::lock because the GC might have run
    // and we've used OnStack<> specificly to deal with that.
    //
    // ObjectHeader::lock doesn't use OnStack<>, it just is sure to
    // not access this if there is chance that a call blocked and GC'd
    // (which is true in the case of this function).

    InflatedHeader* ih = obj->inflated_header();

    if(timed) {
      return ih->lock_mutex_timed(state, &ts);
    } else {
      return ih->lock_mutex(state);
    }
  }

  void ObjectMemory::release_contention(STATE) {
    thread::Mutex::LockGuard lg(contention_lock_);
    contention_var_.broadcast();
  }

  bool ObjectMemory::inflate_and_lock(STATE, ObjectHeader* obj) {
    SYNC(state);

    InflatedHeader* ih = 0;
    int initial_count = 0;

    HeaderWord orig = obj->header;
    HeaderWord tmp = orig;

    switch(tmp.f.meaning) {
    case eAuxWordEmpty:
      // ERROR, we can not be here because it's empty. This is only to
      // be called when the header is already in use.
      return false;
    case eAuxWordInflated:
      // Already inflated. ERROR, let the caller sort it out.
      return false;
    case eAuxWordObjID:
      // We could be have made a header before trying again, so
      // keep using the original one.
      ih = inflated_headers_->allocate(obj);
      ih->set_object_id(tmp.f.aux_word);
      break;
    case eAuxWordLock:
      // We have to locking the object to inflate it, thats the law.
      if(tmp.f.aux_word >> cAuxLockTIDShift != state->thread_id()) {
        return false;
      }

      ih = inflated_headers_->allocate(obj);
      initial_count = orig.f.aux_word & cAuxLockRecCountMask;
    }

    ih->initialize_mutex(state->thread_id(), initial_count + 1);

    tmp.all_flags = ih;
    tmp.f.meaning = eAuxWordInflated;

    while(!obj->header.atomic_set(orig, tmp)) {
      // The header can't have been inflated by another thread, the
      // inflation process holds the OM lock.
      //
      // So some other bits must have changed, so lets just spin and
      // keep trying to update it.

      // Sanity check that the meaning is still the same, if not, then
      // something is really wrong.
      orig = obj->header;
      if(orig.f.meaning != tmp.f.meaning) {
        if(cDebugThreading) {
          std::cerr << "[LOCK object header consistence error detected.]\n";
        }
        return false;
      }

      tmp.all_flags = ih;
      tmp.f.meaning = eAuxWordInflated;
    }

    return true;
  }

  bool ObjectMemory::inflate_for_contention(STATE, ObjectHeader* obj) {
    SYNC(state);

    for(;;) {
      HeaderWord orig = obj->header;
      HeaderWord new_val = orig;

      InflatedHeader* ih = 0;

      switch(orig.f.meaning) {
      case eAuxWordEmpty:
        ih = inflated_headers_->allocate(obj);
        break;
      case eAuxWordObjID:
        // We could be have made a header before trying again, so
        // keep using the original one.
        ih = inflated_headers_->allocate(obj);
        ih->set_object_id(orig.f.aux_word);
        break;
      case eAuxWordLock:
        // We have to be locking the object to inflate it, thats the law.
        if(new_val.f.aux_word >> cAuxLockTIDShift != state->thread_id()) {
          if(cDebugThreading) {
            std::cerr << "[LOCK " << state->thread_id() << " object locked by another thread while inflating for contention]\n";
          }
          return false;
        }
        if(cDebugThreading) {
          std::cerr << "[LOCK " << state->thread_id() << " being unlocked and inflated atomicly]\n";
        }

        ih = inflated_headers_->allocate(obj);
        ih->flags().meaning = eAuxWordEmpty;
        ih->flags().aux_word = 0;
        break;
      case eAuxWordInflated:
        if(cDebugThreading) {
          std::cerr << "[LOCK " << state->thread_id() << " asked to inflated already inflated lock]\n";
        }
        return false;
      }

      ih->flags().LockContended = 0;

      new_val.all_flags = ih;
      new_val.f.meaning = eAuxWordInflated;

      // Try it all over again if it fails.
      if(!obj->header.atomic_set(orig, new_val)) continue;

      if(cDebugThreading) {
        std::cerr << "[LOCK " << state->thread_id() << " inflated lock for contention.]\n";
      }

      // Now inflated but not locked, which is what we want.
      return true;
    }
  }

  // WARNING: This returns an object who's body may not have been initialized.
  // It is the callers duty to initialize it.
  Object* ObjectMemory::new_object_fast(STATE, Class* cls, size_t bytes, object_type type) {
    SYNC(state);

    if(Object* obj = young_->raw_allocate(bytes, &collect_young_now)) {
      gc_stats.young_object_allocated(bytes);

      if(collect_young_now) root_state_->interrupts.set_perform_gc();
      obj->init_header(cls, YoungObjectZone, type);
      return obj;
    } else {
      UNSYNC;
      return new_object_typed(state, cls, bytes, type);
    }
  }

  bool ObjectMemory::refill_slab(STATE, gc::Slab& slab) {
    SYNC(state);

    Address addr = young_->allocate_for_slab(slab_size_);

    gc_stats.slab_allocated(slab.allocations(), slab.byte_used());

    if(addr) {
      slab.refill(addr, slab_size_);
      return true;
    } else {
      slab.refill(0, 0);
      return false;
    }
  }

  void ObjectMemory::set_young_lifetime(size_t age) {
    SYNC_TL;

    young_->set_lifetime(age);
  }

  void ObjectMemory::debug_marksweep(bool val) {
    SYNC_TL;

    if(val) {
      mark_sweep_->free_entries = false;
    } else {
      mark_sweep_->free_entries = true;
    }
  }

  bool ObjectMemory::valid_object_p(Object* obj) {
    if(obj->young_object_p()) {
      return young_->validate_object(obj) == cValid;
    } else if(obj->mature_object_p()) {
      return true;
    } else {
      return false;
    }
  }

  /* Garbage collection */

  Object* ObjectMemory::promote_object(Object* obj) {

    size_t sz = obj->size_in_bytes(root_state_);

    Object* copy = immix_->move_object(obj, sz);

    gc_stats.promoted_object_allocated(sz);
    if(unlikely(!copy)) {
      copy = mark_sweep_->move_object(obj, sz, &collect_mature_now);
    }

#ifdef ENABLE_OBJECT_WATCH
    if(watched_p(obj)) {
      std::cout << "detected object " << obj << " during promotion.\n";
    }
#endif

    return copy;
  }

  void ObjectMemory::collect(STATE, CallFrame* call_frame) {
    // Don't go any further unless we're allowed to GC.
    if(!can_gc()) return;

    SYNC(state);

    // If we were checkpointed, then someone else ran the GC, just return.
    if(state->shared.should_stop()) {
      UNSYNC;
      state->shared.checkpoint(state);
      return;
    }

    if(cDebugThreading) {
      std::cerr << "\n[ " << state << " WORLD beginning GC.]\n";
    }

    // Stops all other threads, so we're only here by ourselves.
    //
    // First, ask them to stop.
    state->shared.ask_for_stopage();

    // Now unlock ObjectMemory so that they can spin to any checkpoints.
    UNSYNC;

    // Wait for them all to check in.
    state->shared.stop_the_world(state);

    // Now we're alone, but we lock again just to safe.
    RESYNC;

    GCData gc_data(state);

    collect_young(gc_data);
    collect_mature(gc_data);

    // Ok, we're good. Get everyone going again.
    state->shared.restart_world(state);
    bool added = added_finalizers_;
    added_finalizers_ = false;

    UNSYNC;

    if(added) {
      if(finalizer_thread_.get() == Qnil) {
        start_finalizer_thread(state);
      }
      finalizer_var_.signal();
    }
  }

  void ObjectMemory::collect_maybe(STATE, CallFrame* call_frame) {
    // Don't go any further unless we're allowed to GC.
    if(!can_gc()) return;

    SYNC(state);

    // If we were checkpointed, then someone else ran the GC, just return.
    if(state->shared.should_stop()) {
      UNSYNC;
      state->shared.checkpoint(state);
      return;
    }

    if(cDebugThreading) {
      std::cerr << "\n[" << state << " WORLD beginning GC.]\n";
    }

    // Stops all other threads, so we're only here by ourselves.
    //
    // First, ask them to stop.
    state->shared.ask_for_stopage();

    // Now unlock ObjectMemory so that they can spin to any checkpoints.
    UNSYNC;

    // Wait for them all to check in.
    state->shared.stop_the_world(state);

    // Now we're alone, but we lock again just to safe.
    RESYNC;

    GCData gc_data(state);

    uint64_t start_time = 0;

    if(collect_young_now) {
      if(state->shared.config.gc_show) {
        start_time = get_current_time();
      }

      YoungCollectStats stats;

#ifdef RBX_PROFILER
      if(unlikely(state->tooling())) {
        tooling::GCEntry method(state, tooling::GCYoung);
        collect_young(gc_data, &stats);
      } else {
        collect_young(gc_data, &stats);
      }
#else
      collect_young(gc_data, &stats);
#endif

      if(state->shared.config.gc_show) {
        uint64_t fin_time = get_current_time();
        int diff = (fin_time - start_time) / 1000000;

        fprintf(stderr, "[GC %0.1f%% %d/%d %d %2dms]\n",
                  stats.percentage_used,
                  stats.promoted_objects,
                  stats.excess_objects,
                  stats.lifetime,
                  diff);
      }
    }

    if(collect_mature_now) {
      int before_kb = 0;

      if(state->shared.config.gc_show) {
        start_time = get_current_time();
        before_kb = mature_bytes_allocated() / 1024;
      }

#ifdef RBX_PROFILER
      if(unlikely(state->tooling())) {
        tooling::GCEntry method(state, tooling::GCMature);
        collect_mature(gc_data);
      } else {
        collect_mature(gc_data);
      }
#else
      collect_mature(gc_data);
#endif

      if(state->shared.config.gc_show) {
        uint64_t fin_time = get_current_time();
        int diff = (fin_time - start_time) / 1000000;
        int kb = mature_bytes_allocated() / 1024;
        fprintf(stderr, "[Full GC %dkB => %dkB %2dms]\n",
            before_kb,
            kb,
            diff);
      }

    }

    state->shared.restart_world(state);
    bool added = added_finalizers_;
    added_finalizers_ = false;

    UNSYNC;

    if(added) {
      if(finalizer_thread_.get() == Qnil) {
        start_finalizer_thread(state);
      }
      finalizer_var_.signal();
    }
  }

  void ObjectMemory::collect_young(GCData& data, YoungCollectStats* stats) {
    collect_young_now = false;

    timer::Running<1000000> timer(gc_stats.total_young_collection_time,
                                  gc_stats.last_young_collection_time);

    // validate_handles(data.handles());
    // validate_handles(data.cached_handles());

    young_->reset_stats();

    young_->collect(data, stats);

    prune_handles(data.handles(), true);
    prune_handles(data.cached_handles(), true);
    gc_stats.young_collection_count++;

    data.global_cache()->prune_young();

    if(data.threads()) {
      for(std::list<ManagedThread*>::iterator i = data.threads()->begin();
          i != data.threads()->end();
          ++i) {
        gc::Slab& slab = (*i)->local_slab();

        gc_stats.slab_allocated(slab.allocations(), slab.byte_used());

        // Reset the slab to a size of 0 so that the thread has to do
        // an allocation to get a proper refill. This keeps the number
        // of threads in the system from starving the available
        // number of slabs.
        slab.refill(0, 0);
      }
    }

  }

  void ObjectMemory::collect_mature(GCData& data) {

    // validate_handles(data.handles());
    // validate_handles(data.cached_handles());

    timer::Running<1000000> timer(gc_stats.total_full_collection_time,
                                  gc_stats.last_full_collection_time);

    collect_mature_now = false;

    code_manager_.clear_marks();

    immix_->reset_stats();

    immix_->collect(data);

    immix_->clean_weakrefs();

    code_manager_.sweep();
{'fraction_non_alphanumeric': 0.1219083555165017, 'fraction_numerical': 0.005168893598325278, 'mean_word_length': 2.5074329224075416, 'pattern_counts': {'":': 0, '<': 207, '<?xml version=': 0, '>': 238, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6232317', 'n_tokens_mistral': 12877, 'n_tokens_neox': 12207, 'n_words': 2941}