<commit_before>/*
    Title:      Multi-Threaded Garbage Collector

    Copyright (c) 2010-12 <NAME>

    Based on the original garbage collector code
        Copyright 2000-2008
        Cambridge University Technical Services Limited

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#elif defined(_WIN32)
#include "winconfig.h"
#else
#error "No configuration file"
#endif

#ifdef HAVE_ASSERT_H
#include <assert.h>
#define ASSERT(x)   assert(x)
#else
#define ASSERT(x)
#endif

#include "globals.h"
#include "run_time.h"
#include "machine_dep.h"
#include "diagnostics.h"
#include "processes.h"
#include "timing.h"
#include "gc.h"
#include "scanaddrs.h"
#include "check_objects.h"
#include "osmem.h"
#include "bitmap.h"
#include "rts_module.h"
#include "memmgr.h"
#include "gctaskfarm.h"
#include "mpoly.h"
#include "statistics.h"
#include "profiling.h"
#include "heapsizing.h"

static GCTaskFarm gTaskFarm; // Global task farm.
GCTaskFarm *gpTaskFarm = &gTaskFarm;

// If the GC converts a weak ref from SOME to NONE it sets this ref.  It can be
// cleared by the signal handler thread.  There's no need for a lock since it
// is only set during GC and only cleared when not GCing.
bool convertedWeak = false;

/*
    How the garbage collector works.
    The GC has two phases.  The minor (quick) GC is a copying collector that
    copies data from the allocation area into the mutable and immutable area.
    The major collector is started when either the mutable or the immutable
    area is full.  The major collector uses a mark/sweep scheme.
    The GC has three phases:

    1.  Mark phase.
    Working from the roots; which are the the permanent mutable segments and
    the RTS roots (e.g. thread stacks), mark all reachable cells.
    Marking involves setting bits in the bitmap for reachable words.

    2. Compact phase.
    Marked objects are copied to try to compact, upwards, the heap segments.  When
    an object is moved the length word of the object in the old location is set as
    a tombstone that points to its new location.  In particular this means that we
    cannot reuse the space where an object previously was during the compaction phase.
    Immutable objects are moved into immutable segments.  When an object is moved
    to a new location the bits are set in the bitmap as though the object had been
    marked at that location.

    3. Update phase.
    The roots and objects marked during the first two phases are scanned and any
    addresses for moved objects are updated.  The lowest address used in the area
    then becomes the base of the area for future allocations.

    There is a sharing phase which may be performed before the mark phase.  This
    merges immutable cells with the same contents with the aim of reducing the
    size of the live data.  It is expensive so is not performed by default.

    Updated DCJM 12/06/12

*/
static bool doGC(const POLYUNSIGNED wordsRequiredToAllocate)
{
    gHeapSizeParameters.RecordAtStartOfMajorGC();
    gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeStart);
    globalStats.incCount(PSC_GC_FULLGC);

    // Remove any empty spaces.  There will not normally be any except
    // if we have triggered a full GC as a result of detecting paging in the
    // minor GC but in that case we want to try to stop the system writing
    // out areas that are now empty.
    gMem.RemoveEmptyLocals();

    if (debugOptions & DEBUG_GC)
        Log("GC: Full GC, %lu words required %" PRI_SIZET " spaces\n", wordsRequiredToAllocate, gMem.lSpaces.size());

    if (debugOptions & DEBUG_HEAPSIZE)
        gMem.ReportHeapSizes("Full GC (before)");

    // Data sharing pass.
    if (true || gHeapSizeParameters.PerformSharingPass())
        GCSharingPhase();
/*
 * There is a really weird bug somewhere.  An extra bit may be set in the bitmap during
 * the mark phase.  It seems to be related to heavy swapping activity.  Duplicating the
 * bitmap causes it to occur only in one copy and write-protecting the bitmap apart from
 * when it is actually being updated does not result in a seg-fault.  So far I've only
 * seen it on 64-bit Linux but it may be responsible for other crashes.  The work-around
 * is to check the number of bits set in the bitmap and repeat the mark phase if it does
 * not match.
 */
    
    for (unsigned p = 3; p > 0; p--)
    {
        for(std::vector<LocalMemSpace*>::iterator i = gMem.lSpaces.begin(); i < gMem.lSpaces.end(); i++)
        {
            LocalMemSpace *lSpace = *i;
            ASSERT (lSpace->top >= lSpace->upperAllocPtr);
            ASSERT (lSpace->upperAllocPtr >= lSpace->lowerAllocPtr);
            ASSERT (lSpace->lowerAllocPtr >= lSpace->bottom);
            // Set upper and lower limits of weak refs.
            lSpace->highestWeak = lSpace->bottom;
            lSpace->lowestWeak = lSpace->top;
            lSpace->fullGCLowerLimit = lSpace->top;
            // Put dummy objects in the unused space.  This allows
            // us to scan over the whole of the space.
            gMem.FillUnusedSpace(lSpace->lowerAllocPtr,
                lSpace->upperAllocPtr-lSpace->lowerAllocPtr);<commit_msg>Remove debugging code that enabled sharing pass on every major GC.<commit_after>/*
    Title:      Multi-Threaded Garbage Collector

    Copyright (c) 2010-12 <NAME>

    Based on the original garbage collector code
        Copyright 2000-2008
        Cambridge University Technical Services Limited

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#elif defined(_WIN32)
#include "winconfig.h"
#else
#error "No configuration file"
#endif

#ifdef HAVE_ASSERT_H
#include <assert.h>
#define ASSERT(x)   assert(x)
#else
#define ASSERT(x)
#endif

#include "globals.h"
#include "run_time.h"
#include "machine_dep.h"
#include "diagnostics.h"
#include "processes.h"
#include "timing.h"
#include "gc.h"
#include "scanaddrs.h"
#include "check_objects.h"
#include "osmem.h"
#include "bitmap.h"
#include "rts_module.h"
#include "memmgr.h"
#include "gctaskfarm.h"
#include "mpoly.h"
#include "statistics.h"
#include "profiling.h"
#include "heapsizing.h"

static GCTaskFarm gTaskFarm; // Global task farm.
GCTaskFarm *gpTaskFarm = &gTaskFarm;

// If the GC converts a weak ref from SOME to NONE it sets this ref.  It can be
// cleared by the signal handler thread.  There's no need for a lock since it
// is only set during GC and only cleared when not GCing.
bool convertedWeak = false;

/*
    How the garbage collector works.
    The GC has two phases.  The minor (quick) GC is a copying collector that
    copies data from the allocation area into the mutable and immutable area.
    The major collector is started when either the mutable or the immutable
    area is full.  The major collector uses a mark/sweep scheme.
    The GC has three phases:

    1.  Mark phase.
    Working from the roots; which are the the permanent mutable segments and
    the RTS roots (e.g. thread stacks), mark all reachable cells.
    Marking involves setting bits in the bitmap for reachable words.

    2. Compact phase.
    Marked objects are copied to try to compact, upwards, the heap segments.  When
    an object is moved the length word of the object in the old location is set as
    a tombstone that points to its new location.  In particular this means that we
    cannot reuse the space where an object previously was during the compaction phase.
    Immutable objects are moved into immutable segments.  When an object is moved
    to a new location the bits are set in the bitmap as though the object had been
    marked at that location.

    3. Update phase.
    The roots and objects marked during the first two phases are scanned and any
    addresses for moved objects are updated.  The lowest address used in the area
    then becomes the base of the area for future allocations.

    There is a sharing phase which may be performed before the mark phase.  This
    merges immutable cells with the same contents with the aim of reducing the
    size of the live data.  It is expensive so is not performed by default.

    Updated DCJM 12/06/12

*/
static bool doGC(const POLYUNSIGNED wordsRequiredToAllocate)
{
    gHeapSizeParameters.RecordAtStartOfMajorGC();
    gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeStart);
    globalStats.incCount(PSC_GC_FULLGC);

    // Remove any empty spaces.  There will not normally be any except
    // if we have triggered a full GC as a result of detecting paging in the
    // minor GC but in that case we want to try to stop the system writing
    // out areas that are now empty.
    gMem.RemoveEmptyLocals();

    if (debugOptions & DEBUG_GC)
        Log("GC: Full GC, %lu words required %" PRI_SIZET " spaces\n", wordsRequiredToAllocate, gMem.lSpaces.size());

    if (debugOptions & DEBUG_HEAPSIZE)
        gMem.ReportHeapSizes("Full GC (before)");

    // Data sharing pass.
    if (gHeapSizeParameters.PerformSharingPass())
        GCSharingPhase();
/*
 * There is a really weird bug somewhere.  An extra bit may be set in the bitmap during
 * the mark phase.  It seems to be related to heavy swapping activity.  Duplicating the
 * bitmap causes it to occur only in one copy and write-protecting the bitmap apart from
 * when it is actually being updated does not result in a seg-fault.  So far I've only
 * seen it on 64-bit Linux but it may be responsible for other crashes.  The work-around
 * is to check the number of bits set in the bitmap and repeat the mark phase if it does
 * not match.
 */
    
    for (unsigned p = 3; p > 0; p--)
    {
        for(std::vector<LocalMemSpace*>::iterator i = gMem.lSpaces.begin(); i < gMem.lSpaces.end(); i++)
        {
            LocalMemSpace *lSpace = *i;
            ASSERT (lSpace->top >= lSpace->upperAllocPtr);
            ASSERT (lSpace->upperAllocPtr >= lSpace->lowerAllocPtr);
            ASSERT (lSpace->lowerAllocPtr >= lSpace->bottom);
            // Set upper and lower limits of weak refs.
            lSpace->highestWeak = lSpace->bottom;
            lSpace->lowestWeak = lSpace->top;
            lSpace->fullGCLowerLimit = lSpace->top;
            // Put dummy objects in the unused space.  This allows
            // us to scan over the whole of the space.
            gMem.FillUnusedSpace(lSpace->lowerAllocPtr,
                lSpace->upperAllocPtr-lSpace->lowerAllocPtr);
{'fraction_non_alphanumeric': 0.0679644519678375, 'fraction_numerical': 0.007109606432501058, 'mean_word_length': 3.114206128133705, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1628084', 'n_tokens_mistral': 3386, 'n_tokens_neox': 3260, 'n_words': 1538}