<commit_before>/********************************************************************************
*                                                                               *
*                              Custom Memory Pool                               *
*                                                                               *
*********************************************************************************
*        Copyright (C) 2003 by <NAME>.   All Rights Reserved.            *
*       NOTE THAT I DO NOT PERMIT ANY OF MY CODE TO BE PROMOTED TO THE GPL      *
*********************************************************************************
* This code is free software; you can redistribute it and/or modify it under    *
* the terms of the GNU Library General Public License v2.1 as published by the  *
* Free Software Foundation EXCEPT that clause 3 does not apply ie; you may not  *
* "upgrade" this code to the GPL without my prior written permission.           *
* Please consult the file "License_Addendum2.txt" accompanying this file.       *
*                                                                               *
* This code is distributed in the hope that it will be useful,                  *
* but WITHOUT ANY WARRANTY; without even the implied warranty of                *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                          *
*********************************************************************************
* $Id:                                                                          *
********************************************************************************/

//#define DEBUG
//#define FXDISABLE_GLOBAL_MARKER 0
//#define FXDISABLE_SEPARATE_POOLS 1

#ifdef USE_POSIX
// Force a "real" copy of global operators new & delete to satisfy GNU
// linker that we've really redefined them. It would seem that when they're
// inline the linker outputs them as weak which is no good for other modules
#define inline
#endif
#include "fxdefs.h"
#ifdef USE_POSIX
#undef inline
#endif

#include "FXProcess.h"
#include "FXThread.h"
#include "FXRollback.h"
#include "FXPtrHold.h"
#include <stdlib.h>
#include <assert.h>
#include <qptrdict.h>
#include <qmemarray.h>
#ifdef _MSC_VER
#include <crtdbg.h>
#endif

//#pragma optimize("gaw", off)
//#pragma inline_depth(0)

/* Define to force exclusive use of the system allocator by the global allocator
replacements (directly using FXMemoryPool still uses the special heap). If you
interchange direct use of FXMemoryPool with parameterised use of the global
allocator functions, you will get a segfault. Note that this comes with a twelve
byte book-keeping overhead on every allocation and is intended only for debugging
purposes (especially memory leak checking) */
#ifndef FXDISABLE_SEPARATE_POOLS
// Define to force use of system allocator
 #ifdef DEBUG
  #define FXDISABLE_SEPARATE_POOLS 1
 #else
  #define FXDISABLE_SEPARATE_POOLS 0
 #endif
#endif

/* Customise global allocator replacements according to system. On Win32,
replace crappy Win32 allocator with our own */
#ifdef _MSC_VER
 #define MYMALLOC(size)       ptmalloc2::dlmalloc(size)
 #define MYREALLOC(ptr, size) ptmalloc2::dlrealloc(ptr, size)
 #define MYFREE(ptr)          ptmalloc2::dlfree(ptr)
#else
 #define MYMALLOC(size)       ::malloc(size)
 #define MYREALLOC(ptr, size) ::realloc(ptr, size)
 #define MYFREE(ptr)          ::free(ptr)
#endif

/* Defining this saves the four or eight byte overhead used to mark a block
as having been allocated by us on every allocation in the global heap. While
this saves memory, it does mean that the allocator replacements can no longer
detect when they are freeing memory allocated by the system allocator and thus
invoke that instead.

By default, this is enabled on both Win32 and POSIX but for different reasons:-
on Win32, the global allocator is totally replaced and this works fine due to
Win32's much better DLL encapsulation. On POSIX, the global allocator is not
replaced and thus the many mismatched allocator function calls don't matter.

Note that allocations from a custom pool \b always contain the overhead (this
is how free() knows when not to pass through to ::free()) */
#ifndef FXDISABLE_GLOBAL_MARKER
 #define FXDISABLE_GLOBAL_MARKER 1
#endif

/* Defining this enables the failonfree() function whereby freeing a specified<commit_msg>Now replaces system allocator on all platforms except for Linux<commit_after>/********************************************************************************
*                                                                               *
*                              Custom Memory Pool                               *
*                                                                               *
*********************************************************************************
*        Copyright (C) 2003 by <NAME>.   All Rights Reserved.            *
*       NOTE THAT I DO NOT PERMIT ANY OF MY CODE TO BE PROMOTED TO THE GPL      *
*********************************************************************************
* This code is free software; you can redistribute it and/or modify it under    *
* the terms of the GNU Library General Public License v2.1 as published by the  *
* Free Software Foundation EXCEPT that clause 3 does not apply ie; you may not  *
* "upgrade" this code to the GPL without my prior written permission.           *
* Please consult the file "License_Addendum2.txt" accompanying this file.       *
*                                                                               *
* This code is distributed in the hope that it will be useful,                  *
* but WITHOUT ANY WARRANTY; without even the implied warranty of                *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                          *
*********************************************************************************
* $Id:                                                                          *
********************************************************************************/

//#define DEBUG
//#define FXDISABLE_GLOBAL_MARKER 0
//#define FXDISABLE_SEPARATE_POOLS 1

#ifdef USE_POSIX
// Force a "real" copy of global operators new & delete to satisfy GNU
// linker that we've really redefined them. It would seem that when they're
// inline the linker outputs them as weak which is no good for other modules
#define inline
#endif
#include "fxdefs.h"
#ifdef USE_POSIX
#undef inline
#endif

#include "FXProcess.h"
#include "FXThread.h"
#include "FXRollback.h"
#include "FXPtrHold.h"
#include <stdlib.h>
#include <assert.h>
#include <qptrdict.h>
#include <qmemarray.h>
#ifdef _MSC_VER
#include <crtdbg.h>
#endif

//#pragma optimize("gaw", off)
//#pragma inline_depth(0)

/* Define to force exclusive use of the system allocator by the global allocator
replacements (directly using FXMemoryPool still uses the special heap). If you
interchange direct use of FXMemoryPool with parameterised use of the global
allocator functions, you will get a segfault. Note that this comes with a twelve
byte book-keeping overhead on every allocation and is intended only for debugging
purposes (especially memory leak checking) */
#ifndef FXDISABLE_SEPARATE_POOLS
// Define to force use of system allocator
 #ifdef DEBUG
  #define FXDISABLE_SEPARATE_POOLS 1
 #else
  #define FXDISABLE_SEPARATE_POOLS 0
 #endif
#endif

/* Customise global allocator replacements according to system.
On all systems except Linux, replace system allocator with our own
superior implementation (which is the same as Linux's) */
#ifndef __linux__
 #define MYMALLOC(size)       ptmalloc2::dlmalloc(size)
 #define MYREALLOC(ptr, size) ptmalloc2::dlrealloc(ptr, size)
 #define MYFREE(ptr)          ptmalloc2::dlfree(ptr)
 #ifndef _MSC_VER
  // Python bindings need this on POSIX
  #define FXDISABLE_GLOBAL_MARKER 0
 #endif
#else
 #define MYMALLOC(size)       ::malloc(size)
 #define MYREALLOC(ptr, size) ::realloc(ptr, size)
 #define MYFREE(ptr)          ::free(ptr)
#endif

/* Defining this saves the four or eight byte overhead used to mark a block
as having been allocated by us on every allocation in the global heap. While
this saves memory, it does mean that the allocator replacements can no longer
detect when they are freeing memory allocated by the system allocator and thus
invoke that instead.

By default, this is enabled on both Win32 and POSIX but for different reasons:-
on Win32, the global allocator is totally replaced and this works fine due to
Win32's much better DLL encapsulation. On POSIX, the global allocator is not
replaced and thus the many mismatched allocator function calls don't matter.

Note that allocations from a custom pool \b always contain the overhead (this
is how free() knows when not to pass through to ::free()) */
#ifndef FXDISABLE_GLOBAL_MARKER
 #define FXDISABLE_GLOBAL_MARKER 1
#endif

/* Defining this enables the failonfree() function whereby freeing a specified
{'fraction_non_alphanumeric': 0.14559216384683882, 'fraction_numerical': 0.005676758682101514, 'mean_word_length': 2.98094816127603, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1961000', 'n_tokens_mistral': 2236, 'n_tokens_neox': 2109, 'n_words': 986}