<commit_before>#ifndef GLOB_OPTS_H
#define GLOB_OPTS_H

#ifdef __cplusplus
extern "C" {
#endif

#include <math.h>


// We do not need alloc functions if EMBEDDED is enabled
#ifndef EMBEDDED

#include <stdlib.h>

/* DATA CUSTOMIZATIONS (depending on memory manager)-----------------------   */
/* define custom printfs and memory allocation (e.g. matlab or python) */
#ifdef MATLAB_MEX_FILE
    #include "mex.h"
    static void* c_calloc(size_t num, size_t size){
        void *m = mxCalloc(num,size);
        mexMakeMemoryPersistent(m);
        return m;
    }
    static void* c_malloc(size_t size){
        void *m = mxMalloc(size);
        mexMakeMemoryPersistent(m);
        return m;
    }
    static void* c_realloc(void *ptr, size_t size){
        void *m = mxRealloc(ptr,size);
        mexMakeMemoryPersistent(m);
        return m;
    }
    #define c_free mxFree
#elif defined PYTHON
    // Define memory allocation for python. Note that in Python 2 memory manager
    // Calloc is not implemented
    #include <Python.h>
    #define c_malloc PyMem_Malloc
    #if PY_MAJOR_VERSION >= 3
    #define c_calloc PyMem_Calloc
    #else
    static void * c_calloc(size_t num, size_t size){
      void *m = PyMem_Malloc(num*size);
      memset(m, 0, num*size);
      return m;
    }
    #endif
    // #define c_calloc(n,s) ({                     \
    //         void * p_calloc = c_malloc((n)*(s)); \
    //         memset(p_calloc, 0, (n)*(s));        \
    //         p_calloc;                            \
    //     })
    #define c_free PyMem_Free
    #define c_realloc PyMem_Realloc
#else
    #define c_malloc malloc
    #define c_calloc calloc
    #define c_free free<commit_msg>Removed multilined comments because of warnings<commit_after>#ifndef GLOB_OPTS_H
#define GLOB_OPTS_H

#ifdef __cplusplus
extern "C" {
#endif

#include <math.h>


// We do not need alloc functions if EMBEDDED is enabled
#ifndef EMBEDDED

#include <stdlib.h>

/* DATA CUSTOMIZATIONS (depending on memory manager)-----------------------   */
/* define custom printfs and memory allocation (e.g. matlab or python) */
#ifdef MATLAB_MEX_FILE
    #include "mex.h"
    static void* c_calloc(size_t num, size_t size){
        void *m = mxCalloc(num,size);
        mexMakeMemoryPersistent(m);
        return m;
    }
    static void* c_malloc(size_t size){
        void *m = mxMalloc(size);
        mexMakeMemoryPersistent(m);
        return m;
    }
    static void* c_realloc(void *ptr, size_t size){
        void *m = mxRealloc(ptr,size);
        mexMakeMemoryPersistent(m);
        return m;
    }
    #define c_free mxFree
#elif defined PYTHON
    // Define memory allocation for python. Note that in Python 2 memory manager
    // Calloc is not implemented
    #include <Python.h>
    #define c_malloc PyMem_Malloc
    #if PY_MAJOR_VERSION >= 3
    #define c_calloc PyMem_Calloc
    #else
    static void * c_calloc(size_t num, size_t size){
      void *m = PyMem_Malloc(num*size);
      memset(m, 0, num*size);
      return m;
    }
    #endif
    // #define c_calloc(n,s) ({
    //         void * p_calloc = c_malloc((n)*(s));
    //         memset(p_calloc, 0, (n)*(s));
    //         p_calloc;
    //     })
    #define c_free PyMem_Free
    #define c_realloc PyMem_Realloc
#else
    #define c_malloc malloc
    #define c_calloc calloc
    #define c_free free
{'fraction_non_alphanumeric': 0.11589008363201912, 'fraction_numerical': 0.0023894862604540022, 'mean_word_length': 2.342315369261477, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1313293', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1165, 'n_words': 310}