<commit_before>/*
 *  gucefMT: GUCEF module providing multithreading solutions
 *
 *  Copyright (C) 1998 - 2020.  <NAME>
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

/*-------------------------------------------------------------------------//
//                                                                         //
//      INCLUDES                                                           //
//                                                                         //
//-------------------------------------------------------------------------*/

#include <assert.h>
#include <malloc.h>

#include "gucefMT_dvmtoswrap.h"         /* the function prototypes */

#if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
  #include <Mmsystem.h>  /* needed for timeBeginPeriod() etc */
  #include <processthreadsapi.h>
#elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
  #include <stdio.h>
  #include <stdlib.h>
  #include <unistd.h>
  #include <pthread.h>
  #include <signal.h>
#endif

/*-------------------------------------------------------------------------//
//                                                                         //
//      TYPES                                                              //
//                                                                         //
//-------------------------------------------------------------------------*/

struct SThreadData
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    DWORD threadid;
    HANDLE threadhandle;
    LPTHREAD_START_ROUTINE func;
    void* data;
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    pthread_t thread;
    TThreadFunc func;
    void* data;
    #endif
};
typedef struct SThreadData TThreadData;

/*-------------------------------------------------------------------------//
//                                                                         //
//      GLOBAL VARS                                                        //
//                                                                         //
//-------------------------------------------------------------------------*/

/*
 *      needed for PrecisionDelay() and PrecisionTickCount()
 *      Initialized in PrecisionTimerInit()
 */
#if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
static LARGE_INTEGER highPerfTimerFrequency = { 0 };
static double highPerfTimerFrequencyTicksPerMs = 0;
#endif

/*-------------------------------------------------------------------------//
//                                                                         //
//      UTILITIES                                                          //
//                                                                         //
//-------------------------------------------------------------------------*/

void
ThreadDelay( UInt32 delay )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    Sleep( delay );
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    sleep( delay );
    #endif
}

/*--------------------------------------------------------------------------*/


static UInt32 GUCEF_CALLSPEC_STD_PREFIX
ThreadMain( void* tdvptr ) GUCEF_CALLSPEC_STD_SUFFIX
{
    UInt32 retval = ( (TThreadData*)tdvptr)->func( ((TThreadData*)tdvptr)->data );
    free( ((TThreadData*)tdvptr) );
    return retval;
}

/*--------------------------------------------------------------------------*/

TThreadData*
ThreadCreate( TThreadFunc func ,
              void* data       )
{
    #ifdef GUCEF_MSWIN_BUILD
    TThreadData* td = malloc( sizeof( TThreadData ) );
    td->func = (LPTHREAD_START_ROUTINE)func;
    td->data = data;
    td->threadhandle = CreateThread( NULL                                ,  /* Security Descriptor */
                                     0                                   ,  /* initial stack size */
                                     (LPTHREAD_START_ROUTINE) ThreadMain ,  /* thread function */
                                     (LPVOID) td                         ,  /* thread argument */
                                     0                                   ,  /* creation option */
                                     &td->threadid                       ); /* thread identifier */
    if ( td->threadhandle == NULL )
    {
        free( td );
        return NULL;
    }
    return td;
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    TThreadData* td = malloc( sizeof( TThreadData ) );
    td->data = data;
    if ( 0 != pthread_create( &td->thread         ,
                              NULL                ,
                              (void*) ThreadMain  ,
                              (void*) td          ) )
    {
        free( td );
        return NULL;
    }
    return td;
    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

GUCEF_MT_PUBLIC_C UInt32
ThreadID( struct SThreadData* td )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    return td->threadid;
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    return (UInt32) td->thread;
    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
ThreadSuspend( struct SThreadData* td )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    return ( -1 != SuspendThread( td->threadhandle ) );
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
ThreadResume( struct SThreadData* td )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    return ( -1 != ResumeThread( td->threadhandle ) );
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
ThreadKill( struct SThreadData* td )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    if ( td != NULL )
    {
        UInt32 retval = TerminateThread( td->threadhandle ,
                                         1                );
        free( td );
        return retval;
    }
    return 0;
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    if ( td != NULL )
    {
        Int32 retval = (Int32) pthread_kill( td->thread, 0 );
        if ( 0 != retval )
        {
            /* an error occured */

        }
        free( td );
        return retval;
    }
    return 0;
    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
ThreadWait( struct SThreadData* td ,
            Int32 timeout          )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    if ( timeout >= 0 )
    {
        return ( WAIT_OBJECT_0 == WaitForSingleObject( td->threadhandle ,
                                                       timeout          ) );
    }
    return ( WAIT_OBJECT_0 == WaitForSingleObject( td->threadhandle ,
                                                   INFINITE         ) );
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
GetCurrentTaskID( void )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    return GetCurrentThreadId();
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    return (UInt32) pthread_self();
    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt64
PrecisionTickCount( void )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    LARGE_INTEGER t;

    assert( sizeof( LARGE_INTEGER ) <= sizeof( UInt64 ) );

    if ( QueryPerformanceCounter( &t ) == TRUE )
    {
        return t.QuadPart;
    }
    else
    {
        return GetTickCount();
    }
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    struct timeval time;
    gettimeofday( &time, NULL );

    return time.tv_usec +    /* Microseconds. */
           time.tv_sec * 1000000; /* Seconds. */

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt64
PrecisionTimerResolution( void )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    ULARGE_INTEGER t;
    if ( QueryPerformanceFrequency( (LARGE_INTEGER*) &t ) == TRUE )
    {
        return t.QuadPart;
    }
    else
    {
        return 100; /* this is the WIN32 resolution of GetTickCount(); which has a time-slice size of about 10 ms */
    }
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    return 1000000;

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

void
PrecisionDelay( UInt32 delayInMs )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )

    /*
     *      note: BE SURE YOU CALL timeBeginPeriod(1) at program startup!!!
     *      note: BE SURE YOU CALL timeEndPeriod(1) at program exit!!!
     *      note: that this code will require linking to winmm.lib
     */

    LARGE_INTEGER startTime;
    LARGE_INTEGER currentTime;
    LONGLONG ticksPassed;
    LONGLONG ticksToWait;
    LONGLONG ticksLeft;
    BOOL qpcSuccess;
    int done = 0;

    qpcSuccess = QueryPerformanceCounter( &startTime );
    if ( FALSE == qpcSuccess )
    {
        /*
         *  Fall back to regular sleep
         */
         Sleep( delayInMs );
         return;
    }
             
    ticksToWait = (LONGLONG) ( highPerfTimerFrequencyTicksPerMs * delayInMs );

    while ( 1 )
    {
        QueryPerformanceCounter( &currentTime );
        ticksPassed = (LONGLONG) ( currentTime.QuadPart - startTime.QuadPart );
        if ( ticksPassed < 0 )
            return;
        
        ticksLeft = (LONGLONG) ( ticksToWait - ticksPassed );
        if ( ticksLeft <= 0 )
            return;

        if ( ticksLeft > (Int32) ( highPerfTimerFrequencyTicksPerMs * 50 ) )
        {
            // Lets not hog the CPU if the time we have to wait is lengthy
            UInt32 longSleepTimeInMs = (UInt32) ( (ticksLeft/highPerfTimerFrequencyTicksPerMs) - (highPerfTimerFrequencyTicksPerMs*15) );
            Sleep( longSleepTimeInMs );
        }
        else        
        if ( ticksLeft > (Int32) ( highPerfTimerFrequencyTicksPerMs * 20 ) )<commit_msg>Fix for Win32 timing adjustments made in an earlier commit. When using PrecisionDelay() your code could get stuck for a long time. The reason for this was simply an incorrect calculation which resulted in an overflow which in turn resulted in an extremely long Sleep()<commit_after>/*
 *  gucefMT: GUCEF module providing multithreading solutions
 *
 *  Copyright (C) 1998 - 2020.  <NAME>
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

/*-------------------------------------------------------------------------//
//                                                                         //
//      INCLUDES                                                           //
//                                                                         //
//-------------------------------------------------------------------------*/

#include <assert.h>
#include <malloc.h>

#include "gucefMT_dvmtoswrap.h"         /* the function prototypes */

#if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
  #include <Mmsystem.h>  /* needed for timeBeginPeriod() etc */
  #include <processthreadsapi.h>
#elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
  #include <stdio.h>
  #include <stdlib.h>
  #include <unistd.h>
  #include <pthread.h>
  #include <signal.h>
#endif

/*-------------------------------------------------------------------------//
//                                                                         //
//      TYPES                                                              //
//                                                                         //
//-------------------------------------------------------------------------*/

struct SThreadData
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    DWORD threadid;
    HANDLE threadhandle;
    LPTHREAD_START_ROUTINE func;
    void* data;
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    pthread_t thread;
    TThreadFunc func;
    void* data;
    #endif
};
typedef struct SThreadData TThreadData;

/*-------------------------------------------------------------------------//
//                                                                         //
//      GLOBAL VARS                                                        //
//                                                                         //
//-------------------------------------------------------------------------*/

/*
 *      needed for PrecisionDelay() and PrecisionTickCount()
 *      Initialized in PrecisionTimerInit()
 */
#if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
static LARGE_INTEGER highPerfTimerFrequency = { 0 };
static double highPerfTimerFrequencyTicksPerMs = 0;
#endif

/*-------------------------------------------------------------------------//
//                                                                         //
//      UTILITIES                                                          //
//                                                                         //
//-------------------------------------------------------------------------*/

void
ThreadDelay( UInt32 delay )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    Sleep( delay );
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    sleep( delay );
    #endif
}

/*--------------------------------------------------------------------------*/


static UInt32 GUCEF_CALLSPEC_STD_PREFIX
ThreadMain( void* tdvptr ) GUCEF_CALLSPEC_STD_SUFFIX
{
    UInt32 retval = ( (TThreadData*)tdvptr)->func( ((TThreadData*)tdvptr)->data );
    free( ((TThreadData*)tdvptr) );
    return retval;
}

/*--------------------------------------------------------------------------*/

TThreadData*
ThreadCreate( TThreadFunc func ,
              void* data       )
{
    #ifdef GUCEF_MSWIN_BUILD
    TThreadData* td = malloc( sizeof( TThreadData ) );
    td->func = (LPTHREAD_START_ROUTINE)func;
    td->data = data;
    td->threadhandle = CreateThread( NULL                                ,  /* Security Descriptor */
                                     0                                   ,  /* initial stack size */
                                     (LPTHREAD_START_ROUTINE) ThreadMain ,  /* thread function */
                                     (LPVOID) td                         ,  /* thread argument */
                                     0                                   ,  /* creation option */
                                     &td->threadid                       ); /* thread identifier */
    if ( td->threadhandle == NULL )
    {
        free( td );
        return NULL;
    }
    return td;
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    TThreadData* td = malloc( sizeof( TThreadData ) );
    td->data = data;
    if ( 0 != pthread_create( &td->thread         ,
                              NULL                ,
                              (void*) ThreadMain  ,
                              (void*) td          ) )
    {
        free( td );
        return NULL;
    }
    return td;
    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

GUCEF_MT_PUBLIC_C UInt32
ThreadID( struct SThreadData* td )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    return td->threadid;
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    return (UInt32) td->thread;
    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
ThreadSuspend( struct SThreadData* td )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    return ( -1 != SuspendThread( td->threadhandle ) );
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
ThreadResume( struct SThreadData* td )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    return ( -1 != ResumeThread( td->threadhandle ) );
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
ThreadKill( struct SThreadData* td )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    if ( td != NULL )
    {
        UInt32 retval = TerminateThread( td->threadhandle ,
                                         1                );
        free( td );
        return retval;
    }
    return 0;
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    if ( td != NULL )
    {
        Int32 retval = (Int32) pthread_kill( td->thread, 0 );
        if ( 0 != retval )
        {
            /* an error occured */

        }
        free( td );
        return retval;
    }
    return 0;
    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
ThreadWait( struct SThreadData* td ,
            Int32 timeout          )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    if ( timeout >= 0 )
    {
        return ( WAIT_OBJECT_0 == WaitForSingleObject( td->threadhandle ,
                                                       timeout          ) );
    }
    return ( WAIT_OBJECT_0 == WaitForSingleObject( td->threadhandle ,
                                                   INFINITE         ) );
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt32
GetCurrentTaskID( void )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    return GetCurrentThreadId();
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )
    return (UInt32) pthread_self();
    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt64
PrecisionTickCount( void )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    LARGE_INTEGER t;

    assert( sizeof( LARGE_INTEGER ) <= sizeof( UInt64 ) );

    if ( QueryPerformanceCounter( &t ) == TRUE )
    {
        return t.QuadPart;
    }
    else
    {
        return GetTickCount();
    }
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    struct timeval time;
    gettimeofday( &time, NULL );

    return time.tv_usec +    /* Microseconds. */
           time.tv_sec * 1000000; /* Seconds. */

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

UInt64
PrecisionTimerResolution( void )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )
    ULARGE_INTEGER t;
    if ( QueryPerformanceFrequency( (LARGE_INTEGER*) &t ) == TRUE )
    {
        return t.QuadPart;
    }
    else
    {
        return 100; /* this is the WIN32 resolution of GetTickCount(); which has a time-slice size of about 10 ms */
    }
    #elif ( ( GUCEF_PLATFORM == GUCEF_PLATFORM_LINUX ) || ( GUCEF_PLATFORM == GUCEF_PLATFORM_ANDROID ) )

    return 1000000;

    #else
    #error unsupported target platform
    #endif
}

/*--------------------------------------------------------------------------*/

void
PrecisionDelay( UInt32 delayInMs )
{
    #if ( GUCEF_PLATFORM == GUCEF_PLATFORM_MSWIN )

    /*
     *      note: BE SURE YOU CALL timeBeginPeriod(1) at program startup!!!
     *      note: BE SURE YOU CALL timeEndPeriod(1) at program exit!!!
     *      note: that this code will require linking to winmm.lib
     */

    LARGE_INTEGER startTime;
    LARGE_INTEGER currentTime;
    LONGLONG ticksPassed;
    LONGLONG ticksToWait;
    LONGLONG ticksLeft;
    BOOL qpcSuccess;
    int done = 0;

    qpcSuccess = QueryPerformanceCounter( &startTime );
    if ( FALSE == qpcSuccess )
    {
        /*
         *  Fall back to regular sleep
         */
         Sleep( delayInMs );
         return;
    }
             
    ticksToWait = (LONGLONG) ( highPerfTimerFrequencyTicksPerMs * delayInMs );

    while ( 1 )
    {
        QueryPerformanceCounter( &currentTime );
        ticksPassed = (LONGLONG) ( currentTime.QuadPart - startTime.QuadPart );
        if ( ticksPassed < 0 )
            return;
        
        ticksLeft = (LONGLONG) ( ticksToWait - ticksPassed );
        if ( ticksLeft <= 0 )
            return;

        if ( ticksLeft > (Int32) ( highPerfTimerFrequencyTicksPerMs * 50 ) )
        {
            // Lets not hog the CPU if the time we have to wait is lengthy
            UInt32 longSleepTimeInMs = (UInt32) ( ( ticksLeft - (highPerfTimerFrequencyTicksPerMs*15) ) / highPerfTimerFrequencyTicksPerMs );
            Sleep( longSleepTimeInMs );
        }
        else        
        if ( ticksLeft > (Int32) ( highPerfTimerFrequencyTicksPerMs * 20 ) )
{'fraction_non_alphanumeric': 0.20565596427812036, 'fraction_numerical': 0.009100574101637253, 'mean_word_length': 1.219537517697027, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 98, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '920602', 'n_tokens_mistral': 6828, 'n_tokens_neox': 6366, 'n_words': 1417}