<commit_before>#include "performance.h"

/************************** time measurement **************************/

#ifdef Q_OS_WIN32

static double QueryPerformanceFrequency_wrapper()
{
    LARGE_INTEGER freq;

    Q_ASSERT(QueryPerformanceFrequency(&freq));

    return static_cast<double>(freq.QuadPart);
}

double ticksPerSecond = QueryPerformanceFrequency_wrapper();

perf_t perf_currentTime()
{
    LARGE_INTEGER ret;

    Q_ASSERT(QueryPerformanceCounter(&ret));

    return ret;
}

double perf_diffTime(perf_t start, perf_t end)
{
    return (end.QuadPart - start.QuadPart) / ticksPerSecond;
}

quint64 perf_diffTimeNative(perf_t start, perf_t end)
{
    return end.QuadPart - start.QuadPart;
}

#else

perf_t perf_currentTime()
{
    return QTime::currentTime();
}

double perf_diffTime(perf_t start, perf_t end)
{
    return start.msecsTo(end) / 1000.0;
}

quint64 perf_diffTimeNative(perf_t start, perf_t end)
{
    return start.msecsTo(end);
}

#endif

/************************** memory consumption **************************/

#ifdef Q_OS_WIN32

#include <psapi.h>

quint64 perf_currentMemUsage()
{
    PROCESS_MEMORY_COUNTERS pmc;

    BOOL success = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc));

    if (!success)
        return -1;

    return pmc.WorkingSetSize;
}

#else
#ifdef Q_OS_MAC

#include <malloc/malloc.h>

quint64 perf_currentMemUsage()
{
    malloc_statistics_t stats;

    malloc_zone_statistics(NULL, &stats);

    return stats.size_allocated; // maybe t.size_in_use?
}

#else
#if  _BSD_SOURCE || _SVID_SOURCE || (_XOPEN_SOURCE >= 500 || _XOPEN_SOURCE && _XOPEN_SOURCE_EXTENDED) && !(_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600)


#include <unistd.h>

static char * sbrk_0_initial = (char*)sbrk(0);

quint64 perf_currentMemUsage()
{
    return (char*)sbrk(0)-sbrk_0_initial);
}

#else
quint64 perf_currentMemUsage()
{
    return 0;
}

#endif
#endif
#endif

/* TODO:
    mem
        - more platforms!
            - mac: malloc_zone_statistics
            -
    leaks?
        - boehm gc
*/
<commit_msg>MapBox: Added a memory usage getter for linux (using procfs) and fixed the generic POSIX one.<commit_after>#include "performance.h"

/************************** time measurement **************************/

#ifdef Q_OS_WIN32

static double QueryPerformanceFrequency_wrapper()
{
    LARGE_INTEGER freq;

    Q_ASSERT(QueryPerformanceFrequency(&freq));

    return static_cast<double>(freq.QuadPart);
}

double ticksPerSecond = QueryPerformanceFrequency_wrapper();

perf_t perf_currentTime()
{
    LARGE_INTEGER ret;

    Q_ASSERT(QueryPerformanceCounter(&ret));

    return ret;
}

double perf_diffTime(perf_t start, perf_t end)
{
    return (end.QuadPart - start.QuadPart) / ticksPerSecond;
}

quint64 perf_diffTimeNative(perf_t start, perf_t end)
{
    return end.QuadPart - start.QuadPart;
}

#else

perf_t perf_currentTime()
{
    return QTime::currentTime();
}

double perf_diffTime(perf_t start, perf_t end)
{
    return start.msecsTo(end) / 1000.0;
}

quint64 perf_diffTimeNative(perf_t start, perf_t end)
{
    return start.msecsTo(end);
}

#endif

/************************** memory consumption **************************/

#ifdef Q_OS_WIN32

#include <psapi.h>

quint64 perf_currentMemUsage()
{
    PROCESS_MEMORY_COUNTERS pmc;

    BOOL success = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc));

    if (!success)
        return -1;

    return pmc.WorkingSetSize;
}

#else
#ifdef Q_OS_LINUX

#include <cstdio>

quint64 perf_currentMemUsage()
{
    FILE * statm = std::fopen("/proc/self/statm", "r");
    long long ret = -1;
    std::fscanf(statm, "%Lu", &ret);
    std::fclose(statm);
    return ret*1024;
}

#else
#ifdef Q_OS_MAC

#include <malloc/malloc.h>

quint64 perf_currentMemUsage()
{
    malloc_statistics_t stats;

    malloc_zone_statistics(NULL, &stats);

    return stats.size_allocated; // maybe t.size_in_use?
}

#else
#if  _BSD_SOURCE || _SVID_SOURCE || (_XOPEN_SOURCE >= 500 || _XOPEN_SOURCE && _XOPEN_SOURCE_EXTENDED) && !(_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600)


#include <unistd.h>

static char * sbrk_0_initial = (char*)sbrk(0);

quint64 perf_currentMemUsage()
{
    return (char*)sbrk(0)-sbrk_0_initial;
}

#else
quint64 perf_currentMemUsage()
{
    return 0;
}

#endif
#endif
#endif
#endif

/* TODO:
    mem
        - more platforms!
            - mac: malloc_zone_statistics
            -
    leaks?
        - boehm gc
*/

{'fraction_non_alphanumeric': 0.15608108108108107, 'fraction_numerical': 0.019144144144144143, 'mean_word_length': 4.169965075669383, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1535287', 'n_tokens_mistral': 1701, 'n_tokens_neox': 1586, 'n_words': 311}