<commit_before>/*
   Copyright (c) 2009-2020, Intel Corporation
   All rights reserved.

   Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

 * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 * Neither the name of Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

 /*!     \file pcm-raw.cpp
         \brief Example of using CPU counters: implements a performance counter monitoring utility with raw events interface
   */
#include <iostream>
#ifdef _MSC_VER
#define strtok_r strtok_s
#include <windows.h>
#include "../PCM_Win/windriver.h"
#else
#include <unistd.h>
#include <signal.h>
#include <sys/time.h> // for gettimeofday()
#endif
#include <math.h>
#include <iomanip>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <string>
#include <assert.h>
#include <bitset>
#include <regex>
#include <unordered_map>
#include "cpucounters.h"
#include "utils.h"
#include "gccversion.h"
#ifndef PCM_GCC_6_OR_BELOW
#pragma warning(push, 0)
#include "simdjson/simdjson.h"
#pragma warning(pop)
#endif

#ifdef _MSC_VER
#include "freegetopt/getopt.h"
#endif

#include <vector>
#define PCM_DELAY_DEFAULT 1.0 // in seconds
#define PCM_DELAY_MIN 0.015 // 15 milliseconds is practical on most modern CPUs
#define MAX_CORES 4096

using namespace std;
using namespace pcm;

void print_usage(const string progname)
{
    cerr << "\n Usage: \n " << progname
        << " --help | [delay] [options] [-- external_program [external_program_options]]\n";
    cerr << "   <delay>                               => time interval to sample performance counters.\n";
    cerr << "                                            If not specified, or 0, with external program given\n";
    cerr << "                                            will read counters only after external program finishes\n";
    cerr << " Supported <options> are: \n";
    cerr << "  -h    | --help      | /h               => print this help and exit\n";
    cerr << "  -csv[=file.csv]     | /csv[=file.csv]  => output compact CSV format to screen or\n"
        << "                                            to a file, in case filename is provided\n";
    cerr << "  [-e event1] [-e event2] [-e event3] .. => list of custom events to monitor\n";
    cerr << "  event description example: -e core/config=0x30203,name=LD_BLOCKS.STORE_FORWARD/ -e core/fixed,config=0x333/ \n";
    cerr << "                             -e cha/config=0,name=UNC_CHA_CLOCKTICKS/ -e imc/fixed,name=DRAM_CLOCKS/\n";
#ifndef PCM_GCC_6_OR_BELOW
    cerr << "                             -e NAME where the NAME is an event from https://download.01.org/perfmon/ event lists\n";
#endif
    cerr << "  -yc   | --yescores  | /yc              => enable specific cores to output\n";
    cerr << "  -f    | /f                             => enforce flushing each line for interactive output\n";
    cerr << "  -i[=number] | /i[=number]              => allow to determine number of iterations\n";
    cerr << "  -tr | /tr                              => transpose output (print single event data in a row)\n";
    cerr << "  -el event_list.txt | /tr event_list.txt  => read event list from event_list.txt file, \n";
    cerr << "                                              each line represents an event group collected together\n";
    print_help_force_rtm_abort_mode(41);
    cerr << " Examples:\n";
    cerr << "  " << progname << " 1                   => print counters every second without core and socket output\n";
    cerr << "  " << progname << " 0.5 -csv=test.log   => twice a second save counter values to test.log in CSV format\n";
    cerr << "  " << progname << " /csv 5 2>/dev/null  => one sampe every 5 seconds, and discard all diagnostic output\n";
    cerr << "\n";
}

#ifndef PCM_GCC_6_OR_BELOW
using namespace simdjson;

std::vector<std::shared_ptr<simdjson::dom::parser> > JSONparsers;
std::unordered_map<std::string, simdjson::dom::object> PMUEventMap;
std::unordered_map<std::string, simdjson::dom::element> PMURegisterDeclarations;

bool initPMUEventMap()
{
    static bool inited = false;

    if (inited == true)
    {
        return true;
    }
    inited = true;
    const auto mapfile = "mapfile.csv";
    std::ifstream in(mapfile);
    std::string line, item;

    if (!in.is_open())
    {
        cerr << "ERROR: File " << mapfile << " can't be open. \n";
        return false;
    }
    int32 FMSPos = -1;
    int32 FilenamePos = -1;
    int32 EventTypetPos = -1;
    if (std::getline(in, line))
    {
        auto header = split(line, ',');
        for (int32 i = 0; i < (int32)header.size(); ++i)
        {
            if (header[i] == "Family-model")
            {
                FMSPos = i;
            }
            else if (header[i] == "Filename")
            {
                FilenamePos = i;
            }
            else if (header[i] == "EventType")
            {
                EventTypetPos = i;
            }
        }
    }
    else
    {
        cerr << "Can't read first line from " << mapfile << " \n";
        return false;
    }
    // cout << FMSPos << " " << FilenamePos << " " << EventTypetPos << "\n";
    assert(FMSPos > 0);
    assert(FilenamePos > 0);
    assert(EventTypetPos > 0);
    const std::string ourFMS = PCM::getInstance()->getCPUFamilyModelString();
    // cout << "Our FMS: " << ourFMS << "\n";
    std::map<std::string, std::string> eventFiles;
    while (std::getline(in, line))
    {
        auto tokens = split(line, ',');
        assert(FMSPos < (int32)tokens.size());
        assert(FilenamePos < (int32)tokens.size());
        assert(EventTypetPos < (int32)tokens.size());
        std::regex FMSRegex(tokens[FMSPos]);
        std::cmatch FMSMatch;
        if (std::regex_search(ourFMS.c_str(), FMSMatch, FMSRegex))
        {
            cout << tokens[FMSPos] << " " << tokens[EventTypetPos] << " " << tokens[FilenamePos] << " matched\n";
            eventFiles[tokens[EventTypetPos]] = tokens[FilenamePos];
        }
    }
    in.close();

    if (eventFiles.empty())
    {
        cerr << "ERROR: CPU " << ourFMS << "not found in " << mapfile << "\n";
        return false;
    }

    for (const auto evfile : eventFiles)
    {
        std::string path = std::string(".") + evfile.second;
        try {

            cout << evfile.first << " " << evfile.second << "\n";

            if (evfile.first == "core" || evfile.first == "uncore")
            {
                JSONparsers.push_back(std::make_shared<simdjson::dom::parser>());
                for (simdjson::dom::object eventObj : JSONparsers.back()->load(path)) {
                    // cout << "Event ----------------\n";
                    const std::string EventName{eventObj["EventName"].get_c_str()};
                    if (EventName.empty())
                    {
                        cerr << "Did not find EventName in JSON object:\n";
                        for (const auto keyValue : eventObj)
                        {
                            cout << "key: " << keyValue.key << " value: " << keyValue.value << "\n";
                        }
                    }
                    else
                    {
                        PMUEventMap[EventName] = eventObj;
                    }
                }
            }
        }
        catch (std::exception& e)
        {
            cerr << "Error while opening and/or parsing " << path << " : " << e.what() << "\n";
        }
    }
    return true;
}

bool addEventFromDB(PCM::RawPMUConfigs& curPMUConfigs, const string & fullEventStr)
{
    if (initPMUEventMap() == false)
    {
        cerr << "ERROR: PMU Event map can not be initialized\n";
        return false;
    }
    if (fullEventStr.empty())
    {
        return true;
    }
    // cerr << "Parsing event " << fullEventStr << "\n";
    // cerr << "size: " << fullEventStr.size() << "\n";
    const auto EventTokens = split(fullEventStr, ':');
    assert(!EventTokens.empty());

    const auto eventStr = EventTokens[0];

    // cerr << "size: " << eventStr.size() << "\n";

    if (eventStr == "MSR_EVENT")
    {
        std::cerr << fullEventStr << " event is not supported. Ignoring the event.\n";
        return true;
    }

    if (PMUEventMap.find(eventStr) == PMUEventMap.end())
    {
        cerr << "ERROR: event " << eventStr << " could not be found in event database. Ignoring the event.\n";
        return true;
    }

    auto mod = EventTokens.begin();
    ++mod;

    const auto eventObj = PMUEventMap[eventStr];
    std::string pmuName;
    PCM::RawEventConfig config = { {0,0,0}, "" };
    bool fixed = false;
    config.second = fullEventStr;

    if (eventObj["Unit"].error() == NO_SUCH_FIELD)
    {
        pmuName = "core";
        const std::string path = std::string("PMURegisterDeclarations/") + PCM::getInstance()->getCPUFamilyModelString() + "." + pmuName + ".json";
        if (PMURegisterDeclarations.find(pmuName) == PMURegisterDeclarations.end())
        {
            // declaration not loaded yet
            try {

                JSONparsers.push_back(std::make_shared<simdjson::dom::parser>());
                PMURegisterDeclarations[pmuName] = JSONparsers.back()->load(path);
            }
            catch (std::exception& e)
            {
                cerr << "Error while opening and/or parsing " << path << " : " << e.what() << "\n";
                return false;
            }
        }
        std::string CounterStr{eventObj["Counter"].get_c_str()};
        // cout << "Counter: " << CounterStr << "\n";
        int fixedCounter = -1;
        if (pcm_sscanf(CounterStr) >> s_expect("Fixed counter ") >> fixedCounter)
        {
            // cout << "Requested fixed counter " << fixedCounter << "\n";
            fixed = true;
        }
        else
        {
            // cout << "Requested programmable counter\n";
        }

        try {
            simdjson::dom::object PMUDeclObj;
            if (fixed)
            {
                PMUDeclObj = PMURegisterDeclarations[pmuName][std::string("fixed") + std::to_string(fixedCounter)].get_object();
            }
            else
            {
                PMUDeclObj = PMURegisterDeclarations[pmuName]["programmable"].get_object();
            }
            for (const auto registerKeyValue : PMUDeclObj)
            {
                // cout << "Setting " << registerKeyValue.key << " : " << registerKeyValue.value << "\n";
                simdjson::dom::object fieldDescriptionObj = registerKeyValue.value;
                // cout << "   config: " << uint64_t(fieldDescriptionObj["Config"]) << "\n";
                // cout << "   Position: " << uint64_t(fieldDescriptionObj["Position"]) << "\n";
                std::string fieldNameStr{ registerKeyValue.key.begin(), registerKeyValue.key.end() };
                const int64_t position = int64_t(fieldDescriptionObj["Position"]);
                if (position == -1)
                {
                    continue; // field ignored
                }
                if (eventObj[fieldNameStr].error() == NO_SUCH_FIELD)
                {
                    // cerr << fieldNameStr << " not found\n";
                    if (fieldDescriptionObj["DefaultValue"].error() == NO_SUCH_FIELD)
                    {
                        cerr << "ERROR: DefaultValue not provided for field \"" << fieldNameStr << "\" in " << path << "\n";
                        return false;
                    }
                    else
                    {
                        config.first[uint64_t(fieldDescriptionObj["Config"])] |= uint64_t(fieldDescriptionObj["DefaultValue"]) << position;
                    }
                }
                else
                {
                    std::string fieldValueStr{ eventObj[fieldNameStr].get_c_str() };
                    fieldValueStr.erase(std::remove(fieldValueStr.begin(), fieldValueStr.end(), '\"'), fieldValueStr.end());
                    // cout << " field value is " << fieldValueStr << " " << read_number(fieldValueStr.c_str()) <<  "\n";
                    config.first[uint64_t(fieldDescriptionObj["Config"])] |= read_number(fieldValueStr.c_str()) << position;
                }
            }

            while (mod != EventTokens.end())
            {
                const auto assigment = split(*mod, '=');
                if (*mod == "SUP")
                {
                    const auto userPos = int64_t(PMUDeclObj["User"]["Position"]);
                    const auto userConfig = uint64_t(PMUDeclObj["User"]["Config"]);
                    config.first[userConfig] &= ~(1ULL << userPos); // clear user bit
                    const auto osPos = int64_t(PMUDeclObj["OS"]["Position"]);
                    const auto osConfig = uint64_t(PMUDeclObj["OS"]["Config"]);
                    config.first[osConfig] |= (1ULL << osPos); // set os bit
                }
                else if (*mod == "USER")
                {
                    const auto userPos = int64_t(PMUDeclObj["User"]["Position"]);
                    const auto userConfig = uint64_t(PMUDeclObj["User"]["Config"]);
                    config.first[userConfig] |= (1ULL << userPos); // set user bit
                    const auto osPos = int64_t(PMUDeclObj["OS"]["Position"]);
                    const auto osConfig = uint64_t(PMUDeclObj["OS"]["Config"]);
                    config.first[osConfig] &= ~(1ULL << osPos); // clear os bit
                }
                else if (mod->length() >= 2 && (*mod)[0] == 'c')
                {
                    // Counter Mask modifier
                    const std::string CounterMaskStr{ mod->begin() + 1, mod->end() };
                    const auto CounterMask = read_number(CounterMaskStr.c_str());
                    const auto cmaskPos = int64_t(PMUDeclObj["CounterMask"]["Position"]);
                    const auto cmaskConfig = uint64_t(PMUDeclObj["CounterMask"]["Config"]);
                    config.first[cmaskConfig] |= CounterMask << cmaskPos;
                }
                else if (mod->length() >= 2 && (*mod)[0] == 'e')
                {
                    // Edge Detect modifier
                    const std::string Str{ mod->begin() + 1, mod->end() };
                    const auto value = read_number(Str.c_str());
                    const auto pos = int64_t(PMUDeclObj["EdgeDetect"]["Position"]);
                    const auto Config = uint64_t(PMUDeclObj["EdgeDetect"]["Config"]);
                    config.first[Config] &= ~(1ULL << pos); // clear bit
                    config.first[Config] |= value << pos; // clear bit
                }
                else if (assigment.size() == 2 && assigment[0] == "request")
                {
                    cerr << "Unsupported event modifier: " << *mod << " in event " << fullEventStr << "\n";
                    return true;
                }
                else if (assigment.size() == 2 && assigment[0] == "response")
                {
                    cerr << "Unsupported event modifier: " << *mod << " in event " << fullEventStr << "\n";
                    return true;
                }
                else
                {
                    cerr << "Unsupported event modifier: " << *mod << " in event " << fullEventStr << "\n";
                    return false;
                }
                ++mod;
            }

            if (fixed)
            {
                curPMUConfigs[pmuName].fixed.push_back(config);
            }
            else
            {
                curPMUConfigs[pmuName].programmable.push_back(config);
            }
        }
        catch (std::exception& e)
        {
            cerr << "Error while setting a register field for event " << eventStr << " : " << e.what() << "\n";<commit_msg>refactor and take the field width into account<commit_after>/*
   Copyright (c) 2009-2020, Intel Corporation
   All rights reserved.

   Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

 * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 * Neither the name of Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

 /*!     \file pcm-raw.cpp
         \brief Example of using CPU counters: implements a performance counter monitoring utility with raw events interface
   */
#include <iostream>
#ifdef _MSC_VER
#define strtok_r strtok_s
#include <windows.h>
#include "../PCM_Win/windriver.h"
#else
#include <unistd.h>
#include <signal.h>
#include <sys/time.h> // for gettimeofday()
#endif
#include <math.h>
#include <iomanip>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <string>
#include <assert.h>
#include <bitset>
#include <regex>
#include <unordered_map>
#include "cpucounters.h"
#include "utils.h"
#include "gccversion.h"
#ifndef PCM_GCC_6_OR_BELOW
#pragma warning(push, 0)
#include "simdjson/simdjson.h"
#pragma warning(pop)
#endif

#ifdef _MSC_VER
#include "freegetopt/getopt.h"
#endif

#include <vector>
#define PCM_DELAY_DEFAULT 1.0 // in seconds
#define PCM_DELAY_MIN 0.015 // 15 milliseconds is practical on most modern CPUs
#define MAX_CORES 4096

using namespace std;
using namespace pcm;

void print_usage(const string progname)
{
    cerr << "\n Usage: \n " << progname
        << " --help | [delay] [options] [-- external_program [external_program_options]]\n";
    cerr << "   <delay>                               => time interval to sample performance counters.\n";
    cerr << "                                            If not specified, or 0, with external program given\n";
    cerr << "                                            will read counters only after external program finishes\n";
    cerr << " Supported <options> are: \n";
    cerr << "  -h    | --help      | /h               => print this help and exit\n";
    cerr << "  -csv[=file.csv]     | /csv[=file.csv]  => output compact CSV format to screen or\n"
        << "                                            to a file, in case filename is provided\n";
    cerr << "  [-e event1] [-e event2] [-e event3] .. => list of custom events to monitor\n";
    cerr << "  event description example: -e core/config=0x30203,name=LD_BLOCKS.STORE_FORWARD/ -e core/fixed,config=0x333/ \n";
    cerr << "                             -e cha/config=0,name=UNC_CHA_CLOCKTICKS/ -e imc/fixed,name=DRAM_CLOCKS/\n";
#ifndef PCM_GCC_6_OR_BELOW
    cerr << "                             -e NAME where the NAME is an event from https://download.01.org/perfmon/ event lists\n";
#endif
    cerr << "  -yc   | --yescores  | /yc              => enable specific cores to output\n";
    cerr << "  -f    | /f                             => enforce flushing each line for interactive output\n";
    cerr << "  -i[=number] | /i[=number]              => allow to determine number of iterations\n";
    cerr << "  -tr | /tr                              => transpose output (print single event data in a row)\n";
    cerr << "  -el event_list.txt | /tr event_list.txt  => read event list from event_list.txt file, \n";
    cerr << "                                              each line represents an event group collected together\n";
    print_help_force_rtm_abort_mode(41);
    cerr << " Examples:\n";
    cerr << "  " << progname << " 1                   => print counters every second without core and socket output\n";
    cerr << "  " << progname << " 0.5 -csv=test.log   => twice a second save counter values to test.log in CSV format\n";
    cerr << "  " << progname << " /csv 5 2>/dev/null  => one sampe every 5 seconds, and discard all diagnostic output\n";
    cerr << "\n";
}

#ifndef PCM_GCC_6_OR_BELOW
using namespace simdjson;

std::vector<std::shared_ptr<simdjson::dom::parser> > JSONparsers;
std::unordered_map<std::string, simdjson::dom::object> PMUEventMap;
std::unordered_map<std::string, simdjson::dom::element> PMURegisterDeclarations;

bool initPMUEventMap()
{
    static bool inited = false;

    if (inited == true)
    {
        return true;
    }
    inited = true;
    const auto mapfile = "mapfile.csv";
    std::ifstream in(mapfile);
    std::string line, item;

    if (!in.is_open())
    {
        cerr << "ERROR: File " << mapfile << " can't be open. \n";
        return false;
    }
    int32 FMSPos = -1;
    int32 FilenamePos = -1;
    int32 EventTypetPos = -1;
    if (std::getline(in, line))
    {
        auto header = split(line, ',');
        for (int32 i = 0; i < (int32)header.size(); ++i)
        {
            if (header[i] == "Family-model")
            {
                FMSPos = i;
            }
            else if (header[i] == "Filename")
            {
                FilenamePos = i;
            }
            else if (header[i] == "EventType")
            {
                EventTypetPos = i;
            }
        }
    }
    else
    {
        cerr << "Can't read first line from " << mapfile << " \n";
        return false;
    }
    // cout << FMSPos << " " << FilenamePos << " " << EventTypetPos << "\n";
    assert(FMSPos > 0);
    assert(FilenamePos > 0);
    assert(EventTypetPos > 0);
    const std::string ourFMS = PCM::getInstance()->getCPUFamilyModelString();
    // cout << "Our FMS: " << ourFMS << "\n";
    std::map<std::string, std::string> eventFiles;
    while (std::getline(in, line))
    {
        auto tokens = split(line, ',');
        assert(FMSPos < (int32)tokens.size());
        assert(FilenamePos < (int32)tokens.size());
        assert(EventTypetPos < (int32)tokens.size());
        std::regex FMSRegex(tokens[FMSPos]);
        std::cmatch FMSMatch;
        if (std::regex_search(ourFMS.c_str(), FMSMatch, FMSRegex))
        {
            cout << tokens[FMSPos] << " " << tokens[EventTypetPos] << " " << tokens[FilenamePos] << " matched\n";
            eventFiles[tokens[EventTypetPos]] = tokens[FilenamePos];
        }
    }
    in.close();

    if (eventFiles.empty())
    {
        cerr << "ERROR: CPU " << ourFMS << "not found in " << mapfile << "\n";
        return false;
    }

    for (const auto evfile : eventFiles)
    {
        std::string path = std::string(".") + evfile.second;
        try {

            cout << evfile.first << " " << evfile.second << "\n";

            if (evfile.first == "core" || evfile.first == "uncore")
            {
                JSONparsers.push_back(std::make_shared<simdjson::dom::parser>());
                for (simdjson::dom::object eventObj : JSONparsers.back()->load(path)) {
                    // cout << "Event ----------------\n";
                    const std::string EventName{eventObj["EventName"].get_c_str()};
                    if (EventName.empty())
                    {
                        cerr << "Did not find EventName in JSON object:\n";
                        for (const auto keyValue : eventObj)
                        {
                            cout << "key: " << keyValue.key << " value: " << keyValue.value << "\n";
                        }
                    }
                    else
                    {
                        PMUEventMap[EventName] = eventObj;
                    }
                }
            }
        }
        catch (std::exception& e)
        {
            cerr << "Error while opening and/or parsing " << path << " : " << e.what() << "\n";
        }
    }
    return true;
}

bool addEventFromDB(PCM::RawPMUConfigs& curPMUConfigs, const string & fullEventStr)
{
    if (initPMUEventMap() == false)
    {
        cerr << "ERROR: PMU Event map can not be initialized\n";
        return false;
    }
    if (fullEventStr.empty())
    {
        return true;
    }
    // cerr << "Parsing event " << fullEventStr << "\n";
    // cerr << "size: " << fullEventStr.size() << "\n";
    const auto EventTokens = split(fullEventStr, ':');
    assert(!EventTokens.empty());

    const auto eventStr = EventTokens[0];

    // cerr << "size: " << eventStr.size() << "\n";

    if (eventStr == "MSR_EVENT")
    {
        std::cerr << fullEventStr << " event is not supported. Ignoring the event.\n";
        return true;
    }

    if (PMUEventMap.find(eventStr) == PMUEventMap.end())
    {
        cerr << "ERROR: event " << eventStr << " could not be found in event database. Ignoring the event.\n";
        return true;
    }

    auto mod = EventTokens.begin();
    ++mod;

    const auto eventObj = PMUEventMap[eventStr];
    std::string pmuName;
    PCM::RawEventConfig config = { {0,0,0}, "" };
    bool fixed = false;
    config.second = fullEventStr;

    if (eventObj["Unit"].error() == NO_SUCH_FIELD)
    {
        pmuName = "core";
        const std::string path = std::string("PMURegisterDeclarations/") + PCM::getInstance()->getCPUFamilyModelString() + "." + pmuName + ".json";
        if (PMURegisterDeclarations.find(pmuName) == PMURegisterDeclarations.end())
        {
            // declaration not loaded yet
            try {

                JSONparsers.push_back(std::make_shared<simdjson::dom::parser>());
                PMURegisterDeclarations[pmuName] = JSONparsers.back()->load(path);
            }
            catch (std::exception& e)
            {
                cerr << "Error while opening and/or parsing " << path << " : " << e.what() << "\n";
                return false;
            }
        }
        std::string CounterStr{eventObj["Counter"].get_c_str()};
        // cout << "Counter: " << CounterStr << "\n";
        int fixedCounter = -1;
        if (pcm_sscanf(CounterStr) >> s_expect("Fixed counter ") >> fixedCounter)
        {
            // cout << "Requested fixed counter " << fixedCounter << "\n";
            fixed = true;
        }
        else
        {
            // cout << "Requested programmable counter\n";
        }

        try {
            simdjson::dom::object PMUDeclObj;
            if (fixed)
            {
                PMUDeclObj = PMURegisterDeclarations[pmuName][std::string("fixed") + std::to_string(fixedCounter)].get_object();
            }
            else
            {
                PMUDeclObj = PMURegisterDeclarations[pmuName]["programmable"].get_object();
            }
            for (const auto registerKeyValue : PMUDeclObj)
            {
                // cout << "Setting " << registerKeyValue.key << " : " << registerKeyValue.value << "\n";
                simdjson::dom::object fieldDescriptionObj = registerKeyValue.value;
                // cout << "   config: " << uint64_t(fieldDescriptionObj["Config"]) << "\n";
                // cout << "   Position: " << uint64_t(fieldDescriptionObj["Position"]) << "\n";
                std::string fieldNameStr{ registerKeyValue.key.begin(), registerKeyValue.key.end() };
                const int64_t position = int64_t(fieldDescriptionObj["Position"]);
                if (position == -1)
                {
                    continue; // field ignored
                }
                if (eventObj[fieldNameStr].error() == NO_SUCH_FIELD)
                {
                    // cerr << fieldNameStr << " not found\n";
                    if (fieldDescriptionObj["DefaultValue"].error() == NO_SUCH_FIELD)
                    {
                        cerr << "ERROR: DefaultValue not provided for field \"" << fieldNameStr << "\" in " << path << "\n";
                        return false;
                    }
                    else
                    {
                        config.first[uint64_t(fieldDescriptionObj["Config"])] |= uint64_t(fieldDescriptionObj["DefaultValue"]) << position;
                    }
                }
                else
                {
                    std::string fieldValueStr{ eventObj[fieldNameStr].get_c_str() };
                    fieldValueStr.erase(std::remove(fieldValueStr.begin(), fieldValueStr.end(), '\"'), fieldValueStr.end());
                    // cout << " field value is " << fieldValueStr << " " << read_number(fieldValueStr.c_str()) <<  "\n";
                    config.first[uint64_t(fieldDescriptionObj["Config"])] |= read_number(fieldValueStr.c_str()) << position;
                }
            }

            auto setField = [&PMUDeclObj, &config](const char* field, const uint64 value)
            {
                const auto pos = int64_t(PMUDeclObj[field]["Position"]);
                const auto cfg = uint64_t(PMUDeclObj[field]["Config"]);
                const auto width = uint64_t(PMUDeclObj[field]["Width"]);
                const uint64 mask = (1ULL << width) - 1ULL; // 1 -> 1b, 2 -> 11b, 3 -> 111b
                config.first[cfg] &= ~(mask << pos); // clear
                config.first[cfg] |= (value & mask) << pos; // set
            };

            while (mod != EventTokens.end())
            {
                const auto assigment = split(*mod, '=');
                if (*mod == "SUP")
                {
                    setField("User", 0);
                    setField("OS", 1);
                }
                else if (*mod == "USER")
                {
                    setField("User", 1);
                    setField("OS", 0);
                }
                else if (mod->length() >= 2 && (*mod)[0] == 'c')
                {
                    // Counter Mask modifier
                    const std::string CounterMaskStr{ mod->begin() + 1, mod->end() };
                    setField("CounterMask", read_number(CounterMaskStr.c_str()));
                }
                else if (mod->length() >= 2 && (*mod)[0] == 'e')
                {
                    // Edge Detect modifier
                    const std::string Str{ mod->begin() + 1, mod->end() };
                    setField("EdgeDetect", read_number(Str.c_str()));
                }
                else if (assigment.size() == 2 && assigment[0] == "request")
                {
                    cerr << "Unsupported event modifier: " << *mod << " in event " << fullEventStr << "\n";
                    return true;
                }
                else if (assigment.size() == 2 && assigment[0] == "response")
                {
                    cerr << "Unsupported event modifier: " << *mod << " in event " << fullEventStr << "\n";
                    return true;
                }
                else
                {
                    cerr << "Unsupported event modifier: " << *mod << " in event " << fullEventStr << "\n";
                    return false;
                }
                ++mod;
            }

            if (fixed)
            {
                curPMUConfigs[pmuName].fixed.push_back(config);
            }
            else
            {
                curPMUConfigs[pmuName].programmable.push_back(config);
            }
        }
        catch (std::exception& e)
        {
            cerr << "Error while setting a register field for event " << eventStr << " : " << e.what() << "\n";
{'fraction_non_alphanumeric': 0.13378259949835303, 'fraction_numerical': 0.00779668187724759, 'mean_word_length': 1.8295852928601966, 'pattern_counts': {'":': 0, '<': 661, '<?xml version=': 0, '>': 120, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1006501', 'n_tokens_mistral': 9777, 'n_tokens_neox': 9224, 'n_words': 2439}