<commit_before>//-----------------------------------
// Copyright <NAME> 2013-2019
//-----------------------------------


#include "OrbitProcess.h"
#include "PrintVar.h"
#include "LinuxPerfEvent.h"
#include "LinuxPerfRingBuffer.h"
#include "LinuxPerfUtils.h"
#include "LinuxEventTracer.h"
#include "LinuxEventTracerVisitor.h"
#include "LinuxPerfEventProcessor.h"
#include "LinuxUtils.h"
#include "Capture.h"
#include "Params.h"
#include "TimerManager.h"
#include "ContextSwitch.h"

#include <vector>


//-----------------------------------------------------------------------------
void LinuxEventTracer::Start()
{
    PRINT_FUNC;
#if __linux__
    m_ExitRequested = false;

    m_Thread = std::make_shared<std::thread>(&LinuxEventTracer::Run, &m_ExitRequested);

    m_Thread->detach();
#endif
}

//-----------------------------------------------------------------------------
void LinuxEventTracer::Stop()
{
    m_ExitRequested = true;
}

void LinuxEventTracer::Run(bool* a_ExitRequested)
{
    // get the number of cpus
    int cpus = std::thread::hardware_concurrency();
    // some compilers does not support hardware_concurrency.
    if (cpus == 0)
        int cpus = std::stoi(LinuxUtils::ExecuteCommand("nproc"));

    uint64_t tracepoint_ID = LinuxUtils::GetTracePointID("sched", "sched_switch");
    
    std::vector<uint32_t> fds;

    // open perf events for all cpus to keep track of process spawning
    for (int cpu = 0; cpu < cpus; cpu++)
    {
        if ( !GParams.m_SystemWideScheduling )
        {
            fds.push_back(LinuxPerfUtils::task_event_open(cpu)); 
        }
        fds.push_back(LinuxPerfUtils::tracepoint_event_open(tracepoint_ID, -1 /*pid*/, cpu)); 
    }

    std::vector<LinuxPerfRingBuffer> ring_buffers;
    for (int32_t fd : fds)
    {
        ring_buffers.emplace_back(fd);
    
        // start recording via ioctl
        LinuxPerfUtils::start_capturing(fd);
    }

    // keep threads in synch.
    auto threads = LinuxUtils::ListThreads(Capture::GTargetProcess->GetID());
    for (uint64_t tid : threads) {
        Capture::GTargetProcess->AddThreadId(tid);
    }

    // we will only use that buffer on non-system wide profiling
    LinuxPerfEventProcessor event_buffer(std::make_unique<LinuxEventTracerVisitor>());

    bool new_events = false;

    while (!(*a_ExitRequested))
    {<commit_msg>added a comment<commit_after>//-----------------------------------
// Copyright <NAME> 2013-2019
//-----------------------------------


#include "OrbitProcess.h"
#include "PrintVar.h"
#include "LinuxPerfEvent.h"
#include "LinuxPerfRingBuffer.h"
#include "LinuxPerfUtils.h"
#include "LinuxEventTracer.h"
#include "LinuxEventTracerVisitor.h"
#include "LinuxPerfEventProcessor.h"
#include "LinuxUtils.h"
#include "Capture.h"
#include "Params.h"
#include "TimerManager.h"
#include "ContextSwitch.h"

#include <vector>


//-----------------------------------------------------------------------------
void LinuxEventTracer::Start()
{
    PRINT_FUNC;
#if __linux__
    m_ExitRequested = false;

    m_Thread = std::make_shared<std::thread>(&LinuxEventTracer::Run, &m_ExitRequested);

    m_Thread->detach();
#endif
}

//-----------------------------------------------------------------------------
void LinuxEventTracer::Stop()
{
    m_ExitRequested = true;
}

void LinuxEventTracer::Run(bool* a_ExitRequested)
{
    // get the number of cpus
    int cpus = std::thread::hardware_concurrency();
    // some compilers does not support hardware_concurrency.
    if (cpus == 0)
        int cpus = std::stoi(LinuxUtils::ExecuteCommand("nproc"));

    uint64_t tracepoint_ID = LinuxUtils::GetTracePointID("sched", "sched_switch");
    
    std::vector<uint32_t> fds;

    // open perf events for all cpus to keep track of process spawning
    for (int cpu = 0; cpu < cpus; cpu++)
    {
        if ( !GParams.m_SystemWideScheduling )
        {
            fds.push_back(LinuxPerfUtils::task_event_open(cpu)); 
        }
        fds.push_back(LinuxPerfUtils::tracepoint_event_open(tracepoint_ID, -1 /*pid*/, cpu)); 
    }

    std::vector<LinuxPerfRingBuffer> ring_buffers;
    for (int32_t fd : fds)
    {
        ring_buffers.emplace_back(fd);
    
        // start recording via ioctl
        LinuxPerfUtils::start_capturing(fd);
    }

    // keep threads in synch.
    auto threads = LinuxUtils::ListThreads(Capture::GTargetProcess->GetID());
    for (uint64_t tid : threads) {
        Capture::GTargetProcess->AddThreadId(tid);
    }

    // We will only use that buffer on non-system wide profiling.
    // On non-system wide profiling, we want only process context-switches that
    //  of threads that correspond to that process. This requires us to keep track of
    //  all threads being created by that process (fork events). As these all events,
    //  come from different ring buffers, we must ensure that we will not process a 
    //  context-switch before processing the corresponding fork event. Otherwise, we would
    //  not add this context-switch to the UI.
    //  The event_buffer, ensures that we process all events in synch to their timestamp.
    // TODO: Instead of using this buffer, which allows use to process all events in the
    //  order of their timestamp, we could do the following:
    //    1) When receiving a context-switch:
    //     a) If the TID already belongs to the PID => process the context-switch
    //     b) Otherwise => add the context-switch to a queue for this TID.
    //    2) When receiving a fork creating a TID belonging to this PID 
    //     => process all events queued for this TID.
    LinuxPerfEventProcessor event_buffer(std::make_unique<LinuxEventTracerVisitor>());

    bool new_events = false;

    while (!(*a_ExitRequested))
    {
{'fraction_non_alphanumeric': 0.18373389063044235, 'fraction_numerical': 0.006966213862765587, 'mean_word_length': 2.467995169082126, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5540582', 'n_tokens_mistral': 1782, 'n_tokens_neox': 1695, 'n_words': 442}