<commit_before>#include <iostream>
#include <fstream>
#include <stdlib.h>
#include <syscall.h>
#include <vector>
#include <deque>
#include <map>

#include <cstdio>
#include <cassert>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <strings.h>
#include <sys/syscall.h>
#include <sys/file.h>
#include <linux/futex.h>
#include <string.h>
#include <pthread.h>

#define __STDC_FORMAT_MACROS
#include <inttypes.h>

#include "pin.H"
#ifdef PINPLAY_SUPPORTED
# include "pinplay.H"
#endif

#include "sift_writer.h"
#include "bbv_count.h"
#include "../../include/sim_api.h"
#include "pinboost_debug.h"

//#define DEBUG_OUTPUT 1
#define DEBUG_OUTPUT 0

#define LINE_SIZE_BYTES 64
#define MAX_NUM_SYSCALLS 4096
#define MAX_NUM_THREADS 128

KNOB<string> KnobOutputFile(KNOB_MODE_WRITEONCE, "pintool", "o", "trace", "output");
KNOB<UINT64> KnobBlocksize(KNOB_MODE_WRITEONCE, "pintool", "b", "0", "blocksize");
KNOB<UINT64> KnobUseROI(KNOB_MODE_WRITEONCE, "pintool", "roi", "0", "use ROI markers");
KNOB<UINT64> KnobFastForwardTarget(KNOB_MODE_WRITEONCE, "pintool", "f", "0", "instructions to fast forward");
KNOB<UINT64> KnobDetailedTarget(KNOB_MODE_WRITEONCE, "pintool", "d", "0", "instructions to trace in detail (default = all)");
KNOB<UINT64> KnobUseResponseFiles(KNOB_MODE_WRITEONCE, "pintool", "r", "0", "use response files (required for multithreaded applications or when emulating syscalls, default = 0)");
KNOB<UINT64> KnobEmulateSyscalls(KNOB_MODE_WRITEONCE, "pintool", "e", "0", "emulate syscalls (required for multithreaded applications, default = 0)");
KNOB<BOOL>   KnobSendPhysicalAddresses(KNOB_MODE_WRITEONCE, "pintool", "pa", "0", "send logical to physical address mapping");
KNOB<UINT64> KnobFlowControl(KNOB_MODE_WRITEONCE, "pintool", "flow", "1000", "number of instructions to send before syncing up");
KNOB<INT64> KnobSiftAppId(KNOB_MODE_WRITEONCE, "pintool", "s", "0", "sift app id (default = 0)");
KNOB<BOOL> KnobDebug(KNOB_MODE_WRITEONCE, "pintool", "debug", "0", "start debugger on internal exception");

# define KNOB_REPLAY_NAME "replay"
# define KNOB_FAMILY "pintool:sift-recorder"
KNOB_COMMENT pinplay_driver_knob_family(KNOB_FAMILY, "PinPlay SIFT Recorder Knobs");
KNOB<BOOL>KnobReplayer(KNOB_MODE_WRITEONCE, KNOB_FAMILY,
                       KNOB_REPLAY_NAME, "0", "Replay a pinball");
#ifdef PINPLAY_SUPPORTED
PINPLAY_ENGINE pinplay_engine;
#endif /* PINPLAY_SUPPORTED */

INT32 app_id;
INT32 num_threads = 0;
UINT64 blocksize;
UINT64 fast_forward_target = 0;
UINT64 detailed_target = 0;
PIN_LOCK access_memory_lock;
PIN_LOCK new_threadid_lock;
std::deque<ADDRINT> tidptrs;
BOOL any_thread_in_detail = false;
const bool verbose = false;

typedef struct {
   Sift::Writer *output;
   std::deque<ADDRINT> *dyn_address_queue;
   Bbv *bbv;
   UINT64 thread_num;
   ADDRINT bbv_base;
   UINT64 bbv_count;
   ADDRINT bbv_last;
   BOOL bbv_end;
   UINT64 blocknum;
   UINT64 icount;
   UINT64 icount_detailed;
   UINT32 last_syscall_number;
   UINT32 last_syscall_returnval;
   UINT64 flowcontrol_target;
   ADDRINT tid_ptr;
   BOOL in_detail;
   BOOL last_syscall_emulated;
   BOOL running;
   #if defined(TARGET_IA32)
      uint8_t __pad[44];
   #elif defined(TARGET_INTEL64)
      uint8_t __pad[20];
   #endif
} __attribute__((packed)) thread_data_t;
thread_data_t *thread_data;

static_assert((sizeof(thread_data_t) % LINE_SIZE_BYTES) == 0, "Error: Thread data should be a multiple of the line size to prevent false sharing");

#if defined(TARGET_IA32)
   typedef uint32_t syscall_args_t[6];
#elif defined(TARGET_INTEL64)
   typedef uint64_t syscall_args_t[6];
#endif

void findMyAppId();
void openFile(THREADID threadid);
void closeFile(THREADID threadid);

ADDRINT handleMagic(THREADID threadid, ADDRINT gax, ADDRINT gbx, ADDRINT gcx)
{
   if (KnobUseROI.Value())
   {
      if (gax == SIM_CMD_ROI_START)
      {
         int numthreads = 0;
         for (unsigned int i = 0 ; i < MAX_NUM_THREADS ; i++)
         {
            if (thread_data[i].running)
               ++numthreads;
         }
         if (numthreads > 1)
         {
            std::cerr << "[SIFT_RECORDER:" << app_id << "] Error: Threads have been spawned before ROI begin. This behavior is not supported." << std::endl;
            exit(-1);
         }

         if (app_id < 0)
            findMyAppId();

         if (any_thread_in_detail)
         {
            std::cerr << "[SIFT_RECORDER:" << app_id << "] Error: ROI_START seen, but we have already started." << std::endl;
         }
         else
         {
            if (verbose)
               std::cerr << "[SIFT_RECORDER:" << app_id << "] ROI Begin" << std::endl;
         }
         any_thread_in_detail = true;
         for (unsigned int i = 0 ; i < MAX_NUM_THREADS ; i++)
         {
            if (thread_data[i].running && !thread_data[i].in_detail)
               openFile(i);
         }
         PIN_RemoveInstrumentation();
      }
      else if (gax == SIM_CMD_ROI_END)
      {
         // Delete our .appid file
         char filename[1024] = {0};
         sprintf(filename, "%s.app%" PRId32 ".appid", KnobOutputFile.Value().c_str(), app_id);
         unlink(filename);

         if (verbose)
            std::cerr << "[SIFT_RECORDER:" << app_id << "] ROI End" << std::endl;
         any_thread_in_detail = false;
         for (unsigned int i = 0 ; i < MAX_NUM_THREADS ; i++)
         {
            if (thread_data[i].running && thread_data[i].in_detail)
               closeFile(i);
         }

         PIN_RemoveInstrumentation();
      }
      else
      {
         if (thread_data[threadid].running && thread_data[threadid].in_detail)
         {
            uint64_t res = thread_data[threadid].output->Magic(gax, gbx, gcx);
            return res;
         }
      }

      for (unsigned int i = 0 ; i < MAX_NUM_THREADS ; i++)
      {
         thread_data[i].in_detail = any_thread_in_detail;
      }
   }
<commit_msg>[sift_recorder] Send SYS_exit_group on ROI-end to make sure the timing models treat the application and all its threads as finished<commit_after>#include <iostream>
#include <fstream>
#include <stdlib.h>
#include <syscall.h>
#include <vector>
#include <deque>
#include <map>

#include <cstdio>
#include <cassert>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <strings.h>
#include <sys/syscall.h>
#include <sys/file.h>
#include <linux/futex.h>
#include <string.h>
#include <pthread.h>

#define __STDC_FORMAT_MACROS
#include <inttypes.h>

#include "pin.H"
#ifdef PINPLAY_SUPPORTED
# include "pinplay.H"
#endif

#include "sift_writer.h"
#include "bbv_count.h"
#include "../../include/sim_api.h"
#include "pinboost_debug.h"

//#define DEBUG_OUTPUT 1
#define DEBUG_OUTPUT 0

#define LINE_SIZE_BYTES 64
#define MAX_NUM_SYSCALLS 4096
#define MAX_NUM_THREADS 128

KNOB<string> KnobOutputFile(KNOB_MODE_WRITEONCE, "pintool", "o", "trace", "output");
KNOB<UINT64> KnobBlocksize(KNOB_MODE_WRITEONCE, "pintool", "b", "0", "blocksize");
KNOB<UINT64> KnobUseROI(KNOB_MODE_WRITEONCE, "pintool", "roi", "0", "use ROI markers");
KNOB<UINT64> KnobFastForwardTarget(KNOB_MODE_WRITEONCE, "pintool", "f", "0", "instructions to fast forward");
KNOB<UINT64> KnobDetailedTarget(KNOB_MODE_WRITEONCE, "pintool", "d", "0", "instructions to trace in detail (default = all)");
KNOB<UINT64> KnobUseResponseFiles(KNOB_MODE_WRITEONCE, "pintool", "r", "0", "use response files (required for multithreaded applications or when emulating syscalls, default = 0)");
KNOB<UINT64> KnobEmulateSyscalls(KNOB_MODE_WRITEONCE, "pintool", "e", "0", "emulate syscalls (required for multithreaded applications, default = 0)");
KNOB<BOOL>   KnobSendPhysicalAddresses(KNOB_MODE_WRITEONCE, "pintool", "pa", "0", "send logical to physical address mapping");
KNOB<UINT64> KnobFlowControl(KNOB_MODE_WRITEONCE, "pintool", "flow", "1000", "number of instructions to send before syncing up");
KNOB<INT64> KnobSiftAppId(KNOB_MODE_WRITEONCE, "pintool", "s", "0", "sift app id (default = 0)");
KNOB<BOOL> KnobDebug(KNOB_MODE_WRITEONCE, "pintool", "debug", "0", "start debugger on internal exception");

# define KNOB_REPLAY_NAME "replay"
# define KNOB_FAMILY "pintool:sift-recorder"
KNOB_COMMENT pinplay_driver_knob_family(KNOB_FAMILY, "PinPlay SIFT Recorder Knobs");
KNOB<BOOL>KnobReplayer(KNOB_MODE_WRITEONCE, KNOB_FAMILY,
                       KNOB_REPLAY_NAME, "0", "Replay a pinball");
#ifdef PINPLAY_SUPPORTED
PINPLAY_ENGINE pinplay_engine;
#endif /* PINPLAY_SUPPORTED */

INT32 app_id;
INT32 num_threads = 0;
UINT64 blocksize;
UINT64 fast_forward_target = 0;
UINT64 detailed_target = 0;
PIN_LOCK access_memory_lock;
PIN_LOCK new_threadid_lock;
std::deque<ADDRINT> tidptrs;
BOOL any_thread_in_detail = false;
const bool verbose = false;

typedef struct {
   Sift::Writer *output;
   std::deque<ADDRINT> *dyn_address_queue;
   Bbv *bbv;
   UINT64 thread_num;
   ADDRINT bbv_base;
   UINT64 bbv_count;
   ADDRINT bbv_last;
   BOOL bbv_end;
   UINT64 blocknum;
   UINT64 icount;
   UINT64 icount_detailed;
   UINT32 last_syscall_number;
   UINT32 last_syscall_returnval;
   UINT64 flowcontrol_target;
   ADDRINT tid_ptr;
   BOOL in_detail;
   BOOL last_syscall_emulated;
   BOOL running;
   #if defined(TARGET_IA32)
      uint8_t __pad[44];
   #elif defined(TARGET_INTEL64)
      uint8_t __pad[20];
   #endif
} __attribute__((packed)) thread_data_t;
thread_data_t *thread_data;

static_assert((sizeof(thread_data_t) % LINE_SIZE_BYTES) == 0, "Error: Thread data should be a multiple of the line size to prevent false sharing");

#if defined(TARGET_IA32)
   typedef uint32_t syscall_args_t[6];
#elif defined(TARGET_INTEL64)
   typedef uint64_t syscall_args_t[6];
#endif

void findMyAppId();
void openFile(THREADID threadid);
void closeFile(THREADID threadid);

ADDRINT handleMagic(THREADID threadid, ADDRINT gax, ADDRINT gbx, ADDRINT gcx)
{
   if (KnobUseROI.Value())
   {
      if (gax == SIM_CMD_ROI_START)
      {
         int numthreads = 0;
         for (unsigned int i = 0 ; i < MAX_NUM_THREADS ; i++)
         {
            if (thread_data[i].running)
               ++numthreads;
         }
         if (numthreads > 1)
         {
            std::cerr << "[SIFT_RECORDER:" << app_id << "] Error: Threads have been spawned before ROI begin. This behavior is not supported." << std::endl;
            exit(-1);
         }

         if (app_id < 0)
            findMyAppId();

         if (any_thread_in_detail)
         {
            std::cerr << "[SIFT_RECORDER:" << app_id << "] Error: ROI_START seen, but we have already started." << std::endl;
         }
         else
         {
            if (verbose)
               std::cerr << "[SIFT_RECORDER:" << app_id << "] ROI Begin" << std::endl;
         }
         any_thread_in_detail = true;
         for (unsigned int i = 0 ; i < MAX_NUM_THREADS ; i++)
         {
            if (thread_data[i].running && !thread_data[i].in_detail)
               openFile(i);
         }
         PIN_RemoveInstrumentation();
      }
      else if (gax == SIM_CMD_ROI_END)
      {
         if (KnobEmulateSyscalls.Value())
         {
            // Send SYS_exit_group to the simulator to end the application
            syscall_args_t args = {0};
            args[0] = 0; // Assume success
            thread_data[threadid].output->Syscall(SYS_exit_group, (char*)args, sizeof(args));
         }

         // Delete our .appid file
         char filename[1024] = {0};
         sprintf(filename, "%s.app%" PRId32 ".appid", KnobOutputFile.Value().c_str(), app_id);
         unlink(filename);

         if (verbose)
            std::cerr << "[SIFT_RECORDER:" << app_id << "] ROI End" << std::endl;
         any_thread_in_detail = false;
         for (unsigned int i = 0 ; i < MAX_NUM_THREADS ; i++)
         {
            if (thread_data[i].running && thread_data[i].in_detail)
               closeFile(i);
         }

         PIN_RemoveInstrumentation();
      }
      else
      {
         if (thread_data[threadid].running && thread_data[threadid].in_detail)
         {
            uint64_t res = thread_data[threadid].output->Magic(gax, gbx, gcx);
            return res;
         }
      }

      for (unsigned int i = 0 ; i < MAX_NUM_THREADS ; i++)
      {
         thread_data[i].in_detail = any_thread_in_detail;
      }
   }

{'fraction_non_alphanumeric': 0.12420486054477246, 'fraction_numerical': 0.018349372043712283, 'mean_word_length': 3.4159164566078504, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2951094', 'n_tokens_mistral': 4606, 'n_tokens_neox': 4355, 'n_words': 996}