<commit_before>#include <libopticon/log.h>
#include <libopticon/var_dump.h>
#include <ctype.h>
#include <time.h>
#include <errno.h>
#include <string.h>
#include <sys/sysctl.h>

var *runprobe_distro (probe *self) {
    var *res = var_alloc();
    char *distro = NULL;
    char *c;
    FILE *F;
    char buf[256];
    F = popen ("/usr/sbin/system_profiler SPSoftwareDataType","r");
    if (! F) return res;
    while (! feof (F)) {
        *buf = 0;
        fgets (buf, 255, F);
        if (strncmp (buf, "      System Version: ", 22) == 0) {
            distro = buf+22;
            c = strchr (distro, '\n');
            if (c) *c = 0;
            log_debug ("(probe_distro) snarked distro: %s", distro);
            break;
        }
    }
    pclose (F);
    if (distro) {
        var *res_os = var_get_dict_forkey (res, "os");
        var_set_str_forkey (res_os, "distro", distro);
    }
    return res;
}

/** Structure for keeping track of top records */
typedef struct toprec_s {
    uint32_t pid;
    char cmd[32];
    double pcpu;
    uint32_t sizekb;
    char user[32];
} toprec;

/** Structure for keeping all the data for a single top round */
typedef struct topinfo_s {
    toprec records[16];
    double cpu_usage;
    uint32_t memsizekb;
    double load1;
    double load5;
    double load15;
    int numproc;
    int numrun;
    int numstuck;
    uint32_t netin_bytes;
    uint32_t netout_bytes;
    int netin_pkt;
    int netout_pkt;
    int iops_read;
    int iops_write;
    int kmem_avail;
    int kmem_free;
} topinfo;

/** Data that will be collected by the topthread. TOP[1] is the copy that is
  * actively being written to, only after completion will it be copied to TOP[0],
  * which should be considered the canonical copy */
topinfo TOP[2];

typedef enum { TOP_HDR, TOP_BODY } topstate;

/** Utility function for copying a substring of a bit of space-formatted text */
static void cpystr (char *into, const char *ptr, int sz) {
    int eos = 0;
    for (int i=0; i<sz; ++i) {
        if (ptr[i] == '\n') break;
        into[i] = ptr[i];
        if (into[i] != ' ') eos = i+1;
    }
    into[eos] = 0;
}

/** Convert size-information in top output (number + B/M/G/K) to a number in
  * kilobytes.
  */
static uint32_t sz2kb (const char *str) {
    int ival = atoi (str);
    if (! ival) return ival;
    
    const char *c = str;
    while (isdigit (*c)) c++;
    if (*c == 'B') ival /= 1024;
    else if (*c == 'M') ival *= 1024;
    else if (*c == 'G') ival *= (1024*1024);
    return ival;
}

static uint32_t sz2b (const char *str) {
    int ival = atoi (str);
    if (! ival) return ival;
    
    const char *c = str;
    while (isdigit (*c)) c++;
    if (*c == 'K') ival *= 1024;
    else if (*c == 'M') ival *= (1024*1024);
    else if (*c == 'G') ival *= (1024*1024*1024);
    return ival;
}

/** Background thread that spawns /usr/bin/top and lets it run, parsing the output
  * as it comes along. */
void run_top (thread *me) {
    char buf[1024];
    topstate st = TOP_HDR;
    int count = 0;
    int offs_cmd, offs_cpu, offs_mem, offs_user;
    memset (TOP, 0, 2*sizeof(topinfo));
    FILE *f = popen ("/usr/bin/top -l 0 -o cpu -n 16 -c n -s 30", "r");
    if (! f) {
        log_error ("Could not open top probe");
        return;
    }
    while (1) {
        if (feof (f)) {
            log_info ("Reopening top");
            pclose (f);
            f = popen ("/usr/bin/top -l 0 -o cpu -n 16 -c n -s 30", "r");
        }
        
        buf[0] = 0;
        fgets (buf, 1023, f);
        
        if (buf[0] == 'P' && buf[1] == 'I' && buf[2] == 'D') {
            st = TOP_BODY;
            memset (TOP[1].records, 0, 16 *sizeof (toprec));
            offs_cmd = strstr (buf, "COMMAND ") - buf;
            offs_cpu = strstr (buf, "%CPU ") - buf;
            offs_mem = strstr (buf, "MEM ") -buf;
            offs_user = strstr (buf, "USER ") - buf;
            continue;
        }
        if (st == TOP_BODY) {
            if (strlen (buf) < (offs_user+2)) continue;
            if (atof (buf+offs_cpu) < 0.0001) {
                memcpy (TOP, TOP+1, sizeof (topinfo));
                memset (TOP+1, 0, sizeof(topinfo));
                st = TOP_HDR;
                count = 0;
                continue;
            }
            TOP[1].records[count].pid = atoi (buf);
            cpystr (TOP[1].records[count].cmd, buf+offs_cmd, 15);
            TOP[1].records[count].pcpu = atof (buf+offs_cpu);
            TOP[1].records[count].sizekb = sz2kb (buf+offs_mem);
            cpystr (TOP[1].records[count].user, buf+offs_user, 14);
            count++;
            if (count == 14) {
                memcpy (TOP, TOP+1, sizeof (topinfo));
                memset (TOP+1, 0, sizeof(topinfo));
                st = TOP_HDR;
                count = 0;
            }
        }
        else {
            if (memcmp (buf, "Load Avg", 8) == 0) {
                TOP[1].load1 = atof (buf + 10);
                char *next = strchr (buf, ',');
                if (next) {
                    TOP[1].load5 = atof (next+2);
                    next = strchr (next+2, ',');
                    if (next) {
                        TOP[1].load15 = atof (next+2);
                    }
                }
            }<commit_msg>fix %mem breakage in osx by not measuring it..<commit_after>#include <libopticon/log.h>
#include <libopticon/var_dump.h>
#include <ctype.h>
#include <time.h>
#include <errno.h>
#include <string.h>
#include <sys/sysctl.h>

var *runprobe_distro (probe *self) {
    var *res = var_alloc();
    char *distro = NULL;
    char *c;
    FILE *F;
    char buf[256];
    F = popen ("/usr/sbin/system_profiler SPSoftwareDataType","r");
    if (! F) return res;
    while (! feof (F)) {
        *buf = 0;
        fgets (buf, 255, F);
        if (strncmp (buf, "      System Version: ", 22) == 0) {
            distro = buf+22;
            c = strchr (distro, '\n');
            if (c) *c = 0;
            log_debug ("(probe_distro) snarked distro: %s", distro);
            break;
        }
    }
    pclose (F);
    if (distro) {
        var *res_os = var_get_dict_forkey (res, "os");
        var_set_str_forkey (res_os, "distro", distro);
    }
    return res;
}

/** Structure for keeping track of top records */
typedef struct toprec_s {
    uint32_t pid;
    char cmd[32];
    double pcpu;
    uint32_t sizekb;
    char user[32];
} toprec;

/** Structure for keeping all the data for a single top round */
typedef struct topinfo_s {
    toprec records[16];
    double cpu_usage;
    uint32_t memsizekb;
    double load1;
    double load5;
    double load15;
    int numproc;
    int numrun;
    int numstuck;
    uint32_t netin_bytes;
    uint32_t netout_bytes;
    int netin_pkt;
    int netout_pkt;
    int iops_read;
    int iops_write;
    int kmem_avail;
    int kmem_free;
} topinfo;

/** Data that will be collected by the topthread. TOP[1] is the copy that is
  * actively being written to, only after completion will it be copied to TOP[0],
  * which should be considered the canonical copy */
topinfo TOP[2];

typedef enum { TOP_HDR, TOP_BODY } topstate;

/** Utility function for copying a substring of a bit of space-formatted text */
static void cpystr (char *into, const char *ptr, int sz) {
    int eos = 0;
    for (int i=0; i<sz; ++i) {
        if (ptr[i] == '\n') break;
        into[i] = ptr[i];
        if (into[i] != ' ') eos = i+1;
    }
    into[eos] = 0;
}

/** Convert size-information in top output (number + B/M/G/K) to a number in
  * kilobytes.
  */
static uint32_t sz2kb (const char *str) {
    int ival = atoi (str);
    if (! ival) return ival;
    
    const char *c = str;
    while (isdigit (*c)) c++;
    if (*c == 'B') ival /= 1024;
    else if (*c == 'M') ival *= 1024;
    else if (*c == 'G') ival *= (1024*1024);
    return ival;
}

static uint32_t sz2b (const char *str) {
    int ival = atoi (str);
    if (! ival) return ival;
    
    const char *c = str;
    while (isdigit (*c)) c++;
    if (*c == 'K') ival *= 1024;
    else if (*c == 'M') ival *= (1024*1024);
    else if (*c == 'G') ival *= (1024*1024*1024);
    return ival;
}

/** Background thread that spawns /usr/bin/top and lets it run, parsing the output
  * as it comes along. */
void run_top (thread *me) {
    char buf[1024];
    topstate st = TOP_HDR;
    int count = 0;
    int offs_cmd, offs_cpu, offs_mem, offs_user;
    memset (TOP, 0, 2*sizeof(topinfo));
    FILE *f = popen ("/usr/bin/top -l 0 -o cpu -n 16 -c d -s 30", "r");
    if (! f) {
        log_error ("Could not open top probe");
        return;
    }
    while (1) {
        if (feof (f)) {
            log_info ("Reopening top");
            pclose (f);
            f = popen ("/usr/bin/top -l 0 -o cpu -n 16 -c d -s 30", "r");
        }
        
        buf[0] = 0;
        fgets (buf, 1023, f);
        
        if (buf[0] == 'P' && buf[1] == 'I' && buf[2] == 'D') {
            st = TOP_BODY;
            memset (TOP[1].records, 0, 16 *sizeof (toprec));
            offs_cmd = strstr (buf, "COMMAND ") - buf;
            offs_cpu = strstr (buf, "%CPU ") - buf;
            offs_mem = strstr (buf, "MEM ") -buf;
            offs_user = strstr (buf, "USER ") - buf;
            continue;
        }
        if (st == TOP_BODY) {
            if (strlen (buf) < (offs_user+2)) continue;
            if (atof (buf+offs_cpu) < 0.0001) {
                memcpy (TOP, TOP+1, sizeof (topinfo));
                memset (TOP+1, 0, sizeof(topinfo));
                st = TOP_HDR;
                count = 0;
                continue;
            }
            TOP[1].records[count].pid = atoi (buf);
            cpystr (TOP[1].records[count].cmd, buf+offs_cmd, 15);
            TOP[1].records[count].pcpu = atof (buf+offs_cpu);
            TOP[1].records[count].sizekb = 0; /* sz2kb (buf+offs_mem); */
            cpystr (TOP[1].records[count].user, buf+offs_user, 14);
            count++;
            if (count == 14) {
                memcpy (TOP, TOP+1, sizeof (topinfo));
                memset (TOP+1, 0, sizeof(topinfo));
                st = TOP_HDR;
                count = 0;
            }
        }
        else {
            if (memcmp (buf, "Load Avg", 8) == 0) {
                TOP[1].load1 = atof (buf + 10);
                char *next = strchr (buf, ',');
                if (next) {
                    TOP[1].load5 = atof (next+2);
                    next = strchr (next+2, ',');
                    if (next) {
                        TOP[1].load15 = atof (next+2);
                    }
                }
            }
{'fraction_non_alphanumeric': 0.1479727169382342, 'fraction_numerical': 0.029840848806366047, 'mean_word_length': 1.7958156779661016, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7200699', 'n_tokens_mistral': 4004, 'n_tokens_neox': 3701, 'n_words': 1107}