<commit_before>#!/usr/bin/env python
#
# h2olog - A BPF-backed request logging client for the H2O server.
#
# USAGE: $ sudo h2olog -p $(pgrep -o h2o)
#
# Copyright 2019 Fastly, <NAME>

from bcc import BPF, USDT
import getopt, sys

bpf = """
#define MAX_STR_LEN 128
#define MIN(x, y) (((x) < (y)) ? (x) : (y))

/*
 * "req_line_t" represents an individual log line for a given request.
 * This structure is pushed into the BPF ring buffer for later collection
 * by the user-space script (Python part of this script).
 */
struct req_line_t {
    u64 conn_id;
    u64 req_id;
    u32 http_version;
    u64 header_name_len;
    u64 header_value_len;
    char header_name[MAX_STR_LEN];
    char header_value[MAX_STR_LEN];
};
BPF_PERF_OUTPUT(reqbuf);

int trace_receive_req(struct pt_regs *ctx) {
    struct req_line_t line = {};
    uint64_t conn_id, req_id;
    uint32_t http_version;

    bpf_usdt_readarg(1, ctx, &line.conn_id);
    bpf_usdt_readarg(2, ctx, &line.req_id);
    bpf_usdt_readarg(3, ctx, &line.http_version);

    if (reqbuf.perf_submit(ctx, &line, sizeof(line)) < 0)
        bpf_trace_printk("failed to perf_submit\\n");

    return 0;
}

int trace_receive_req_header(struct pt_regs *ctx) {
    struct req_line_t line = {};
    uint64_t conn_id, req_id = 0;
    size_t n_len, v_len;
    void *pos = NULL;

    bpf_usdt_readarg(1, ctx, &line.conn_id);
    bpf_usdt_readarg(2, ctx, &line.req_id);

    // Extract the Header Name
    bpf_usdt_readarg(3, ctx, &pos);
    bpf_usdt_readarg(4, ctx, &n_len);
    line.header_name_len = MIN(MAX_STR_LEN, n_len);
    bpf_probe_read(&line.header_name, line.header_name_len, pos);

    // Extract the Header Value
    bpf_usdt_readarg(5, ctx, &pos);
    bpf_usdt_readarg(6, ctx, &v_len);
    line.header_value_len = MIN(MAX_STR_LEN, v_len);
    bpf_probe_read(&line.header_value, line.header_value_len, pos);

    if (reqbuf.perf_submit(ctx, &line, sizeof(line)) < 0)
        bpf_trace_printk("failed to perf_submit\\n");

    return 0;
}
"""

def print_req_line(line):
    if line.http_version:
        v = "HTTP/%d.%d" % (line.http_version / 256, line.http_version % 256)
        print("%u %u: %s" % (line.conn_id, line.req_id, v))
    else:
        print("%u %u: %s %s" % (line.conn_id, line.req_id, line.header_name, line.header_value))

def handle_req_line(cpu, data, size):
    print_req_line(b["reqbuf"].event(data))

try:
    h2o_pid = 0
    opts, args = getopt.getopt(sys.argv[1:], 'p:')
    for opt, arg in opts:
        if opt == "-p":
            h2o_pid = arg
except getopt.error as msg:
    print(msg)
    sys.exit(2)

if h2o_pid == 0:
    sys.exit("USAGE: h2olog -p PID")

u = USDT(pid=int(h2o_pid))
u.enable_probe(probe="receive_request", fn_name="trace_receive_req")
u.enable_probe(probe="receive_request_header", fn_name="trace_receive_req_header")

b = BPF(text=bpf, usdt_contexts=[u])
b["reqbuf"].open_perf_buffer(handle_req_line)

while 1:
    try:
        b.perf_buffer_poll()
    except KeyboardInterrupt:
        exit()
<commit_msg>h2olog: label the line with varnishlog semantics<commit_after>#!/usr/bin/env python
#
# h2olog - A BPF-backed request logging client for the H2O server.
#
# USAGE: $ sudo h2olog -p $(pgrep -o h2o)
#
# Copyright 2019 Fastly, <NAME>

from bcc import BPF, USDT
import getopt, sys

bpf = """
#define MAX_STR_LEN 128
#define MIN(x, y) (((x) < (y)) ? (x) : (y))

/*
 * "req_line_t" represents an individual log line for a given request.
 * This structure is pushed into the BPF ring buffer for later collection
 * by the user-space script (Python part of this script).
 */
struct req_line_t {
    u64 conn_id;
    u64 req_id;
    u32 http_version;
    u64 header_name_len;
    u64 header_value_len;
    char header_name[MAX_STR_LEN];
    char header_value[MAX_STR_LEN];
};
BPF_PERF_OUTPUT(reqbuf);

int trace_receive_req(struct pt_regs *ctx) {
    struct req_line_t line = {};
    uint64_t conn_id, req_id;
    uint32_t http_version;

    bpf_usdt_readarg(1, ctx, &line.conn_id);
    bpf_usdt_readarg(2, ctx, &line.req_id);
    bpf_usdt_readarg(3, ctx, &line.http_version);

    if (reqbuf.perf_submit(ctx, &line, sizeof(line)) < 0)
        bpf_trace_printk("failed to perf_submit\\n");

    return 0;
}

int trace_receive_req_header(struct pt_regs *ctx) {
    struct req_line_t line = {};
    uint64_t conn_id, req_id = 0;
    size_t n_len, v_len;
    void *pos = NULL;

    bpf_usdt_readarg(1, ctx, &line.conn_id);
    bpf_usdt_readarg(2, ctx, &line.req_id);

    // Extract the Header Name
    bpf_usdt_readarg(3, ctx, &pos);
    bpf_usdt_readarg(4, ctx, &n_len);
    line.header_name_len = MIN(MAX_STR_LEN, n_len);
    bpf_probe_read(&line.header_name, line.header_name_len, pos);

    // Extract the Header Value
    bpf_usdt_readarg(5, ctx, &pos);
    bpf_usdt_readarg(6, ctx, &v_len);
    line.header_value_len = MIN(MAX_STR_LEN, v_len);
    bpf_probe_read(&line.header_value, line.header_value_len, pos);

    if (reqbuf.perf_submit(ctx, &line, sizeof(line)) < 0)
        bpf_trace_printk("failed to perf_submit\\n");

    return 0;
}
"""

def print_req_line(line):
    if line.http_version:
        v = "HTTP/%d.%d" % (line.http_version / 256, line.http_version % 256)
        print("%u %u RxProtocol %s" % (line.conn_id, line.req_id, v))
    else:
        print("%u %u RxHeader   %s %s" % (line.conn_id, line.req_id, line.header_name, line.header_value))

def handle_req_line(cpu, data, size):
    print_req_line(b["reqbuf"].event(data))

try:
    h2o_pid = 0
    opts, args = getopt.getopt(sys.argv[1:], 'p:')
    for opt, arg in opts:
        if opt == "-p":
            h2o_pid = arg
except getopt.error as msg:
    print(msg)
    sys.exit(2)

if h2o_pid == 0:
    sys.exit("USAGE: h2olog -p PID")

u = USDT(pid=int(h2o_pid))
u.enable_probe(probe="receive_request", fn_name="trace_receive_req")
u.enable_probe(probe="receive_request_header", fn_name="trace_receive_req_header")

b = BPF(text=bpf, usdt_contexts=[u])
b["reqbuf"].open_perf_buffer(handle_req_line)

while 1:
    try:
        b.perf_buffer_poll()
    except KeyboardInterrupt:
        exit()

{'fraction_non_alphanumeric': 0.1329708353929807, 'fraction_numerical': 0.018948755972977425, 'mean_word_length': 3.6513409961685825, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5024409', 'n_tokens_mistral': 2492, 'n_tokens_neox': 2385, 'n_words': 534}