<commit_before>#!/usr/bin/env python

"""Benchmark loading the output of "lsof -F fnaptd0" from a big system

Usage:
  benchmark_ipcmap.py <FILE>

FILE is a file containing the output of "lsof -F fnaptd0".

This program will parse that output and make an IPC map of the process that has
the highest number of entries in that file.
"""

import time

import docopt
import testutils
import px_file


def get_most_common_pid(files):
    counts = {}
    for file in files:
        pid = file.pid
        if pid not in counts:
            counts[pid] = 0
        counts[pid] += 1
    return sorted(counts.keys(), key=lambda pid: counts[pid])[-1]


def main(args):
    t0 = time.time()
    files = None
    with open(args['<FILE>'], "r") as lsof_output:
        files = px_file.lsof_to_files(lsof_output.read())
    t1 = time.time()
    dt = t1 - t0
    print("Parsing lsof output: {}s".format(dt))

    pid = get_most_common_pid(files)
    print("Most popular PID: {}".format(pid))

    t0 = time.time()
    testutils.create_ipc_map(pid, files)
    t1 = time.time()
    dt = t1 - t0
    print("Creating PID {} IPC map: {}s".format(pid, dt))


if __name__ == "__main__":
    main(docopt.docopt(__doc__))<commit_msg>Perf Benchmark: Run for 30s, make simple statistics<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Benchmark loading the output of "lsof -F fnaptd0" from a big system

Usage:
  benchmark_ipcmap.py <FILE>

FILE is a file containing the output of "lsof -F fnaptd0".

This program will parse that output and make an IPC map of the process that has
the highest number of entries in that file.
"""

import time

import docopt
import testutils
import px_file

# For how long should we do the benchmarking run (in seconds)
DURATION_S = 30


def get_most_common_pid(files):
    counts = {}
    for file in files:
        pid = file.pid
        if pid not in counts:
            counts[pid] = 0
        counts[pid] += 1
    return sorted(counts.keys(), key=lambda pid: counts[pid])[-1]


def get_timings(file, pid):
    """
    Loads file and creates an IPC map for PID.

    Returns timings in a tuple (load, mapping) in seconds.
    """
    t0 = time.time()
    files = None
    with open(file, "r") as lsof_output:
        files = px_file.lsof_to_files(lsof_output.read())
    t1 = time.time()
    dt_load = t1 - t0

    t0 = time.time()
    testutils.create_ipc_map(pid, files)
    t1 = time.time()
    dt_mapping = t1 - t0

    return (dt_load, dt_mapping)


def print_statistics(name, values):
    lowest = min(values)
    highest = max(values)
    middle = (lowest + highest) / 2
    radius = (highest - lowest) / 2
    print("{} is {:.2f}s±{:.2f}s".format(name, middle, radius))


def main(args):
    lsof_file = args['<FILE>']

    print("Finding most popular PID...")
    files = None
    with open(lsof_file, "r") as lsof_output:
        files = px_file.lsof_to_files(lsof_output.read())
    pid = get_most_common_pid(files)
    print("Most popular PID: {}".format(pid))

    end = time.time() + DURATION_S
    lap_number = 0
    load_times = []
    mapping_times = []
    while time.time() < end:
        lap_number += 1
        print("Lap {}, {:.0f}s left...".format(lap_number, end - time.time()))
        load_time, mapping_time = get_timings(lsof_file, pid)
        load_times.append(load_time)
        mapping_times.append(mapping_time)

    print_statistics("Loading time", load_times)
    print_statistics("Mapping time", mapping_times)


if __name__ == "__main__":
    main(docopt.docopt(__doc__))
{'fraction_non_alphanumeric': 0.12231030577576443, 'fraction_numerical': 0.010758776896942242, 'mean_word_length': 2.9696629213483146, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1965474', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1241, 'n_words': 359}