<commit_before># Virtual memory analysis scripts.
# Developed 2012-2014 by <NAME>, <EMAIL>
# Copyright (c) 2012-2014 <NAME> and University of Washington

from util.pjh_utils import *
import conf.system_conf as sysconf
import datetime
import os
import shlex
import shutil
import signal
import subprocess
import sys
import time

# Linux "perf" tool:
# On verbena, there are just two "generic" hw performance counters, so
# when more than two events are selected here, the events are multiplexed
# over the counters during the "perf record" run. So, for example, if
# four events are active here, they will each only be counted for about
# 50% of the record's execution.
# To see the full list of available events, run "perf list".
#PERF_TRACE_DEFAULT_ON = False
PERF_TRACE_DEFAULT_ON = True
  # can be overridden by individual trace_on() callers.
PERF_EVENTS = [
	'dTLB-loads',
	'dTLB-load-misses',
	'dTLB-stores',
	'dTLB-store-misses',
	#'dTLB-prefetches',
	#'dTLB-prefetch-misses',
#	'iTLB-loads',
#	'iTLB-load-misses',
	#'L1-dcache-loads',
	#'L1-dcache-load-misses',
	#'L1-dcache-stores',
	#'L1-dcache-store-misses',
	#'L1-dcache-prefetches',
	#'L1-dcache-prefetch-misses',
	#'L1-icache-loads',
	#'L1-icache-load-misses',
	#'L1-icache-prefetches',
	#'L1-icache-prefetch-misses',
	#'LLC-loads',
	#'LLC-load-misses',
	#'LLC-stores',
	#'LLC-store-misses',
	#'LLC-prefetches',
	#'LLC-prefetch-misses',
	]
PERF_FREQ = 1000
  # By using the -F flag to perf record, the number of events counted
  # before generating a sample interrupt will be dynamically tuned to
  # sample approximately F times every second (so higher PERF_FREQ == 
  # more samples with smaller periods). For example, set to 1000
  # to get 1000 samples/s, or about 1 sample every millisecond. (going
  # lower than 1 ms is probably a poor idea).
  #   I verified that this is right - for a small graph500 run, -F 100
  #   captures 1473 dTLB-load samples, whereas -F 1000 captures about
  #   10x as many, 14575 samples.
  # I examined the plot output for Graph500, and using -F 1000 results
  # in a less-jagged plot than using -F 100, with no other apparent
  # side effects other than 10x more samples. The general calculated
  # miss rates look pretty much the same, so a less-jagged plot seems
  # to indicate a more-accurate plot. I guess I'll stick with -F 1000
  # for now, unless it seems to cause sample periods to become too
  # small, which could potentially be problematic...
PERF_CMD  = "{}/perf".format(sysconf.PERF_DIR)
PERF_EVENTS_STR = ','.join(PERF_EVENTS)
PERF_RECORD_OPTS = "record -vvv -a -i -F {} -e {}".format(
		PERF_FREQ, PERF_EVENTS_STR)
  # Use -a and don't specify a particular command: perf record will
  #   just run until it gets ctrl-c (SIGINT).
  # -i: child tasks don't inherit counters. I'm not sure if this applies
  #   to hw events anyway, but do it to be sure; I should be able to account
  #   for parent/child stuff in my own scripts.
PERF_STDOUT     = 'perf.stdout'
PERF_STDERR     = 'perf.stderr'
PERF_DATA       = 'perf.data'
PERF_DUMPFILE   = 'perf.dump'
PERF_REPORTFILE = 'perf.report'

# Tracing directories and parameters: 1 for true / enable, 0 for false /
# disable.
# Comments:
#   - With sched_switch tracing enabled, the trace buffer will fill up
#     very quickly - even with 678 MB of buffer per-CPU, I couldn't
#     complete a full Cassandra start+load+run.
#   - Does enabling userstack_syms also make the trace buffer fill
#     up more quickly, or not?
#   - clock: use 'local' for a nanosecond-granularity clock, which should
#     approximately sync up with events in a 'perf record' trace. Never
#     use 'global' or your system will become unresponsive! 'x86-tsc'
#     works, but doesn't seem to be too helpful at the moment.
tracing_dir = "{}/tracing".format(sysconf.sys_debug_dir)
#trace_buf_mb_per_core = 512       # per-CPU!
trace_buf_mb_per_core = int(sysconf.suggested_tracebuf_size_kb / 1024)
trace_clock          = 'local'
trace_userstacks     = 0
trace_userstack_syms = 0
  # tracing/options/sym-userobj: attempt to look up stack fns. Note
  # that this flag may not work if other tracing/options/ fields (besides
  # userstacktrace) are enabled.
trace_vma_events     = 1
trace_pte_events     = 0
trace_sched_switch   = 0
  # Note: depending on how kernel is built, may cause deadlock during
  # userstacktrace collection??
trace_sched_fork     = 0
trace_sys_mprotect   = 0
all_cpus_prog = "{}/test-programs/all_cpus".format(sysconf.apps_dir)

tracefilename   = 'trace-events-full'

'''
  ...
'''
class traceinfo:
	tag = 'traceinfo'

	# Members:
	appname = None
	tracing_on = None
	trace_outputdir = None
	trace_on_perf_too = None   # was perf turned on via trace_on()?
	perf_outputdir = None
	perf_tracing_on = None    # is perf trace process active?
	perf_p = None             # Popen object for perf process
	perf_stdout = None
	perf_stderr = None
	pdata_fname = None
	trace_on_time = None
	perf_on_time = None

	def __init__(self, appname):
		tag = "{}.__init__".format(self.tag)

		if not appname:
			print_error_exit(tag, ("missing argument: appname={}").format(
				appname))
		self.appname = appname
		self.tracing_on = False
		self.trace_outputdir = None
		self.perf_outputdir = None
		self.perf_tracing_on = False
		self.trace_on_perf_too = False
		self.perf_p = None
		self.perf_stdout = None
		self.perf_stderr = None
		self.pdata_fname = None
		self.trace_on_time = None
		self.perf_on_time = None

		return

	# This method performs the following steps:
	#   - Set the kernel tracing options and save them in the outputdir
	#   - Turn on kernel tracing
	#   - Run a small program to ensure that all CPU tracing buffers are active
	# Returns: True on success, False on error.
	def trace_on(self, outputdir, descr, use_perf=PERF_TRACE_DEFAULT_ON,
			targetpid=None):
		tag = "{}.trace_on".format(self.tag)

		if self.tracing_on:
			print_error(tag, ("tracing is already activated!"))
			return False

		success = True
		tdir = tracing_dir

		if not os.path.exists(outputdir):
			os.makedirs(outputdir)

		if trace_userstacks != 0 and trace_pte_events != 0:
			print_error_exit(tag, ("can't set both trace_userstacks={} "
				"and trace_pte_events={} - otherwise, when collecting "
				"userstack entries for 'do_page_fault' code path that "
				"contains pte trace events, you may invoke further "
				"page faults, causing recursive trace events or "
				"whatever and leading to deadlock!").format(
				trace_userstacks, trace_pte_events))

		# Set kernel tracing options:
		options = []
		options.append(("echo 0 > {}/tracing_on").format(tdir))
		options.append(("echo {} > {}/buffer_size_kb").format(
			int(trace_buf_mb_per_core*1024), tdir))
		options.append(("echo {} > {}/trace_clock").format(trace_clock, tdir))
		options.append(("echo 0 > {}/options/overwrite").format(tdir))
		options.append(("echo {} > {}/options/sym-userobj").format(
			trace_userstack_syms, tdir))
		options.append(("echo {} > {}/options/userstacktrace").format(
			trace_userstacks, tdir))
		options.append(("echo {} > {}/events/mmap/enable").format(
			trace_vma_events, tdir))
		options.append(("echo {} > {}/events/pte/enable").format(
			trace_pte_events, tdir))
		options.append(("echo {} > {}/events/sched/sched_switch/"
			"enable").format(trace_sched_switch, tdir))
		options.append(
			("echo {} > {}/events/sched/sched_process_fork/enable").format(
			trace_sched_fork, tdir))
		options.append(
			("echo {} > {}/events/syscalls/sys_enter_mprotect/enable").format(<commit_msg>Added flag to enable rss events; tested and works, but not analyzing these events yet.<commit_after># Virtual memory analysis scripts.
# Developed 2012-2014 by <NAME>, <EMAIL>
# Copyright (c) 2012-2014 <NAME> and University of Washington

from util.pjh_utils import *
import conf.system_conf as sysconf
import datetime
import os
import shlex
import shutil
import signal
import subprocess
import sys
import time

# Linux "perf" tool:
# On verbena, there are just two "generic" hw performance counters, so
# when more than two events are selected here, the events are multiplexed
# over the counters during the "perf record" run. So, for example, if
# four events are active here, they will each only be counted for about
# 50% of the record's execution.
# To see the full list of available events, run "perf list".
#PERF_TRACE_DEFAULT_ON = False
PERF_TRACE_DEFAULT_ON = True
  # can be overridden by individual trace_on() callers.
PERF_EVENTS = [
	'dTLB-loads',
	'dTLB-load-misses',
	'dTLB-stores',
	'dTLB-store-misses',
	#'dTLB-prefetches',
	#'dTLB-prefetch-misses',
#	'iTLB-loads',
#	'iTLB-load-misses',
	#'L1-dcache-loads',
	#'L1-dcache-load-misses',
	#'L1-dcache-stores',
	#'L1-dcache-store-misses',
	#'L1-dcache-prefetches',
	#'L1-dcache-prefetch-misses',
	#'L1-icache-loads',
	#'L1-icache-load-misses',
	#'L1-icache-prefetches',
	#'L1-icache-prefetch-misses',
	#'LLC-loads',
	#'LLC-load-misses',
	#'LLC-stores',
	#'LLC-store-misses',
	#'LLC-prefetches',
	#'LLC-prefetch-misses',
	]
PERF_FREQ = 1000
  # By using the -F flag to perf record, the number of events counted
  # before generating a sample interrupt will be dynamically tuned to
  # sample approximately F times every second (so higher PERF_FREQ == 
  # more samples with smaller periods). For example, set to 1000
  # to get 1000 samples/s, or about 1 sample every millisecond. (going
  # lower than 1 ms is probably a poor idea).
  #   I verified that this is right - for a small graph500 run, -F 100
  #   captures 1473 dTLB-load samples, whereas -F 1000 captures about
  #   10x as many, 14575 samples.
  # I examined the plot output for Graph500, and using -F 1000 results
  # in a less-jagged plot than using -F 100, with no other apparent
  # side effects other than 10x more samples. The general calculated
  # miss rates look pretty much the same, so a less-jagged plot seems
  # to indicate a more-accurate plot. I guess I'll stick with -F 1000
  # for now, unless it seems to cause sample periods to become too
  # small, which could potentially be problematic...
PERF_CMD  = "{}/perf".format(sysconf.PERF_DIR)
PERF_EVENTS_STR = ','.join(PERF_EVENTS)
PERF_RECORD_OPTS = "record -vvv -a -i -F {} -e {}".format(
		PERF_FREQ, PERF_EVENTS_STR)
  # Use -a and don't specify a particular command: perf record will
  #   just run until it gets ctrl-c (SIGINT).
  # -i: child tasks don't inherit counters. I'm not sure if this applies
  #   to hw events anyway, but do it to be sure; I should be able to account
  #   for parent/child stuff in my own scripts.
PERF_STDOUT     = 'perf.stdout'
PERF_STDERR     = 'perf.stderr'
PERF_DATA       = 'perf.data'
PERF_DUMPFILE   = 'perf.dump'
PERF_REPORTFILE = 'perf.report'

# Tracing directories and parameters: 1 for true / enable, 0 for false /
# disable.
# Comments:
#   - With sched_switch tracing enabled, the trace buffer will fill up
#     very quickly - even with 678 MB of buffer per-CPU, I couldn't
#     complete a full Cassandra start+load+run.
#   - Does enabling userstack_syms also make the trace buffer fill
#     up more quickly, or not?
#   - clock: use 'local' for a nanosecond-granularity clock, which should
#     approximately sync up with events in a 'perf record' trace. Never
#     use 'global' or your system will become unresponsive! 'x86-tsc'
#     works, but doesn't seem to be too helpful at the moment.
tracing_dir = "{}/tracing".format(sysconf.sys_debug_dir)
#trace_buf_mb_per_core = 512       # per-CPU!
trace_buf_mb_per_core = int(sysconf.suggested_tracebuf_size_kb / 1024)
trace_clock          = 'local'
trace_userstacks     = 0
trace_userstack_syms = 0
  # tracing/options/sym-userobj: attempt to look up stack fns. Note
  # that this flag may not work if other tracing/options/ fields (besides
  # userstacktrace) are enabled.
trace_vma_events     = 1
trace_pte_events     = 0
trace_rss_events     = 0
trace_sched_switch   = 0
  # Note: depending on how kernel is built, may cause deadlock during
  # userstacktrace collection??
trace_sched_fork     = 0
trace_sys_mprotect   = 0
all_cpus_prog = "{}/test-programs/all_cpus".format(sysconf.apps_dir)

tracefilename   = 'trace-events-full'

'''
  ...
'''
class traceinfo:
	tag = 'traceinfo'

	# Members:
	appname = None
	tracing_on = None
	trace_outputdir = None
	trace_on_perf_too = None   # was perf turned on via trace_on()?
	perf_outputdir = None
	perf_tracing_on = None    # is perf trace process active?
	perf_p = None             # Popen object for perf process
	perf_stdout = None
	perf_stderr = None
	pdata_fname = None
	trace_on_time = None
	perf_on_time = None

	def __init__(self, appname):
		tag = "{}.__init__".format(self.tag)

		if not appname:
			print_error_exit(tag, ("missing argument: appname={}").format(
				appname))
		self.appname = appname
		self.tracing_on = False
		self.trace_outputdir = None
		self.perf_outputdir = None
		self.perf_tracing_on = False
		self.trace_on_perf_too = False
		self.perf_p = None
		self.perf_stdout = None
		self.perf_stderr = None
		self.pdata_fname = None
		self.trace_on_time = None
		self.perf_on_time = None

		return

	# This method performs the following steps:
	#   - Set the kernel tracing options and save them in the outputdir
	#   - Turn on kernel tracing
	#   - Run a small program to ensure that all CPU tracing buffers are active
	# Returns: True on success, False on error.
	def trace_on(self, outputdir, descr, use_perf=PERF_TRACE_DEFAULT_ON,
			targetpid=None):
		tag = "{}.trace_on".format(self.tag)

		if self.tracing_on:
			print_error(tag, ("tracing is already activated!"))
			return False

		success = True
		tdir = tracing_dir

		if not os.path.exists(outputdir):
			os.makedirs(outputdir)

		if (trace_userstacks != 0 and 
            (trace_pte_events != 0 or trace_rss_events != 0)):
			print_error_exit(tag, ("can't set both trace_userstacks={} "
				"and trace_pte_events={} or trace_rss_events={} - "
                "otherwise, when collecting "
				"userstack entries for 'do_page_fault' code path that "
				"contains pte trace events, you may invoke further "
				"page faults, causing recursive trace events or "
				"whatever and leading to deadlock!").format(
				trace_userstacks, trace_pte_events, trace_rss_events))

		# Set kernel tracing options:
		options = []
		options.append(("echo 0 > {}/tracing_on").format(tdir))
		options.append(("echo {} > {}/buffer_size_kb").format(
			int(trace_buf_mb_per_core*1024), tdir))
		options.append(("echo {} > {}/trace_clock").format(trace_clock, tdir))
		options.append(("echo 0 > {}/options/overwrite").format(tdir))
		options.append(("echo {} > {}/options/sym-userobj").format(
			trace_userstack_syms, tdir))
		options.append(("echo {} > {}/options/userstacktrace").format(
			trace_userstacks, tdir))
		options.append(("echo {} > {}/events/mmap/enable").format(
			trace_vma_events, tdir))
		options.append(("echo {} > {}/events/pte/enable").format(
			trace_pte_events, tdir))
		options.append(("echo {} > {}/events/rss/enable").format(
			trace_rss_events, tdir))
		options.append(("echo {} > {}/events/sched/sched_switch/"
			"enable").format(trace_sched_switch, tdir))
		options.append(
			("echo {} > {}/events/sched/sched_process_fork/enable").format(
			trace_sched_fork, tdir))
		options.append(
			("echo {} > {}/events/syscalls/sys_enter_mprotect/enable").format(
{'fraction_non_alphanumeric': 0.10975609756097561, 'fraction_numerical': 0.014216773183774619, 'mean_word_length': 3.8651649746192893, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7453831', 'n_tokens_mistral': 5579, 'n_tokens_neox': 5094, 'n_words': 1593}