<commit_before><commit_msg>added performance extractor to read in performance data<commit_after>#! /usr/bin/env python2.7

import os
import os.path
import matplotlib.pyplot as plt
import numpy as np
d=os.path.join('.', 'performance')
dirs = [o for o in os.listdir(d) if os.path.isdir(os.path.join(d,o))]

#ok, now we have a list of data directories, open each and look for the text files
data = {}
step_data = {}
for directory in dirs:
	data[directory] = {}
	step_data[directory] = {}

	thedir = os.path.join(d,directory)

	#get text files
	files = [os.path.join(thedir, o) for o in os.listdir(thedir) if os.path.isfile(os.path.join(thedir,o)) and o.endswith('.txt')]
	for filename in files:
		with open(filename) as file:
			lines = [l.strip() for l in file.readlines() if l.strip()]
		if 'cpu' in filename:
			#format is num_threads, num_odes, runtime (ms)
			basename = 'CPU'
			if '_co_output' in filename:
				basename += ' (cache-opt)'
			for line in lines:
				vals = [float(f) for f in line.split(',')]
				name = basename + ' {:d} thread(s)'.format(int(vals[0]))
				if not name in data[directory]:
					data[directory][name] = []
				data[directory][name].append(vals[2] / vals[1])
		else:
			basename = 'GPU'
			if '_co_' in filename:
				basename += '(cache-opt, '
			if '_sm_' in filename:
				if not '_co_' in filename:
					basename += '('
				basename == 'shared-mem)'
			mydict = step_data if '_steps' in filename else data
			#format is num_odes, runtime (ms)
			for line in lines:
				vals = [float(f) for f in line.split(',')]
				if not name in mydict[directory]:
					mydict[directory][name] = []
				mydict[directory][name].append(vals[1] / vals[0])

for thedir in data:
	for name in data[thedir]:
		avg = np.mean(data[thedir][name])
		dev = np.std(data[thedir][name])
		data[thedir][name] = (avg, dev)

for thedir in step_data:
	for name in step_data[thedir]:
		avg = np.mean(data[thedir][name])
		dev = np.std(data[thedir][name])
		data[thedir][name] = (avg, dev)

print data
print step_data
{'fraction_non_alphanumeric': 0.1527777777777778, 'fraction_numerical': 0.003472222222222222, 'mean_word_length': 3.790973871733967, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2726019', 'n_tokens_mistral': 829, 'n_tokens_neox': 748, 'n_words': 215}