<commit_before><commit_msg>Format.py to render benchmarks as html.<commit_after>from plot import *
from oprofile import *
from profiles import *
import os
import time
import StringIO

class dbench():
    log_file = 'bench_log.txt'
    hostname = 'newton'
    www_dir = '/var/www/code.rethinkdb.com/htdocs/'
    prof_dir = 'prof_data'

    def __init__(self, dir):
        rundirs = os.listdir(dir)
        rundirs.remove(self.log_file)
        rundirs.sort(key = lambda x: int(x))
        self.runs = []
        for rundir in rundirs:
            self.runs.append(self.run(dir + '/' + rundir))

    class run():
        iostat_path     = '/iostat/output.txt'
        vmstat_path     = '/vmstat/output.txt'
        latency_path    = '/client/latency.txt'
        qps_path        = '/client/qps.txt'
        rdbstat_path    = '/rdbstat/output.txt'
        oprofile_path   = '/oprofile/oprof.out.rethinkdb'

        def __init__(self, dir):
            self.timeseries = [IOStat().read(dir + self.iostat_path),
                               VMStat().read(dir + self.vmstat_path),
                               Latency().read(dir + self.latency_path),
                               QPS().read(dir + self.qps_path),
                               RDBStats().read(dir + self.rdbstat_path)]
            self.timeseries_names = ['iostat', 'vmstat', 'latency', 'qps', 'rdbstats']
#self.oprofile   = parser().parse_file(dir + self.oprofile_path)

    def report_as_html(self, dir):
        def image(source):
            return "<a href=\"%s\"> <img src=\"%s\" height=\"1200\" width=\"900\" /> </a>" % (source, source)

        res = open(dir + '/index.html', 'w')
        dir_str = time.asctime().replace(' ', '_').replace(':', '_')
        os.mkdir(dir + '/' + dir_str)

        print >>res, "<p> RethinkDB profiling report, autogenerated on %s </p>" % dir_str
        for name, timeseries in zip(self.runs[0].timeseries_names, self.runs[0].timeseries):
            if timeseries.data:
                timeseries.plot(dir + '/' + dir_str + '/' + name)
                print >>res, "<p> %s: </p>" % name
                print >>res, image('http://' + self.hostname + '/' + self.prof_dir + '/' + dir_str + '/' + name + '.png') #TODO use no-ip


#prog_report = sum(map(lambda x: x.oprofile, self.runs))
#print >>res, prog_report.report_as_html(sum(map(lambda x: x.ratios, small_packet_profiles)), CPU_CLK_UNHALTED, 15)
        res.close()
        #send stuff to host
        os.system('scp -r "%s" "%s:%s"' % (dir + '/' + dir_str, self.hostname, self.www_dir + self.prof_dir))
        os.system('scp "%s" "%s:%s"' % (dir + '/' + 'index.html', self.hostname, self.www_dir + self.prof_dir))
{'fraction_non_alphanumeric': 0.16567164179104477, 'fraction_numerical': 0.004104477611940298, 'mean_word_length': 2.241837968561064, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3897353', 'n_tokens_mistral': 930, 'n_tokens_neox': 903, 'n_words': 182}