<commit_before>
import time

class Timer:
    def __init__(self, verbose=False, messagelen=20):
        now = time.time()
        self.times = [(now, now, "", [])]
        self.subtimes = []
        self.verbose = verbose
        self.messagelen = messagelen
    def duration_ms(self, entry):
        return int((entry[1] - entry[0]) * 1000)
    def print_message(self, message, time_ms):
        print "{{:{}}} {{:8}}ms".format(self.messagelen).format(message, time_ms)
    def measure(self, message):
        if len(message) > self.messagelen:
            self.messagelen = len(message) + 10
        now = time.time()
        self.times.append((self.times[-1][1], now, message, self.subtimes))
        self.subtimes = []
        if self.verbose:
            self.print_message(message, self.duration_ms(self.times[-1]))
    def submeasure(self, message):
        message = "    " + message
        if len(message) > self.messagelen:
            self.messagelen = len(message) + 10
        now = time.time()
        if self.subtimes:
            self.subtimes.append((self.subtimes[-1][1], now, message))
        else:
            self.subtimes.append((self.times[-1][1], now, message))
        if self.verbose:
            if len(self.subtimes) == 1:
                print
            self.print_message(message, self.duration_ms(self.subtimes[-1]))
    def print_all(self):
        for t in self.times[1:]:
            begin, end, message, subtimes = t
            self.print_message(message, self.duration_ms(t))
            for st in subtimes:
                sbegin, send, smessage = st
                self.print_message(message, self.duration_ms(st))
    def full_duration(self):
        return self.times[-1][1] - self.times[0][0]
    def print_total(self):
        print "in total: {}".format(self.duration_ms(self.full_duration()))<commit_msg>might fix instrumenting.py<commit_after>
import time

class Timer:
    def __init__(self, verbose=False, messagelen=20):
        now = time.time()
        self.times = [(now, now, "", [])]
        self.subtimes = []
        self.verbose = verbose
        self.messagelen = messagelen
    def duration_ms(self, entry):
        return int((entry[1] - entry[0]) * 1000)
    def print_message(self, message, time_ms):
        print "{{:{}}} {{:8}}ms".format(self.messagelen).format(message, time_ms)
    def measure(self, message):
        if len(message) > self.messagelen:
            self.messagelen = len(message) + 10
        now = time.time()
        self.times.append((self.times[-1][1], now, message, self.subtimes))
        self.subtimes = []
        if self.verbose:
            self.print_message(message, self.duration_ms(self.times[-1]))
    def submeasure(self, message):
        message = "    " + message
        if len(message) > self.messagelen:
            self.messagelen = len(message) + 10
        now = time.time()
        if self.subtimes:
            self.subtimes.append((self.subtimes[-1][1], now, message))
        else:
            self.subtimes.append((self.times[-1][1], now, message))
        if self.verbose:
            if len(self.subtimes) == 1:
                print
            self.print_message(message, self.duration_ms(self.subtimes[-1]))
    def print_all(self):
        for t in self.times[1:]:
            begin, end, message, subtimes = t
            self.print_message(message, self.duration_ms(t))
            for st in subtimes:
                sbegin, send, smessage = st
                self.print_message(message, self.duration_ms(st))
    def full_duration(self):
        return self.times[-1][1] - self.times[0][0]
    def print_total(self):
        print "in total: {}".format(int(self.full_duration() * 1000))
{'fraction_non_alphanumeric': 0.14131897711978467, 'fraction_numerical': 0.015612382234185734, 'mean_word_length': 2.2454148471615722, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '99829', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1138, 'n_words': 236}