<commit_before>import sys
import os
from optparse import OptionParser
import mimetypes

from twisted.internet import reactor
from twisted.python import log
from twisted.internet.defer import gatherResults

from txaws.credentials import AWSCredentials
from txaws.service import AWSServiceRegion

from bafload.stats import ThroughputCounter
from bafload.up import MultipartUploadsManager


parser = OptionParser()
parser.add_option('-a', '--acess-key', dest='access_key',
    help='AWS Acess Key ID')
parser.add_option('-s', '--secret-key', dest='secret_key',
    help='AWS Secret Access Key')
parser.add_option('-r', '--region', dest='region', default='US',
    help='EC2 Service Regison')
parser.add_option('-b', '--bucket', dest='bucket',
    help='Name of the bucket to upload to')
options, paths = parser.parse_args()


def error(message):
    sys.stderr.write(message + '\n')
    reactor.stop()


def show_stats(result, throughput_counter):
    table = throughput_counter.read()
    slot_dur = throughput_counter.stats.slot_duration_secs
    (index0, (t0, _)) = [(i, (t, count))
            for (i, (t, count)) in enumerate(table) if count][0]
    (index1, (tk, _)) = [(i, (t, count))
            for (i, (t, count)) in enumerate(reversed(table)) if count][0]
    index1 = throughput_counter.stats.size - index1
    elapsed = (tk - t0) + slot_dur
    counts = [slot[1] for slot in table[index0:index1+1]]
    tx = sum(counts)
    max_count = max(counts)
    max_mbps = max_count / float(slot_dur) / (2**20)
    min_mbps = min(counts) / float(slot_dur) / (2**20)
    avg_mbps = tx / float(elapsed) / (2**20)
    star_chart = []
    for e in table[index0:index1+1]:
        ct = int((e[1] / float(max_count)) * 60)
        star_chart.append('*' * ct)
    print '========= Throughput graph:'
    print '\n'.join(star_chart)
    print 'Average Transfer: %3.3fMBs' % avg_mbps
    print 'Max: %3.3fMBs' % max_mbps
    print 'Min: %3.3fMBs' % min_mbps
    print 'Total tx: %2.2fMB' % (tx / 2.0**20)
    return result<commit_msg>cleanup<commit_after>import sys
import os
from optparse import OptionParser
import mimetypes

from twisted.internet import reactor
from twisted.python import log
from twisted.internet.defer import gatherResults

from txaws.credentials import AWSCredentials
from txaws.service import AWSServiceRegion

from bafload.stats import ThroughputCounter
from bafload.up import MultipartUploadsManager


parser = OptionParser()
parser.add_option('-a', '--acess-key', dest='access_key',
    help='AWS Acess Key ID')
parser.add_option('-s', '--secret-key', dest='secret_key',
    help='AWS Secret Access Key')
parser.add_option('-r', '--region', dest='region', default='US',
    help='EC2 Service Regison')
parser.add_option('-b', '--bucket', dest='bucket',
    help='Name of the bucket to upload to')
options, paths = parser.parse_args()


def error(message):
    sys.stderr.write(message + '\n')
    reactor.stop()


def show_stats(result, throughput_counter):
    table = throughput_counter.read()
    slot_dur = throughput_counter.stats.slot_duration_secs
    (index0, (t0, _)) = [(i, (t, count))
            for (i, (t, count)) in enumerate(table) if count][0]
    (index1, (tk, _)) = [(i, (t, count))
            for (i, (t, count)) in enumerate(reversed(table)) if count][0]
    index1 = throughput_counter.stats.size - index1
    elapsed = (tk - t0) + slot_dur
    counts = [slot[1] for slot in table[index0:index1+1]]
    tx = sum(counts)
    max_count = max(counts)
    max_mbps = max_count / float(slot_dur) / (2**20)
    min_mbps = min(counts) / float(slot_dur) / (2**20)
    avg_mbps = tx / float(elapsed) / (2**20)
    print 'Average Transfer: %3.3fMBs' % avg_mbps
    print 'Max: %3.3fMBs' % max_mbps
    print 'Min: %3.3fMBs' % min_mbps
    print 'Total tx: %2.2fMB' % (tx / 2.0**20)
    return result
{'fraction_non_alphanumeric': 0.14778067885117493, 'fraction_numerical': 0.019321148825065273, 'mean_word_length': 3.8249370277078087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1173362', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1342, 'n_words': 339}