<commit_before>#!/usr/bin/python

# This was originally created to target inclusion in soumith's benchmarks as
# https://github.com/soumith/convnet-benchmarks
# extending this to handle also a couple of clarke and storkey type layers
# and some plausible mnist layers

from __future__ import print_function

import os
import sys
import time
import array
import random
import PyDeepCL

num_epochs = 10
batch_size = 128  # always use this, seems pretty standard
runs = [
    ('soumith1', '3i128-96c11'),  # format: ('[label]','[inputplanes]i[inputsize]-[numfilters]c[filtersize]')
    ('soumith2', '64i64-128c9'),
    ('soumith3', '128i32-128c9'),
    ('soumith4', '128i16-128c7'),
    ('soumith5', '384i13-384c3'),
    ('maddison-convolve', '128i19-128c3'),
    ('maddison-fc', '128i19-361n'), # this crashes currently, not sure why, since it's just allocated 63MB
    # memory for weights, so it's probably a bug in my code, rather than a 
    # theoretical limitation of current implementation
    ('mnist-c1', '1i28-8c5'),
    ('mnist-c2', '8i14-16c5'),
    ('mnist-fc', '16i7-150n')
]

def writeResults( resultsLine ):
    f = open('results.txt', 'a')
    f.write( resultsLine + '\n' )
    f.close()

def time_layer(num_epochs, label, batch_size, net_string):
    print('building network...')
    input_string, layer_string = net_string.split('-')
    input_planes, input_size = map(lambda x: int(x), input_string.split('i'))
    net = PyDeepCL.NeuralNet( input_planes, input_size )
    net.addLayer( PyDeepCL.ForceBackpropMaker() ) # this forces the next layer to backprop gradients to
                          # this layer
    print( net.asString() )
    if 'c' in layer_string:
        num_filters, filter_size = map(lambda x: int(x), layer_string.split('c'))
        net.addLayer( PyDeepCL.ConvolutionalMaker().numFilters(num_filters)
            .filterSize(filter_size).biased().linear() )
    elif 'n' in layer_string:
        num_neurons = int(layer_string.split('n')[0])
        net.addLayer( PyDeepCL.FullyConnectedMaker().numPlanes(num_neurons).imageSize(1).biased().linear() )
    else:
        raise Exception('layer_string {layer_string} not recognized'.format(
            layer_string=layer_string))
    print( net.asString() )
    net.addLayer( PyDeepCL.FullyConnectedMaker().numPlanes(1).imageSize(1) )
    net.addLayer( PyDeepCL.SoftMaxMaker() )
    print( net.asString() )

    images = array.array( 'f', [0] * (batch_size*input_planes*input_size*input_size) )
    for i in range( batch_size*input_planes*input_size*input_size ):
        images[i] = random.random() - 0.5
#    grad = array.array('f',[0] * batch_size * outputPlanes * (input_size - filterSize + 1) )
#    for i in range( batch_size * outputPlanes * (input_size - filterSize + 1) ):
#        grad[i] = random.random() - 0.5
    labels = array.array('i',[0] * batch_size )
    
    print('warming up...')
    #try:
    net.setBatchSize(batch_size)

    # warm up forward
    for i in range(8):
        last = time.time()
        net.propagate( images )
        now = time.time()
        print('  warm up propagate all-layer time', now - last )
        last = now
    net.backPropFromLabels( 0.001, labels )
    now = time.time()
    print('   warm up backprop all-layer time', now - last )
    last = now

    layer = net.getLayer(2)
    print('running forward prop timings:')
    for i in range(num_epochs):
        layer.propagate()
    now = time.time()
    print('forward layer total time', now - last )
    print('forward layer average time', ( now - last ) / float(num_epochs) )
    writeResults( label + ', ' + net_string + ', ' + layer.asString() + ', forward=' + str( ( now - last ) / float(num_epochs) * 1000 ) + 'ms' )

    print('warm up backwards again')
    layer.backProp(0.001)
    layer.backProp(0.001)
    print('warm up backwards done. start timings:')

    now = time.time()
    last = now
    for i in range(num_epochs):
        layer.backProp(0.001)
    now = time.time()
    print('backwar layer total time', now - last )
    print('backwar layer average time', ( now - last ) / float(num_epochs) )
    writeResults( label + ', ' + net_string + ', ' + layer.asString() + ', backward=' + str( ( now - last ) / float(num_epochs) * 1000 ) + 'ms' )
    last = now

def time_run(fn):
    times = []
    fn()  # warm-up call, outputPlanest timed
    for _ in range(repeat):
        start = time.time()
        for _ in range(number):
            fn()
        times.append((time.time() - start) / number)
    return min(times)

#def parse_custom_config(s):
#    # parses a custom configuration string of the format:
#    # AxB-CcD where A: input channels, B: input size,
#    # C: output channels, D: kernel size, E: batchsize
#    run = {'batch_size': 128 }
#    defs = {'i': ['input_planes', 'input_size'],<commit_msg>improve benchmarking format; additional information on cmd_line etc<commit_after>#!/usr/bin/python

# This was originally created to target inclusion in soumith's benchmarks as
# https://github.com/soumith/convnet-benchmarks
# extending this to handle also a couple of clarke and storkey type layers
# and some plausible mnist layers

from __future__ import print_function
import os
import sys
import time
import array
import random
import json
import PyDeepCL

cmd_line = 'cd python; python setup.py build_ext -i; PYTHONPATH=. python'
for arg in sys.argv:
    cmd_line += ' ' + arg
print('cmd_line: [' + cmd_line + ']')
    # benchmarking/deepcl_benchmark2.py'
num_epochs = 10
batch_size = 128  # always use this, seems pretty standard
runs = [
    ('soumith1', '3i128-96c11'),  # format: ('[label]','[inputplanes]i[inputsize]-[numfilters]c[filtersize]')
    ('soumith2', '64i64-128c9'),
    ('soumith3', '128i32-128c9'),
    ('soumith4', '128i16-128c7'),
    ('soumith5', '384i13-384c3'),
    ('maddison-convolve', '128i19-128c3'),
    ('maddison-fc', '128i19-361n'), # this crashes currently, not sure why, since it's just allocated 63MB
    # memory for weights, so it's probably a bug in my code, rather than a 
    # theoretical limitation of current implementation
    ('mnist-c1', '1i28-8c5'),
    ('mnist-c2', '8i14-16c5'),
    ('mnist-fc', '16i7-150n')
]

def write_results( label, net_string, layer, forward_backward, time_ms ):
    global cmd_line
    results_dict = {}
    results_dict['label'] = label
    results_dict['format'] = 'v0.3'
    results_dict['direction'] = forward_backward
    results_dict['net_string'] = net_string
    results_dict['layer_string'] = layer.asString()
    results_dict['time_ms'] = str(time_ms)
    results_dict['cmd_line'] = cmd_line

    f = open('results.txt', 'a')
    json.dump(results_dict, f)
    f.write( '\n' )
    f.close()

def time_layer(num_epochs, label, batch_size, net_string):
    print('building network...')
    input_string, layer_string = net_string.split('-')
    input_planes, input_size = map(lambda x: int(x), input_string.split('i'))
    net = PyDeepCL.NeuralNet( input_planes, input_size )
    net.addLayer( PyDeepCL.ForceBackpropMaker() ) # this forces the next layer to backprop gradients to
                          # this layer
    print( net.asString() )
    if 'c' in layer_string:
        num_filters, filter_size = map(lambda x: int(x), layer_string.split('c'))
        net.addLayer( PyDeepCL.ConvolutionalMaker().numFilters(num_filters)
            .filterSize(filter_size).biased().linear() )
    elif 'n' in layer_string:
        num_neurons = int(layer_string.split('n')[0])
        net.addLayer( PyDeepCL.FullyConnectedMaker().numPlanes(num_neurons).imageSize(1).biased().linear() )
    else:
        raise Exception('layer_string {layer_string} not recognized'.format(
            layer_string=layer_string))
    print( net.asString() )
    net.addLayer( PyDeepCL.FullyConnectedMaker().numPlanes(1).imageSize(1) )
    net.addLayer( PyDeepCL.SoftMaxMaker() )
    print( net.asString() )

    images = array.array( 'f', [0] * (batch_size*input_planes*input_size*input_size) )
    for i in range( batch_size*input_planes*input_size*input_size ):
        images[i] = random.random() - 0.5
#    grad = array.array('f',[0] * batch_size * outputPlanes * (input_size - filterSize + 1) )
#    for i in range( batch_size * outputPlanes * (input_size - filterSize + 1) ):
#        grad[i] = random.random() - 0.5
    labels = array.array('i',[0] * batch_size )
    
    print('warming up...')
    #try:
    net.setBatchSize(batch_size)

    # warm up forward
    for i in range(8):
        last = time.time()
        net.propagate( images )
        now = time.time()
        print('  warm up propagate all-layer time', now - last )
        last = now
    net.backPropFromLabels( 0.001, labels )
    now = time.time()
    print('   warm up backprop all-layer time', now - last )
    last = now

    layer = net.getLayer(2)
    print('running forward prop timings:')
    for i in range(num_epochs):
        layer.propagate()
    now = time.time()
    print('forward layer total time', now - last )
    print('forward layer average time', ( now - last ) / float(num_epochs) )
    # forward_time_per_layer_ms = ( now - last ) / float(num_epochs) * 1000
    # writeResults( label + ', ' + net_string + ', ' + layer.asString() + ', forward=' + str( ( now - last ) / float(num_epochs) * 1000 ) + 'ms' )
    write_results( label, net_string, layer, 'forward', ( now - last ) / float(num_epochs) * 1000 )

    print('warm up backwards again')
    layer.backProp(0.001)
    layer.backProp(0.001)
    print('warm up backwards done. start timings:')

    now = time.time()
    last = now
    for i in range(num_epochs):
        layer.backProp(0.001)
    now = time.time()
    print('backwar layer total time', now - last )
    print('backwar layer average time', ( now - last ) / float(num_epochs) )
    # writeResults( label + ', ' + net_string + ', ' + layer.asString() + ', backward=' + str( ( now - last ) / float(num_epochs) * 1000 ) + 'ms' )
    write_results( label, net_string, layer, 'backward', ( now - last ) / float(num_epochs) * 1000 )
    last = now

def time_run(fn):
    times = []
    fn()  # warm-up call, outputPlanest timed
    for _ in range(repeat):
        start = time.time()
        for _ in range(number):
            fn()
        times.append((time.time() - start) / number)
    return min(times)

#def parse_custom_config(s):
#    # parses a custom configuration string of the format:
#    # AxB-CcD where A: input channels, B: input size,
#    # C: output channels, D: kernel size, E: batchsize
#    run = {'batch_size': 128 }
#    defs = {'i': ['input_planes', 'input_size'],
{'fraction_non_alphanumeric': 0.1414141414141414, 'fraction_numerical': 0.02652695176059662, 'mean_word_length': 3.1923229125445194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5513223', 'n_tokens_mistral': 3763, 'n_tokens_neox': 3547, 'n_words': 889}