<commit_before># coding: utf-8

from __future__ import division, print_function

import pickle
import numpy as np
import download 
from importlib import reload

import net_constructor
reload(net_constructor)
from net_constructor import NetConstructor


def maybe_download_and_extract():
    url = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
    download.maybe_download_and_extract(url, '.')

    
def process_data(file_name):

    with open(file_name, 'rb') as file:
        dict_data = pickle.load(file, encoding = 'latin1')

    x_data = dict_data['data']
    t_data = dict_data['labels']

    N = x_data.shape[0]

    x_data = x_data.reshape(N, 3, 32, 32).transpose(0, 2, 3, 1)
    t_onehot = np.zeros((N, 10))
    t_onehot[np.arange(N), t_data] = 1

    return x_data, t_onehot


def get_data():

    FILES_TRAIN = ['./cifar-10-batches-py/data_batch_{}'.format(k)
                   for k in range(1, 6)]
    FILE_TEST = './cifar-10-batches-py/test_batch'

    x_test, t_test = process_data(FILE_TEST)

    x_list = []
    t_list = []
    for file_name in FILES_TRAIN:
        x_data, t_data = process_data(file_name)
        x_list.append(x_data)
        t_list.append(t_data)

    x_data = np.vstack(x_list)
    t_data = np.vstack(t_list)

    return x_data, x_test, t_data, t_test


if __name__ == '__main__':

    maybe_download_and_extract()
    x_data, x_test, t_data, t_test = get_data()

    layer_list = []

    layer =  {'dim': (32, 32, 3)}
    layer_list.append(layer)

    layer = {'type': 'conv',
             'channels': 64,
             'k_size': (5, 5),
             'strides': (1, 1),
             'padding': 'SAME',
             'activation': 'relu',
             'init_w': 'truncated_normal',
             'stddev_w': 1.0,
             'init_b': 'zeros'}
    layer_list.append(layer)

    layer = {'type': 'maxpool',
             'k_size': (3, 3),
             'strides': (2, 2),
             'padding': 'SAME'}
    layer_list.append(layer)

    layer = {'type': 'LRN',
             'r': 4,
             'k': 1.0,
             'alpha': 0.001 / 9.0,
             'beta': 0.75}
    layer_list.append(layer)

    layer = {'type': 'conv',
             'channels': 64,
             'k_size': (5, 5),
             'strides': (1, 1),
             'padding': 'SAME',
             'activation': 'relu',
             'init_w': 'truncated_normal',
             'stddev_w': 1.0,
             'init_b': 'zeros'}
    layer_list.append(layer)

    layer = {'type': 'LRN',
             'r': 4,
             'k': 1.0,
             'alpha': 0.001 / 9.0,
             'beta': 0.75}
    layer_list.append(layer)
<commit_msg>Cambiados strides de (1, 1) a (3, 3)<commit_after># coding: utf-8

from __future__ import division, print_function

import pickle
import numpy as np
import download 
from importlib import reload

import net_constructor
reload(net_constructor)
from net_constructor import NetConstructor


def maybe_download_and_extract():
    url = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
    download.maybe_download_and_extract(url, '.')

    
def process_data(file_name):

    with open(file_name, 'rb') as file:
        dict_data = pickle.load(file, encoding = 'latin1')

    x_data = dict_data['data']
    t_data = dict_data['labels']

    N = x_data.shape[0]

    x_data = x_data.reshape(N, 3, 32, 32).transpose(0, 2, 3, 1)
    t_onehot = np.zeros((N, 10))
    t_onehot[np.arange(N), t_data] = 1

    return x_data, t_onehot


def get_data():

    FILES_TRAIN = ['./cifar-10-batches-py/data_batch_{}'.format(k)
                   for k in range(1, 6)]
    FILE_TEST = './cifar-10-batches-py/test_batch'

    x_test, t_test = process_data(FILE_TEST)

    x_list = []
    t_list = []
    for file_name in FILES_TRAIN:
        x_data, t_data = process_data(file_name)
        x_list.append(x_data)
        t_list.append(t_data)

    x_data = np.vstack(x_list)
    t_data = np.vstack(t_list)

    return x_data, x_test, t_data, t_test


if __name__ == '__main__':

    maybe_download_and_extract()
    x_data, x_test, t_data, t_test = get_data()

    layer_list = []

    layer =  {'dim': (32, 32, 3)}
    layer_list.append(layer)

    layer = {'type': 'conv',
             'channels': 64,
             'k_size': (5, 5),
             'strides': (3, 3),
             'padding': 'SAME',
             'activation': 'relu',
             'init_w': 'truncated_normal',
             'stddev_w': 1.0,
             'init_b': 'zeros'}
    layer_list.append(layer)

    layer = {'type': 'maxpool',
             'k_size': (3, 3),
             'strides': (2, 2),
             'padding': 'SAME'}
    layer_list.append(layer)

    layer = {'type': 'LRN',
             'r': 4,
             'k': 1.0,
             'alpha': 0.001 / 9.0,
             'beta': 0.75}
    layer_list.append(layer)

    layer = {'type': 'conv',
             'channels': 64,
             'k_size': (5, 5),
             'strides': (3, 3),
             'padding': 'SAME',
             'activation': 'relu',
             'init_w': 'truncated_normal',
             'stddev_w': 1.0,
             'init_b': 'zeros'}
    layer_list.append(layer)

    layer = {'type': 'LRN',
             'r': 4,
             'k': 1.0,
             'alpha': 0.001 / 9.0,
             'beta': 0.75}
    layer_list.append(layer)

{'fraction_non_alphanumeric': 0.15080113100848255, 'fraction_numerical': 0.027898209236569275, 'mean_word_length': 2.077726218097448, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6386380', 'n_tokens_mistral': 2074, 'n_tokens_neox': 1909, 'n_words': 403}