<commit_before>from __future__ import print_function
import plac
import dill as pickle

from thinc.neural.vec2vec import Model, ReLu, Softmax
from thinc.neural._classes.batchnorm import BatchNorm as BN
from thinc.api import clone, chain
from thinc.loss import categorical_crossentropy

from thinc.extra import datasets


def main(depth=4, width=128, nb_epoch=5):
    with Model.define_operators({'**': clone, '>>': chain}):
        model = BN(ReLu(width)) ** depth >> Softmax()
   
    train_data, dev_data, _ = datasets.mnist()
    train_X, train_y = model.ops.unzip(train_data)
    dev_X, dev_y = model.ops.unzip(dev_data)

    with model.begin_training(train_X, train_y) as (trainer, optimizer):
        trainer.each_epoch.append(
            lambda: print(model.evaluate(dev_X, dev_y)))
        trainer.nb_epoch = nb_epoch
        trainer.dropout = 0.2
        trainer.dropout_decay = 0.0
        for X, y in trainer.iterate(train_X, train_y):
            X = model.ops.asarray(X)
            y = model.ops.asarray(y)
            yh, backprop = model.begin_update(X, drop=trainer.dropout)
            d_loss, loss = categorical_crossentropy(yh, y)
            optimizer.set_loss(loss)
            backprop(d_loss, optimizer)
    with model.use_params(optimizer.averages):
        print('Avg dev.: %.3f' % model.evaluate(dev_X, dev_y))
        with open('out.pickle', 'wb') as file_:
            pickle.dump(model, file_, -1)


if __name__ == '__main__':
    plac.call(main)<commit_msg>Clean up mnist example<commit_after>from __future__ import print_function
import plac
import dill as pickle
from tqdm import tqdm
from thinc.neural.vec2vec import Model, ReLu, Softmax
from thinc.api import clone, chain

from thinc.extra import datasets
from thinc.neural.ops import CupyOps


def main(depth=2, width=512, nb_epoch=20):
    Model.ops = CupyOps()
    # Configuration here isn't especially good. But, for demo..
    with Model.define_operators({'**': clone, '>>': chain}):
        model = ReLu(width) >> ReLu(width) >> Softmax()
   
    train_data, dev_data, _ = datasets.mnist()
    train_X, train_y = model.ops.unzip(train_data)
    dev_X, dev_y = model.ops.unzip(dev_data)

    with model.begin_training(train_X, train_y) as (trainer, optimizer):
        epoch_loss = [0.]
        def report_progress():
            with model.use_params(optimizer.averages):
                print(epoch_loss[-1], model.evaluate(dev_X, dev_y), trainer.dropout)
            epoch_loss.append(0.)
 
        trainer.each_epoch.append(report_progress)
        trainer.nb_epoch = nb_epoch
        trainer.dropout = 0.75
        trainer.batch_size = 128
        trainer.dropout_decay = 1e-4
        train_X = model.ops.asarray(train_X, dtype='float32')
        y_onehot = model.ops.allocate((train_X.shape[0], 10), dtype='float32')
        for i, label in enumerate(train_y):
            y_onehot[i, int(label)] = 1.
        for X, y in trainer.iterate(train_X, y_onehot):
            yh, backprop = model.begin_update(X, drop=trainer.dropout)
            loss = ((yh-y)**2.).sum() / y.shape[0]
            backprop(yh-y, optimizer)
            epoch_loss[-1] += loss
        with model.use_params(optimizer.averages):
            print('Avg dev.: %.3f' % model.evaluate(dev_X, dev_y))
            with open('out.pickle', 'wb') as file_:
                pickle.dump(model, file_, -1)


if __name__ == '__main__':
    plac.call(main)
{'fraction_non_alphanumeric': 0.12067955477445812, 'fraction_numerical': 0.012595196250732278, 'mean_word_length': 2.8762769580022702, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1739030', 'n_tokens_mistral': 1193, 'n_tokens_neox': 1170, 'n_words': 255}