<commit_before>#!/usr/bin/env python
"""Chainer example: train a multi-layer perceptron on MNIST

This is a minimal example to write a feed-forward net. It requires scikit-learn
to load MNIST dataset.

"""
import numpy as np

import six

import chainer
from chainer import cuda, FunctionSet
import chainer.functions as F
from chainer import optimizers


class MNIST(object):

    def __init__(self, model_path, gpu):
        self.gpu = gpu

        n_units = 1000

        if model_path != '':
            with open(model_path, 'rb') as model_pickle:
                self.model = six.moves.cPickle.load(model_pickle)
        else:
            self.model = FunctionSet(
                l1=F.Linear(784, n_units),
                l2=F.Linear(n_units, n_units),
                l3=F.Linear(n_units, 10))

        if self.gpu >= 0:
            cuda.init(self.gpu)
            self.model.to_gpu()

        # Setup optimizer
        self.optimizer = optimizers.Adam()
        self.optimizer.setup(self.model.collect_parameters())

    def forward(self, x_data, train=True):
        x = chainer.Variable(x_data)
        h1 = F.dropout(F.relu(self.model.l1(x)),  train=train)
        h2 = F.dropout(F.relu(self.model.l2(h1)), train=train)
        return self.model.l3(h2)

    def fit(self, xys):
        x = []
        y = []
        for d in xys:
            x.append(d['data'])
            y.append(d['label'])
        x_batch = np.array(x, dtype=np.float32)
        y_batch = np.array(y, dtype=np.int32)

        if self.gpu >= 0:
            x_batch = cuda.to_gpu(x_batch)
            y_batch = cuda.to_gpu(y_batch)

        self.optimizer.zero_grads()
        y = self.forward(x_batch)

        t = chainer.Variable(y_batch)
        loss = F.softmax_cross_entropy(y, t)
        acc = F.accuracy(y, t)

        loss.backward()
        self.optimizer.update()

        nloss = float(cuda.to_cpu(loss.data)) * len(y_batch)
        naccuracy = float(cuda.to_cpu(acc.data)) * len(y_batch)

        retmap = {
            'loss': nloss,
            'accuracy': naccuracy,
        }

        return retmap

    def predict(self, x):
        # non batch
        xx = []
        xx.append(x)
        x_data = np.array(xx, dtype=np.float32)
        if self.gpu >= 0:
            x_data = cuda.to_gpu(x_data)

        y = self.forward(x_data, train=False)
        y = y.data.reshape(y.data.shape[0], y.data.size / y.data.shape[0])
        pred = y.argmax(axis=1)

        return int(pred[0])

    def get_model(self):
        return self.model
<commit_msg>Add save/load method to mnist example<commit_after>#!/usr/bin/env python
"""Chainer example: train a multi-layer perceptron on MNIST

This is a minimal example to write a feed-forward net. It requires scikit-learn
to load MNIST dataset.

"""
import numpy as np

import six

import chainer
from chainer import cuda, FunctionSet
import chainer.functions as F
from chainer import optimizers


class MNIST(object):

    def __init__(self, model_path, gpu):
        self.gpu = gpu

        n_units = 1000

        if model_path != '':
            with open(model_path, 'rb') as model_pickle:
                self.model = six.moves.cPickle.load(model_pickle)
        else:
            self.model = FunctionSet(
                l1=F.Linear(784, n_units),
                l2=F.Linear(n_units, n_units),
                l3=F.Linear(n_units, 10))

        if self.gpu >= 0:
            cuda.init(self.gpu)
            self.model.to_gpu()

        # Setup optimizer
        self.optimizer = optimizers.Adam()
        self.optimizer.setup(self.model.collect_parameters())

    def forward(self, x_data, train=True):
        x = chainer.Variable(x_data)
        h1 = F.dropout(F.relu(self.model.l1(x)),  train=train)
        h2 = F.dropout(F.relu(self.model.l2(h1)), train=train)
        return self.model.l3(h2)

    def fit(self, xys):
        x = []
        y = []
        for d in xys:
            x.append(d['data'])
            y.append(d['label'])
        x_batch = np.array(x, dtype=np.float32)
        y_batch = np.array(y, dtype=np.int32)

        if self.gpu >= 0:
            x_batch = cuda.to_gpu(x_batch)
            y_batch = cuda.to_gpu(y_batch)

        self.optimizer.zero_grads()
        y = self.forward(x_batch)

        t = chainer.Variable(y_batch)
        loss = F.softmax_cross_entropy(y, t)
        acc = F.accuracy(y, t)

        loss.backward()
        self.optimizer.update()

        nloss = float(cuda.to_cpu(loss.data)) * len(y_batch)
        naccuracy = float(cuda.to_cpu(acc.data)) * len(y_batch)

        retmap = {
            'loss': nloss,
            'accuracy': naccuracy,
        }

        return retmap

    def predict(self, x):
        # non batch
        xx = []
        xx.append(x)
        x_data = np.array(xx, dtype=np.float32)
        if self.gpu >= 0:
            x_data = cuda.to_gpu(x_data)

        y = self.forward(x_data, train=False)
        y = y.data.reshape(y.data.shape[0], y.data.size / y.data.shape[0])
        pred = y.argmax(axis=1)

        return int(pred[0])

    def get_model(self):
        return self.model

    def save(self):
        return bytearray(six.moves.cPickle.dumps(self.model))

    def load(self, model_data):
        self.model = six.moves.cPickle.loads(str(model_data))

{'fraction_non_alphanumeric': 0.12273071104387293, 'fraction_numerical': 0.012102874432677761, 'mean_word_length': 1.976364659538548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4415614', 'n_tokens_mistral': 1909, 'n_tokens_neox': 1815, 'n_words': 374}