<commit_before>import os
import sys
sys.path.insert(0, os.path.abspath('..'))
import functools
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import metrics

import numpy as np
from aorun.models import Model
from aorun.layers import ProbabilisticDense
from aorun.layers import Activation
from aorun.optimizers import SGD
from aorun.losses import variational_loss
from aorun.losses import categorical_crossentropy

X, y = datasets.load_iris(return_X_y=True)
X = X.astype('float32')
y = np.eye(y.max() + 1)[y].astype('float32')
X = StandardScaler().fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
print(X_train.shape, y_train.shape)

model = Model(
    ProbabilisticDense(10, input_dim=X_train.shape[-1]),
    Activation('relu'),
    ProbabilisticDense(10),
    Activation('relu'),
    ProbabilisticDense(y_test.shape[-1]),
    Activation('softmax')
)

sgd = SGD(lr=0.001)
loss = functools.partial(variational_loss,
                         model=model,
                         log_likelihood=categorical_crossentropy)
history = model.fit(X_train, y_train, n_epochs=500, loss=loss, optimizer=sgd)

y_pred = model.forward(X_test)
acc = metrics.accuracy_score(y_test.argmax(axis=1), y_pred.argmax(axis=1))
print('Accuracy:', acc)<commit_msg>fix probabilistic classification example<commit_after>import os
import sys
sys.path.insert(0, os.path.abspath('..'))
import functools
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import metrics

import numpy as np
from aorun.models import Model
from aorun.layers import ProbabilisticDense
from aorun.layers import Activation
from aorun.optimizers import SGD
from aorun.losses import variational_loss

X, y = datasets.load_digits(return_X_y=True)
X = X.astype('float32')
y = np.eye(y.max() + 1)[y].astype('float32')
X = StandardScaler().fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
print(X_train.shape, y_train.shape)

model = Model(
    ProbabilisticDense(200, input_dim=X_train.shape[-1]),
    Activation('relu'),
    ProbabilisticDense(200),
    Activation('relu'),
    ProbabilisticDense(y_test.shape[-1]),
    Activation('softmax')
)

opt = SGD(lr=0.1, momentum=0.9)
loss = variational_loss(model, 'categorical_crossentropy')
history = model.fit(X_train, y_train, n_epochs=20, loss=loss, optimizer=opt)

y_pred = model.forward(X_test)
acc = metrics.accuracy_score(y_test.argmax(axis=1), y_pred.argmax(axis=1))
print('test samples:', len(y_test))
print('classes:', len(y_test[0]))
print('Accuracy:', acc)
{'fraction_non_alphanumeric': 0.12023460410557185, 'fraction_numerical': 0.017595307917888565, 'mean_word_length': 5.788557213930348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6527235', 'n_tokens_mistral': 1005, 'n_tokens_neox': 966, 'n_words': 201}