<commit_before>"""
======================================================
Online Learning with Operator-Valued kernels
======================================================

An example to illustrate online learning with operator-valued
kernels.
"""

import operalib as ovk
import numpy as np
import matplotlib.pyplot as plt

import time

from IPython import embed

np.random.seed(0)

n = 5000
d = 20
p = 4
X = np.random.rand(n, d)


def phi(X):
    return np.hstack((X[:, 0:1] ** 2,
                      X[:, 3:4] ** 2,
                      X[:, 0:1] * X[:, 1:2],
                      X[:, 2:3] * X[:, 4:5],
                      X[:, 1:2],
                      X[:, 3:4],
                      np.ones((n, 1))))


print('Generating Data')
y = np.dot(phi(X), np.random.multivariate_normal(np.zeros(7),
                                                 np.diag([0.5, 0.25, 0.1, 0.05,
                                                          0.15, 0.1, 0.15]),
                                                 p).T)

# Link components to a common mean.
y = .5 * y + 0.5 * np.mean(y, axis=1).reshape(-1, 1)

est = ovk.ONORMA('DGauss', A=.8 * np.eye(p) + .2 * np.ones((p, p)), gamma=.25,
                 learning_rate=ovk.InvScaling(1., .5), lbda=0.00001)

print('Fitting Joint...')
start = time.time()
est.partial_fit(X[0, :].reshape(1, -1), y[0, :].reshape(1, -1), n, p)
err = np.empty(n)
err[0] = np.linalg.norm(y[0, :]) ** 2
for t in range(1, n):
    err[t] = np.linalg.norm(est.predict(X[t, :].reshape(1, -1)) -
                            y[t, :].reshape(1, -1)) ** 2
    est.partial_fit(X[t, :].reshape(1, -1), y[t, :].reshape(1, -1))
print('Joint training time:', time.time() - start)
print('Joint MSE:', err[-1])

err_c = np.cumsum(err) / (np.arange(n) + 1)
plt.semilogy(np.linspace(0, 100, err_c.size), err_c, label='Joint')

est = ovk.ONORMA('DGauss', A=1. * np.eye(p) + .0 * np.ones((p, p)), gamma=.25,
                 learning_rate=ovk.InvScaling(1., .5), lbda=0.00001)

print('Fitting Independant...')
start = time.time()
est.partial_fit(X[0, :].reshape(1, -1), y[0, :].reshape(1, -1), n, p)
err = np.empty(n)
err[0] = np.linalg.norm(y[0, :]) ** 2
for t in range(1, n):
    err[t] = np.linalg.norm(est.predict(X[t, :].reshape(1, -1)) -
                            y[t, :].reshape(1, -1)) ** 2
    est.partial_fit(X[t, :].reshape(1, -1), y[t, :].reshape(1, -1))
print('Independant training time:', time.time() - start)
print('Independant MSE:', err[-1])

err_c = np.cumsum(err) / (np.arange(n) + 1)
plt.semilogy(np.linspace(0, 100, err_c.size), err_c, label='Independant')

plt.title('Online learning with ONORMA')
plt.xlabel('Size of the Training set (%)')
plt.ylabel('MSE')
plt.legend()
plt.show()
<commit_msg>Removed IPython<commit_after>"""
======================================================
Online Learning with Operator-Valued kernels
======================================================

An example to illustrate online learning with operator-valued
kernels.
"""

import operalib as ovk
import numpy as np
import matplotlib.pyplot as plt

import time

np.random.seed(0)

n = 5000
d = 20
p = 4
X = np.random.rand(n, d)


def phi(X):
    return np.hstack((X[:, 0:1] ** 2,
                      X[:, 3:4] ** 2,
                      X[:, 0:1] * X[:, 1:2],
                      X[:, 2:3] * X[:, 4:5],
                      X[:, 1:2],
                      X[:, 3:4],
                      np.ones((n, 1))))


print('Generating Data')
y = np.dot(phi(X), np.random.multivariate_normal(np.zeros(7),
                                                 np.diag([0.5, 0.25, 0.1, 0.05,
                                                          0.15, 0.1, 0.15]),
                                                 p).T)

# Link components to a common mean.
y = .5 * y + 0.5 * np.mean(y, axis=1).reshape(-1, 1)

est = ovk.ONORMA('DGauss', A=.8 * np.eye(p) + .2 * np.ones((p, p)), gamma=.25,
                 learning_rate=ovk.InvScaling(1., .5), lbda=0.00001)

print('Fitting Joint...')
start = time.time()
est.partial_fit(X[0, :].reshape(1, -1), y[0, :].reshape(1, -1), n, p)
err = np.empty(n)
err[0] = np.linalg.norm(y[0, :]) ** 2
for t in range(1, n):
    err[t] = np.linalg.norm(est.predict(X[t, :].reshape(1, -1)) -
                            y[t, :].reshape(1, -1)) ** 2
    est.partial_fit(X[t, :].reshape(1, -1), y[t, :].reshape(1, -1))
print('Joint training time:', time.time() - start)
print('Joint MSE:', err[-1])

err_c = np.cumsum(err) / (np.arange(n) + 1)
plt.semilogy(np.linspace(0, 100, err_c.size), err_c, label='Joint')

est = ovk.ONORMA('DGauss', A=1. * np.eye(p) + .0 * np.ones((p, p)), gamma=.25,
                 learning_rate=ovk.InvScaling(1., .5), lbda=0.00001)

print('Fitting Independant...')
start = time.time()
est.partial_fit(X[0, :].reshape(1, -1), y[0, :].reshape(1, -1), n, p)
err = np.empty(n)
err[0] = np.linalg.norm(y[0, :]) ** 2
for t in range(1, n):
    err[t] = np.linalg.norm(est.predict(X[t, :].reshape(1, -1)) -
                            y[t, :].reshape(1, -1)) ** 2
    est.partial_fit(X[t, :].reshape(1, -1), y[t, :].reshape(1, -1))
print('Independant training time:', time.time() - start)
print('Independant MSE:', err[-1])

err_c = np.cumsum(err) / (np.arange(n) + 1)
plt.semilogy(np.linspace(0, 100, err_c.size), err_c, label='Independant')

plt.title('Online learning with ONORMA')
plt.xlabel('Size of the Training set (%)')
plt.ylabel('MSE')
plt.legend()
plt.show()

{'fraction_non_alphanumeric': 0.24268090591051372, 'fraction_numerical': 0.04640029460504511, 'mean_word_length': 2.3182651191203423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4198211', 'n_tokens_mistral': 2263, 'n_tokens_neox': 2090, 'n_words': 461}