<jupyter_start><jupyter_text>All the bonds and customers from challenge are contained in the trade matrix (sparse with 0.8% of ones)<jupyter_code>len(g) / (g.CustomerIdx.nunique() * g.IsinIdx.nunique())
len(all_g) / (all_g.CustomerIdx.nunique() * all_g.IsinIdx.nunique())
g.head()
from surprise import Reader, Dataset
reader = Reader(rating_scale=(0, 1))
data   = Dataset.load_from_df(all_g, reader)
data.split(n_folds=5)
from surprise import SVD, evaluate, NMF
%%time
algo = SVD(n_factors=15, n_epochs=50, biased=True)
evaluate(algo, data)
%%time
algo = SVD()
evaluate(algo, data)
from surprise import NormalPredictor
%%time
baseline = NormalPredictor()
evaluate(baseline, data)
from surprise import SVD
svd = SVD()
reader = Reader(rating_scale=(0, 1))
data   = Dataset.load_from_df(all_g, reader)
from surprise.model_selection import train_test_split
trainset, testset = train_test_split(data, test_size=1e-40)
%%time
svd.fit(trainset)
from surprise import accuracy
predictions = svd.test(all_g.values)
accuracy.rmse(predictions)
all_g.sample(5)
all_g.sample(10)
from tqdm import tqdm_notebook
train_preds = []
for cIdx, iIdx in tqdm_notebook(zip(all_g.CustomerIdx, all_g.IsinIdx), total=len(all_g)):
    train_preds.append(svd.predict(cIdx, iIdx))
preds = [x[3] for x in train_preds]
from sklearn.metrics import roc_auc_score
all_g.CustomerInterest.value_counts()
roc_auc_score(all_g.CustomerInterest, preds)
svd.predict(2523, 3172)
from surprise import accuracy
predictions = svd.test(testset)
accuracy.rmse(predictions)
g.describe().transpose()
algo.pu.shape, algo.bu.shape, algo.qi.shape, algo.bi.shape
g.CustomerIdx.nunique(), g.IsinIdx.nunique()
algo.bu[]
g.sample(5)
all_groups = trade.groupby(['CustomerIdx', 'IsinIdx'], as_index=False)
all_g = all_groups['CustomerInterest'].max()
all_g.sample(5)
algo.estimate(1314, 25423)<jupyter_output><empty_output><jupyter_text>Baseline predictor<jupyter_code>%%time
from surprise import NormalPredictor, evaluate, Reader, Dataset
baseline = NormalPredictor()
reader = Reader(rating_scale=(0, 1))
data   = Dataset.load_from_df(all_g, reader)
trainset = data.build_full_trainset()
%%time
baseline.fit(trainset)
from tqdm import tqdm_notebook
train_preds = []
for cIdx, iIdx in tqdm_notebook(zip(all_g.CustomerIdx, all_g.IsinIdx), total=len(all_g)):
    train_preds.append(baseline.predict(cIdx, iIdx)[3])
roc_auc_score(all_g.CustomerInterest, train_preds)<jupyter_output><empty_output><jupyter_text>NMF<jupyter_code>%%time
from surprise import NMF, evaluate, Reader, Dataset
nmf = NMF(n_factors=15, n_epochs=50, biased=True, verbose=True)
reader = Reader(rating_scale=(0, 1))
data   = Dataset.load_from_df(all_g.sample(frac=1, random_state=42), reader)
trainset = data.build_full_trainset()
%%time
nmf.fit(trainset)
from tqdm import tqdm_notebook
train_preds = []
for cIdx, iIdx in tqdm_notebook(zip(all_g.CustomerIdx, all_g.IsinIdx), total=len(all_g)):
    train_preds.append(nmf.predict(cIdx, iIdx)[3])
roc_auc_score(all_g.CustomerInterest, train_preds)<jupyter_output><empty_output><jupyter_text>SVD++<jupyter_code>%%time
from surprise import SVDpp, evaluate, Reader, Dataset
svdpp = SVDpp(n_factors=15, n_epochs=50, verbose=True)
reader = Reader(rating_scale=(0, 1))
data   = Dataset.load_from_df(all_g, reader)
trainset, testset = train_test_split(data, test_size=1e-40)
%%time
svdpp.fit(trainset)
from tqdm import tqdm_notebook
train_preds = []
for cIdx, iIdx in tqdm_notebook(zip(all_g.CustomerIdx, all_g.IsinIdx), total=len(all_g)):
    train_preds.append(svdpp.predict(cIdx, iIdx)[3])
roc_auc_score(all_g.CustomerInterest, train_preds)<jupyter_output><empty_output><jupyter_text>SVD<jupyter_code>%%time
from surprise import SVD, evaluate, Reader, Dataset
svd = SVD(n_factors=15, n_epochs=50, biased=True, verbose=True)
reader = Reader(rating_scale=(0, 1))
data   = Dataset.load_from_df(all_g.sample(frac=1, random_state=42), reader)
trainset = data.build_full_trainset()
%%time
svd.fit(trainset)
from tqdm import tqdm_notebook
train_preds = []
for cIdx, iIdx in tqdm_notebook(zip(all_g.CustomerIdx, all_g.IsinIdx), total=len(all_g)):
    train_preds.append(svd.predict(cIdx, iIdx)[3])
roc_auc_score(all_g.CustomerInterest, train_preds)<jupyter_output><empty_output><jupyter_text>Export info<jupyter_code>svd.pu.shape, svd.qi.shape, svd.bu.shape, svd.bi.shape
all_g.sample(5)
svd.predict(408, 16501)
svd.trainset.to_inner_uid(2579), svd.trainset.to_inner_iid(22225)
svd.trainset._raw2inner_id_users[2579], svd.trainset._raw2inner_id_items[22225]
all_g.CustomerIdx.nunique(), all_g.IsinIdx.nunique()
len(svd.trainset._raw2inner_id_users), len(svd.trainset._raw2inner_id_items)
uid = svd.trainset._raw2inner_id_users
iid = svd.trainset._raw2inner_id_items
user_id = 2549
item_id = 26187
svd.predict(user_id, item_id)
svd.trainset.global_mean
pd.Series(svd.bu).hist(bins=50)
pd.Series(svd.bi).hist(bins=50)
svd.bu[uid[408]], svd.bi[iid[16501]]
np.dot(svd.qi[uid[408]], svd.pu[iid[16501]])
res = svd.trainset.global_mean + svd.bu[uid[user_id]] + svd.bi[iid[item_id]] + \
        np.dot(svd.pu[uid[user_id]], svd.qi[iid[item_id]])
res
plt.scatter(svd.pu[:,0], svd.pu[:,1])
plt.scatter(nmf.pu[:,0], nmf.pu[:,1])
from surprise import dump
dump.dump(PROCESSED/'svd.dump', algo=svd)
dump.dump(PROCESSED/'nmf.dump', algo=nmf)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13900414937759337, 'fraction_numerical': 0.025650697849867975, 'mean_word_length': 7.882747068676717, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '645797', 'license': 'MIT', 'n_tokens_mistral': 2252, 'n_tokens_neox': 2137, 'n_words': 352, 'path': 'notebooks/robert/RA-16-Matrix-Factorization.ipynb', 'repo_name': 'Robert-Alonso/DSG-2018', 'size': '5302'}