<jupyter_start><jupyter_text>Fastai MixedInputModel NN (concatenates embeddings to cont vars)<jupyter_code>FAI_model = data.get_learner(emb_szs=emb_szs, n_cont=2, emb_drop=.05, out_sz=out_sz, szs = szs, drops = [.1, .2, .3, .4], use_bn=True)
FAI_model.lr_find()
%matplotlib inline
FAI_model.sched.plot(100)
lr = 1.2e-3
# def accuracy(preds, y):
#     return np.equal(y,np.round(preds[:,0])).sum()/len(y)

from sklearn.metrics import mean_squared_log_error as msle, mean_squared_error as mse
if is_reg:
    metric=msle
else:
    metric=accuracy
FAI_model.fit(lr, n_cycle=2, wds = 1e-5, cycle_len=1, metrics=[metric])
FAI_model.fit??<jupyter_output><empty_output><jupyter_text>Other models<jupyter_code>example_batch = next(iter(data.trn_dl))
x_cat, x_cont, ys = example_batch
x_cont[0]=np.nan
x_cat, x_cont = V(x_cat[:5]), V(x_cont[:5])
# ys
# test_output = torch.tensor(np.matrix([[0,100],[0,1],[0,1],[0,1]])).double()
# F.nll_loss(test_output.cuda(), ys)
class LinReg(nn.Module):
    def __init__(self, emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False):
        super().__init__()
        self.out_sz = out_sz
        self.n_cont = n_cont
        self.lin = nn.Linear(self.n_cont, out_sz)
        nn.init.kaiming_normal_(self.lin.weight.data)

    def forward(self, x_cat, x_cont):
        return self.lin(x_cont) #torch.log(self.lin(x_cont).float()+total_shift)

    def predict(self, is_test=True):
        data_load = data.test_dl
        to_cat = []
        for batch in iter(data_load):
            x_cat, x_cont, _ = batch
            to_cat.append(self.forward(x_cat, x_cont))
        preds = torch.cat(to_cat) #torch.exp(torch.cat(to_cat))-total_shift
        return to_np(preds)

class BiInterPool_of_NFM(nn.Module):
    def __init__(self, emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False):
        super().__init__()
        self.cat_embs = nn.ModuleList([nn.Embedding(c, emb_dim) for c, s in emb_szs])
        self.cont_embs =  nn.ModuleList([nn.Embedding(2, emb_dim) for i in range(n_cont)])
        for emb in self.cat_embs: emb_init(emb)
        for emb in self.cont_embs: emb_init(emb)
        self.n_cat = len(emb_szs)
        self.n_cont = n_cont
#         self.n_feats = self.n_cat+self.n_cont
        szs = [emb_dim]+szs
        self.lins = nn.ModuleList([
            nn.Linear(szs[i], szs[i+1]) for i in range(len(szs)-1)])
        self.bns = nn.ModuleList([
            nn.BatchNorm1d(sz) for sz in szs[1:]])
        for o in self.lins: nn.init.kaiming_normal_(o.weight.data)
        self.outp = nn.Linear(szs[-1], out_sz)
        nn.init.kaiming_normal_(self.outp.weight.data)
        self.emb_drop = nn.Dropout(emb_drop)
        self.drops = nn.ModuleList([nn.Dropout(drop) for drop in drops])
        self.bn = nn.BatchNorm1d(n_cont)
        self.use_bn,self.y_range = use_bn,y_range
        self.is_reg = is_reg
        self.is_multi = is_multi
        
    def forward(self, x_cat, x_cont):
        if self.n_cat > 0:
            cat_emb_vecs = []
            for i,e in enumerate(self.cat_embs):
                cat_emb_vecs.append(e(x_cat[:, i]))
        if self.n_cont > 0:
            x_cont_emb_idxs = torch.where(torch.isnan(x_cont), torch.zeros(x_cont.shape).cuda(), torch.ones(x_cont.shape).cuda()).long()
#             print(x_cont_emb_idxs)
            cont_emb_vecs = []
            x_cont_clone = x_cont.clone()
            x_cont_clone[x_cont_clone != x_cont_clone]=0
#             print(x_cont_clone)
            for i, e in enumerate(self.cont_embs):
                cont_emb_vecs.append(torch.unsqueeze(x_cont_clone[:,i],1) * e(x_cont_emb_idxs[:, i]))
#                 print(e(x_cont_emb_idxs[:, i]))
        if self.n_cat > 0:
            list_of_vecs = cat_emb_vecs + cont_emb_vecs
        else:
            list_of_vecs = cont_emb_vecs
            
        mult_vec = []
        for vec in list_of_vecs:
            mult_vec.append(vec.unsqueeze(1))
        v_sets = torch.cat(mult_vec, 1)    
        bi_inter_pool = .5*(v_sets.sum(1)**2 - (v_sets**2).sum(1))
        x = bi_inter_pool
        for l,d,b in zip(self.lins, self.drops, self.bns):
            x = F.relu(l(x))
            if self.use_bn: x = b(x)
            x = d(x)
        x = self.outp(x)
        if not self.is_reg:
            if self.is_multi:
                x = F.sigmoid(x)
            else:
                x = F.log_softmax(x)
        elif self.y_range:
            x = F.sigmoid(x)
            x = x*(self.y_range[1] - self.y_range[0])
            x = x+self.y_range[0]
        return x #torch.log(x+total_shift)
    
    def predict(self, is_test=True):
        data_load = data.test_dl
        to_cat = []
        for batch in iter(data_load):
            x_cat, x_cont, _ = batch
            to_cat.append(self.forward(x_cat, x_cont))
        preds = torch.cat(to_cat)#torch.exp(torch.cat(to_cat))-total_shift
        return to_np(preds)
    
class Just_Concat_SimpNet(nn.Module):
    def __init__(self, emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False):
        super().__init__()
        self.cat_embs = nn.ModuleList([nn.Embedding(c, emb_dim) for c, s in emb_szs])
        self.cont_embs =  nn.ModuleList([nn.Embedding(2, emb_dim) for i in range(n_cont)])
        for emb in self.cat_embs: emb_init(emb)
        for emb in self.cont_embs: emb_init(emb)
        self.n_cat = len(emb_szs)
        self.n_cont = n_cont
        self.n_feats = self.n_cat+self.n_cont
        szs = [emb_dim*self.n_feats]+szs
        self.lins = nn.ModuleList([
            nn.Linear(szs[i], szs[i+1]) for i in range(len(szs)-1)])
        self.bns = nn.ModuleList([
            nn.BatchNorm1d(sz) for sz in szs[1:]])
        for o in self.lins: nn.init.kaiming_normal_(o.weight.data)
        self.outp = nn.Linear(szs[-1], out_sz)
        nn.init.kaiming_normal_(self.outp.weight.data)
        self.emb_drop = nn.Dropout(emb_drop)
        self.drops = nn.ModuleList([nn.Dropout(drop) for drop in drops])
        self.bn = nn.BatchNorm1d(n_cont)
        self.use_bn,self.y_range = use_bn,y_range
        self.is_reg = is_reg
        self.is_multi = is_multi
        
    def forward(self, x_cat, x_cont):
        if self.n_cat > 0:
            cat_emb_vecs = []
            for i,e in enumerate(self.cat_embs):
                cat_emb_vecs.append(e(x_cat[:, i]))
        if self.n_cont > 0:
            x_cont_emb_idxs = torch.where(torch.isnan(x_cont), torch.zeros(x_cont.shape).cuda(), torch.ones(x_cont.shape).cuda()).long()
#             print(x_cont_emb_idxs)
            cont_emb_vecs = []
            x_cont_clone = x_cont.clone()
            x_cont_clone[x_cont_clone != x_cont_clone]=0
#             print(x_cont_clone)
            for i, e in enumerate(self.cont_embs):
                cont_emb_vecs.append(torch.unsqueeze(x_cont_clone[:,i],1) * e(x_cont_emb_idxs[:, i]))
#                 print(e(x_cont_emb_idxs[:, i]))
        if self.n_cat > 0:
            list_of_vecs = cat_emb_vecs + cont_emb_vecs
        else:
            list_of_vecs = cont_emb_vecs
#         print(list_of_vecs)
#         print(torch.cat(list_of_vecs, dim=1).shape)
#         print(torch.cat(list_of_vecs, dim=1))
        
        x = torch.cat(list_of_vecs, dim=1)
        for l,d,b in zip(self.lins, self.drops, self.bns):
            x = F.relu(l(x))
            if self.use_bn: x = b(x)
            x = d(x)
        x = self.outp(x)
        if not self.is_reg:
            if self.is_multi:
                x = F.sigmoid(x)
            else:
                x = F.log_softmax(x)
        elif self.y_range:
            x = F.sigmoid(x)
            x = x*(self.y_range[1] - self.y_range[0])
            x = x+self.y_range[0]
        return x #torch.log(x+total_shift)
    
    def predict(self, is_test=True):
        data_load = data.test_dl
        to_cat = []
        for batch in iter(data_load):
            x_cat, x_cont, _ = batch
            to_cat.append(self.forward(x_cat, x_cont))
        preds = torch.cat(to_cat)#torch.exp(torch.cat(to_cat))-total_shift
        return to_np(preds)
    
class NFM(nn.Module):
    def __init__(self, emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False):
        super().__init__()
        self.lin_reg = LinReg(emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False)
        self.FM = BiInterPool_of_NFM(emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False)
        
    def forward(self, x_cat, x_cont):
        lin_part = self.lin_reg.forward(x_cat, x_cont)
        fm_part = self.FM.forward(x_cat, x_cont)
        x = lin_part + fm_part
#         import ipdb; ipdb.set_trace()
        return x
    
    def predict(self, is_test=True):
        data_load = data.test_dl
        to_cat = []
        for batch in iter(data_load):
            x_cat, x_cont, _ = batch
            to_cat.append(self.forward(x_cat, x_cont))
        preds = torch.cat(to_cat) #torch.exp(torch.cat(to_cat))-total_shift
        return to_np(preds)
    
class Lin_JSN(nn.Module):
    def __init__(self, emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False):
        super().__init__()
        self.lin_reg = LinReg(emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False)
        self.JSN = Just_Concat_SimpNet(emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False)
        
    def forward(self, x_cat, x_cont):
        lin_part = self.lin_reg.forward(x_cat, x_cont)
        fm_part = self.JSN.forward(x_cat, x_cont)
        x = lin_part + fm_part
#         import ipdb; ipdb.set_trace()
        return x
    
    def predict(self, is_test=True):
        data_load = data.test_dl
        to_cat = []
        for batch in iter(data_load):
            x_cat, x_cont, _ = batch
            to_cat.append(self.forward(x_cat, x_cont))
        preds = torch.cat(to_cat) #torch.exp(torch.cat(to_cat))-total_shift
        return to_np(preds)    
    

# lin_reg_model = LinReg(emb_dim=emb_dim, emb_szs=emb_szs, n_cont=2, emb_drop=.05,
#                     out_sz=out_sz, szs=szs, drops=[.1,.2,.3,.4], use_bn=True, is_reg=is_reg)

# BIP_model = BiInterPool_of_NFM(emb_dim=emb_dim, emb_szs=emb_szs, n_cont=2, emb_drop=.05,
#                     out_sz=out_sz, szs=szs, drops=[.1,.2,.3,.4], use_bn=True, is_reg=is_reg)

jcsimpnet_model = Just_Concat_SimpNet(emb_dim=emb_dim, emb_szs=emb_szs, n_cont=2, emb_drop=.05,
                    out_sz=out_sz, szs=szs, drops=[.1,.2,.3,.4], use_bn=True, is_reg=is_reg)

# NFM_model = NFM(emb_dim=emb_dim, emb_szs=emb_szs, n_cont=2, emb_drop=.05,
#                     out_sz=out_sz, szs=szs, drops=[.1,.2,.3,.4], use_bn=True, is_reg=is_reg)

# Lin_JSN_model = Lin_JSN(emb_dim=emb_dim, emb_szs=emb_szs, n_cont=2, emb_drop=.05,
#                     out_sz=out_sz, szs=szs, drops=[.1,.2,.3,.4], use_bn=True, is_reg=is_reg)

# lin_reg_model.cuda()
# BIP_model.cuda()
jcsimpnet_model.cuda()
# NFM_model.cuda()
# Lin_JSN_model.cuda()
# model.forward(x_cat, x_cont)
# list(model.parameters())
lin_reg_opt = optim.Adam(lin_reg_model.parameters(), 1e-2)
BIP_opt = optim.Adam(BIP_model.parameters(), 1e-2)
jcsimpnet_opt = optim.Adam(jcsimpnet_model.parameters(), 1e-2)
NFM_opt = optim.Adam(NFM_model.parameters(), 1e-2)
Lin_JSN_opt = optim.Adam(Lin_JSN_model.parameters(), 1e-2)
if is_reg:
    metric = F.mse_loss
else:
    metric = F.nll_loss
# fit(lin_reg_model, data, 2, lin_reg_opt, F.mse_loss)
# fit(BIP_model, data, 2, BIP_opt, F.mse_loss)
fit(jcsimpnet_model, data, 2, jcsimpnet_opt, metric)
# fit(NFM_model, data, 2, NFM_opt, F.mse_loss)
# fit(Lin_JSN_model, data, 2, Lin_JSN_opt, metric)<jupyter_output><empty_output><jupyter_text>Validation<jupyter_code>FAI_preds = FAI_model.predict(is_test=True)
# lin_reg_preds = lin_reg_model.predict(is_test=True)
# BIP_preds = BIP_model.predict(is_test=True)
jcsimpnet_preds = jcsimpnet_model.predict(is_test=True)
# NFM_preds = NFM_model.predict(is_test=True)
# Lin_JSN_preds = Lin_JSN_model.predict(is_test=True)
accuracy(V(np.exp(FAI_preds)), V(y))
accuracy(V(np.exp(jcsimpnet_preds)), V(y))
FAI_msle = msle(y, FAI_preds)
FAI_mse = mse(y, FAI_preds)

lin_reg_msle = msle(y, lin_reg_preds)
lin_reg_mse = mse(y, lin_reg_preds)

BIP_msle = msle(y, BIP_preds)
BIP_mse = mse(y, BIP_preds)

jcsimpnet_msle = msle(y, jcsimpnet_preds)
jcsimpnet_mse = mse(y, jcsimpnet_preds)

NFM_msle = msle(y, NFM_preds)
NFM_mse = mse(y, NFM_preds)

Lin_JSN_msle = msle(y, Lin_JSN_preds)
Lin_JSN_mse = mse(y, Lin_JSN_preds)
print("mean squarred log errors:")
print("FAI: {0}, \nlin_reg: {1}, \nBIP: {2} \njcsimpnet: {3} \nNFM: {4} \nLin_JSN: {5}".format(FAI_msle, lin_reg_msle, BIP_msle, jcsimpnet_msle, NFM_msle, Lin_JSN_msle))
print(" ")
print("mean squarred errors")
print("FAI: {0}, \nlin_reg: {1}, \nBIP: {2} \njcsimpnet: {3} \nNFM: {4} \nLin_JSN: {5}".format(FAI_mse, lin_reg_mse, BIP_mse, jcsimpnet_mse, NFM_mse, Lin_JSN_mse))
# FAI version
%matplotlib notebook
if is_reg:
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_wireframe(X = X, Y = Y, Z = FAI_preds.reshape(X.shape))
else:
    print(accuracy(torch.tensor(preds), torch.tensor(y)))
# just lin_reg portion
%matplotlib notebook
if is_reg:
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_wireframe(X = X, Y = Y, Z = lin_reg_preds.reshape(X.shape))
else:
    print(accuracy(torch.tensor(preds), torch.tensor(y)))
# just the bipoolfm portion emb_dim = 20
%matplotlib notebook
if is_reg:
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_wireframe(X = X, Y = Y, Z = BIP_preds.reshape(X.shape))
else:
    print(accuracy(torch.tensor(preds), torch.tensor(y)))
# jcsimpnet
%matplotlib notebook
if is_reg:
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_wireframe(X = X, Y = Y, Z = jcsimpnet_preds.reshape(X.shape))
else:
    print(accuracy(torch.tensor(preds), torch.tensor(y)))
# full NFM
%matplotlib notebook
if is_reg:
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_wireframe(X = X, Y = Y, Z = NFM_preds.reshape(X.shape))
else:
    print(accuracy(torch.tensor(preds), torch.tensor(y)))
# Lin_JSN
%matplotlib notebook
if is_reg:
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_wireframe(X = X, Y = Y, Z = Lin_JSN_preds.reshape(X.shape))
else:
    print(accuracy(torch.tensor(preds), torch.tensor(y)))
# actual data
%matplotlib notebook
if is_reg:
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_wireframe(X = X, Y = Y, Z = y.reshape(X.shape))
else:
    print(accuracy(torch.tensor(preds), torch.tensor(y)))<jupyter_output><empty_output><jupyter_text>Error Analysis```Rules:if df['cat1'] == 1: error 1    class = 1 elif df['cont1'] < df['cont2']: error 2    class = 1elif: (df['cat2'] == 0) & (df['cont1'] + df['cont2'] >=1.2): error 3    class = 1else: error 4    class = 0```      training error analysis<jupyter_code>preds
# on train set
train_idxs = list(set(np.arange(len(df))).difference(set(val_idxs)))
train_preds = np.exp(MIM.predict(is_test=True))[train_idxs]
ex_train_preds = pd.DataFrame({'preds':np.round(train_preds[:,1]),
                             'y': y[train_idxs]})
print('train accuracy: {0}'.format((ex_train_preds['preds'] == ex_train_preds['y']).sum()/len(ex_train_preds)))
train = df.loc[train_idxs]
train['preds'] = ex_train_preds['preds'].values
train_wrong = train[train['preds'] != train['y']]
# train_wrong = df.loc[train_wrong.index,:]
train_wrong.head()
def get_error(df):
    return np.where((df['cat1'] == 0) & (df['preds'] == 0), 1,
                   np.where((df['cont1'] < df['cont2']) & (df['preds'] == 0), 2, 
                            np.where(((df['cat2'] == 0) & (df['cont1'] + df['cont2'] >=1.2)) & (df['preds'] == 0), 3, 4)))
train_wrong['error_type'] = get_error(train_wrong)<jupyter_output>/home/justin/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  """<jupyter_text>```Rules:if df['cat1'] == 1: error 1    class = 1 elif df['cont1'] < df['cont2']: error 2    class = 1elif: (df['cat2'] == 0) & (df['cont1'] + df['cont2'] >=1.2): error 3    class = 1else: error 4    class = 0```<jupyter_code>train_wrong['error_type'].value_counts(dropna=False)<jupyter_output><empty_output><jupyter_text>validation analysis<jupyter_code># val_preds = np.exp(MIM.predict())
val_preds = predict(fm, data.val_dl)
val_preds = np.exp(val_preds)
ex_val_preds = pd.DataFrame({'preds':np.round(val_preds[:,1]),
                             'y': y[val_idxs]})
(ex_val_preds['preds'] == ex_val_preds['y']).sum()/len(ex_val_preds)
ex_val_preds.head()
log_loss(y[val_idxs], val_preds)
def accuracy(y, val_idxs, val_preds):
    return np.equal(y[val_idxs],np.round(val_preds[:,1])).sum()/len(val_idxs)
accuracy(y, val_idxs, val_preds)
import matplotlib.pyplot as plt
from sklearn.metrics import (confusion_matrix, precision_recall_curve, auc,
                             roc_curve, recall_score, classification_report, f1_score,
                             precision_recall_fscore_support)

fpr, tpr, thresholds = roc_curve(y[val_idxs], val_preds[:,1])
roc_auc = auc(fpr, tpr)

plt.title('Receiver Operating Characteristic')
plt.plot(fpr, tpr, label='AUC = %0.4f'% roc_auc)
plt.legend(loc='lower right')
plt.plot([0,1],[0,1],'r--')
plt.xlim([-0.001, 1])
plt.ylim([0, 1.001])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show();
print(roc_auc)<jupyter_output><empty_output><jupyter_text>Stuff to work with later<jupyter_code>class DeepFM(nn.Module):
    def __init__(self, emb_szs, cont_emb_dim, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False):
        super().__init__()
        self.embs = nn.ModuleList([nn.Embedding(c, s) for c, s in emb_szs])
        self.cont_embs = nn.ModuleList(
            [nn.Embedding(1, cont_emb_dim) for i in np.arange(n_cont)])
        for emb in self.embs:
            emb_init(emb)
        n_emb = sum(e.embedding_dim for e in self.embs)
        self.n_emb, self.n_cont = n_emb, n_cont

        szs = [cont_emb_dim] + szs
        self.lins = nn.ModuleList([
            nn.Linear(szs[i], szs[i+1]) for i in range(len(szs)-1)])
        self.bns = nn.ModuleList([
            nn.BatchNorm1d(sz) for sz in szs[1:]])
        for o in self.lins:
            kaiming_normal(o.weight.data)
        self.outp = nn.Linear(szs[-1], out_sz)
        kaiming_normal(self.outp.weight.data)

        self.emb_drop = nn.Dropout(emb_drop)
        self.drops = nn.ModuleList([nn.Dropout(drop) for drop in drops])
        self.bn = nn.BatchNorm1d(n_cont)
        self.use_bn, self.y_range = use_bn, y_range
        self.is_reg = is_reg
        self.is_multi = is_multi

    def forward(self, x_cat, x_cont):
        # embedding and bi_inter_pool_vec
        if self.n_emb != 0:
            emb_vecs = [e(x_cat[:, i]) for i, e in enumerate(self.embs)]
        if self.n_cont != 0:
            cont_emb_vecs = [x_cont[:, i].unsqueeze(
                1)*e(V(torch.zeros(x_cont.shape[0]).long())) for i, e in enumerate(self.cont_embs)]
        all_emb_vecs = emb_vecs + cont_emb_vecs
        bi_inter_pool_vec = V(torch.ones(all_emb_vecs[0].size()))
        for combo in itertools.combinations(all_emb_vecs, 2):
            bi_inter_pool_vec += combo[0]*combo[1]
        # deep portion

        for l, d, b in zip(self.lins, self.drops, self.bns):
            bi_inter_pool_vec = F.relu(l(bi_inter_pool_vec))
            if self.use_bn:
                bi_inter_pool_vec = b(bi_inter_pool_vec)
            bi_inter_pool_vec = d(bi_inter_pool_vec)
        bi_inter_pool_vec = self.outp(bi_inter_pool_vec)
        if not self.is_reg:
            if self.is_multi:
                bi_inter_pool_vec = F.sigmoid(bi_inter_pool_vec)
            else:
                bi_inter_pool_vec = F.log_softmax(bi_inter_pool_vec)
        elif self.y_range:
            bi_inter_pool_vec = F.sigmoid(bi_inter_pool_vec)
            bi_inter_pool_vec = bi_inter_pool_vec * \
                (self.y_range[1] - self.y_range[0])
            bi_inter_pool_vec = bi_inter_pool_vec+self.y_range[0]
        return bi_inter_pool_vec<jupyter_output><empty_output><jupyter_text>reference stuff to probably delete<jupyter_code>data.get_learner(emb_szs=emb_szs, n_cont=2, emb_drop=.05, out_sz=out_sz, szs = [50, 50, 50, 25], drops = [.1, .2, .3, .4], use_bn=True)

class MixedInputModel(nn.Module):
    def __init__(self, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False):
        super().__init__()
        self.embs = nn.ModuleList([nn.Embedding(c, s) for c,s in emb_szs])
        for emb in self.embs: emb_init(emb)
        n_emb = sum(e.embedding_dim for e in self.embs)
        self.n_emb, self.n_cont=n_emb, n_cont
        
        szs = [n_emb+n_cont] + szs
        self.lins = nn.ModuleList([
            nn.Linear(szs[i], szs[i+1]) for i in range(len(szs)-1)])
        self.bns = nn.ModuleList([
            nn.BatchNorm1d(sz) for sz in szs[1:]])
        for o in self.lins: kaiming_normal(o.weight.data)
        self.outp = nn.Linear(szs[-1], out_sz)
        kaiming_normal(self.outp.weight.data)

        self.emb_drop = nn.Dropout(emb_drop)
        self.drops = nn.ModuleList([nn.Dropout(drop) for drop in drops])
        self.bn = nn.BatchNorm1d(n_cont)
        self.use_bn,self.y_range = use_bn,y_range
        self.is_reg = is_reg
        self.is_multi = is_multi

    def forward(self, x_cat, x_cont):
        if self.n_emb != 0:
            x = [e(x_cat[:,i]) for i,e in enumerate(self.embs)]
            x = torch.cat(x, 1)
            x = self.emb_drop(x)
        if self.n_cont != 0:
            x2 = self.bn(x_cont)
            x = torch.cat([x, x2], 1) if self.n_emb != 0 else x2
        for l,d,b in zip(self.lins, self.drops, self.bns):
            x = F.relu(l(x))
            if self.use_bn: x = b(x)
            x = d(x)
        x = self.outp(x)
        if not self.is_reg:
            if self.is_multi:
                x = F.sigmoid(x)
            else:
                x = F.log_softmax(x)
        elif self.y_range:
            x = F.sigmoid(x)
            x = x*(self.y_range[1] - self.y_range[0])
            x = x+self.y_range[0]
        return x
    
class FM(nn.Module):
    def __init__(self, cat_levels, emb_dim, n_cat, n_cont, out_sz):
        super().__init__()
        self.emb_dim, self.n_cat, self.n_cont = emb_dim, n_cat, n_cont
        self.cat_embs = nn.ModuleList([get_emb(v, emb_dim) for k, v in cat_levels])
        self.cont_embs = nn.ModuleList([get_emb(2, emb_dim) for i in np.arange(n_cont)])
        self.beg_n_feat = self.n_cat+self.n_cont
        self.fin_n_feat = int(scipy.special.comb(self.beg_n_feat,2))
        self.out_sz = out_sz
#         self.outp = nn.Linear(self.fin_n_feat, out_sz)
#         self.hl1 = nn.Linear(self.n_cat * self.emb_dim + self.n_cont, 100)
        self.hl1 = nn.Linear(self.beg_n_feat * self.emb_dim, 100)
        nn.init.kaiming_normal_(self.hl1.weight.data)
        self.bn1 = nn.BatchNorm1d(100)
        self.outp = nn.Linear(100, out_sz)

    def forward(self, x_cat, x_cont):
        if self.n_cat > 0:
            cat_emb_vecs = []
            for i,e in enumerate(self.cat_embs):
                cat_emb_vecs.append(e(x_cat[:, i]))
#             x1 = torch.cat(cat_emb_vecs,1)
        if self.n_cont > 0:
#             cont_emb_vecs = [x_cont]
#             x2 = x_cont
            x_cont_emb_idxs = torch.where(torch.isnan(x_cont), torch.zeros(x_cont.shape).cuda(), torch.ones(x_cont.shape).cuda()).long()
            cont_emb_vecs = []
            for i, e in enumerate(self.cont_embs):
                x_cont_clone = x_cont.clone()
                x_cont_clone[x_cont_clone != x_cont_clone]=0
                cont_emb_vecs.append(torch.unsqueeze(x_cont_clone[:,i],1) * e(x_cont_emb_idxs[:, i]))
        list_of_vecs = cat_emb_vecs + cont_emb_vecs
#         base_vec = torch.cat([x1, x2], 1)

#         base_vec = torch.cat(list_of_vecs,1)
        mult_vec = []
        for vec in list_of_vecs:
            mult_vec.append(vec.unsqueeze(1))
        v_sets = torch.cat(mult_vec, 1)
        import ipdb; ipdb.set_trace()
        to_concat = []
        for i in np.arange(self.beg_n_feat-1):
            temp_tensor = V(torch.matmul(v_sets[:,i,:].unsqueeze(1), v_sets[:,i+1:,: ].unsqueeze(1).permute(0,2,1)))
            tt_shape = temp_tensor.shape
            to_concat.append(temp_tensor.resize_(tt_shape[0], tt_shape[2]))
        base_vec = torch.cat(to_concat,1)

#         print(list(self.parameters()))
#         print(base_vec)

        base_vec = self.bn1(F.relu(self.hl1(base_vec)))
        return F.log_softmax(self.outp(base_vec))    

class FM(nn.Module):
    def __init__(self, cat_levels, emb_dim, n_cat, n_cont, out_sz):
        super().__init__()
        self.emb_dim, self.n_cat, self.n_cont = emb_dim, n_cat, n_cont
        self.cat_embs = nn.ModuleList([get_emb(v, emb_dim) for k, v in cat_levels])
        self.cont_embs = nn.ModuleList([get_emb(2, emb_dim) for i in np.arange(n_cont)])
        self.beg_n_feat = self.n_cat+self.n_cont
        self.fin_n_feat = int(scipy.special.comb(self.beg_n_feat,2))
        self.out_sz = out_sz
#         self.outp = nn.Linear(self.fin_n_feat, out_sz)
#         self.hl1 = nn.Linear(self.n_cat * self.emb_dim + self.n_cont, 100)
        self.hl1 = nn.Linear(self.beg_n_feat * self.emb_dim, 100)
        nn.init.kaiming_normal_(self.hl1.weight.data)
        self.bn1 = nn.BatchNorm1d(100)
        self.outp = nn.Linear(100, out_sz)

    def forward(self, x_cat, x_cont):
        if self.n_cat > 0:
            cat_emb_vecs = []
            for i,e in enumerate(self.cat_embs):
                cat_emb_vecs.append(e(x_cat[:, i]))
#             x1 = torch.cat(cat_emb_vecs,1)
        if self.n_cont > 0:
#             cont_emb_vecs = [x_cont]
#             x2 = x_cont
            x_cont_emb_idxs = torch.where(torch.isnan(x_cont), torch.zeros(x_cont.shape).cuda(), torch.ones(x_cont.shape).cuda()).long()
            cont_emb_vecs = []
            for i, e in enumerate(self.cont_embs):
                x_cont_clone = x_cont.clone()
                x_cont_clone[x_cont_clone != x_cont_clone]=0
                cont_emb_vecs.append(torch.unsqueeze(x_cont_clone[:,i],1) * e(x_cont_emb_idxs[:, i]))
        list_of_vecs = cat_emb_vecs + cont_emb_vecs
#         base_vec = torch.cat([x1, x2], 1)

#         base_vec = torch.cat(list_of_vecs,1)
        mult_vec = []
        for vec in list_of_vecs:
            mult_vec.append(vec.unsqueeze(1))
        v_sets = torch.cat(mult_vec, 1)
        import ipdb; ipdb.set_trace()
        to_concat = []
        for i in np.arange(self.beg_n_feat-1):
            temp_tensor = V(torch.matmul(v_sets[:,i,:].unsqueeze(1), v_sets[:,i+1:,: ].unsqueeze(1).permute(0,2,1)))
            tt_shape = temp_tensor.shape
            to_concat.append(temp_tensor.resize_(tt_shape[0], tt_shape[2]))
        base_vec = torch.cat(to_concat,1)

#         print(list(self.parameters()))
#         print(base_vec)

        base_vec = self.bn1(F.relu(self.hl1(base_vec)))
        return F.log_softmax(self.outp(base_vec))
class Embed_All(nn.Module):
    def __init__(self, emb_dim, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
                 y_range=None, use_bn=False, is_reg=True, is_multi=False):
        super().__init__()
        self.cat_embs = nn.ModuleList([nn.Embedding(c, emb_dim) for c, s in emb_szs])
        self.cont_embs =  nn.ModuleList([nn.Embedding(2, emb_dim) for i in range(n_cont)])
        for emb in self.cat_embs: emb_init(emb)
        for emb in self.cont_embs: emb_init(emb)
        self.n_cat = len(emb_szs)
        self.n_cont = n_cont
        self.n_feats = self.n_cat+self.n_cont
        szs = [emb_dim*self.n_feats]+szs
        self.lins = nn.ModuleList([
            nn.Linear(szs[i], szs[i+1]) for i in range(len(szs)-1)])
        self.bns = nn.ModuleList([
            nn.BatchNorm1d(sz) for sz in szs[1:]])
        for o in self.lins: nn.init.kaiming_normal_(o.weight.data)
        self.outp = nn.Linear(szs[-1], out_sz)
        nn.init.kaiming_normal_(self.outp.weight.data)
        self.emb_drop = nn.Dropout(emb_drop)
        self.drops = nn.ModuleList([nn.Dropout(drop) for drop in drops])
        self.bn = nn.BatchNorm1d(n_cont)
        self.use_bn,self.y_range = use_bn,y_range
        self.is_reg = is_reg
        self.is_multi = is_multi

    def forward(self, x_cat, x_cont):
        if self.n_emb != 0:
            x = [e(x_cat[:,i]) for i,e in enumerate(self.embs)]
            x = torch.cat(x, 1)
            x = self.emb_drop(x)
#         if self.n_cont != 0:
#             x2 = self.bn(x_cont)
#             x = torch.cat([x, x2], 1) if self.n_emb != 0 else x2
        if self.n_cont > 0:
            x_cont_emb_idxs = torch.where(torch.isnan(x_cont), torch.zeros(x_cont.shape).cuda(), torch.ones(x_cont.shape).cuda()).long()
#             print(x_cont_emb_idxs)
            cont_emb_vecs = []
            x_cont_clone = x_cont.clone()
            x_cont_clone[x_cont_clone != x_cont_clone]=0
#             print(x_cont_clone)
            for i, e in enumerate(self.cont_embs):
                cont_emb_vecs.append(torch.unsqueeze(x_cont_clone[:,i],1) * e(x_cont_emb_idxs[:, i]))
        if self.n_cat > 0:
            list_of_vecs = cat_emb_vecs + cont_emb_vecs
        else:
            list_of_vecs = cont_emb_vecs
#         print(list_of_vecs)
#         print(torch.cat(list_of_vecs, dim=1).shape)
#         print(torch.cat(list_of_vecs, dim=1))
        
        x = torch.cat(list_of_vecs, dim=1)
            
            
        for l,d,b in zip(self.lins, self.drops, self.bns):
            x = F.relu(l(x))
            if self.use_bn: x = b(x)
            x = d(x)
        x = self.outp(x)
        if not self.is_reg:
            if self.is_multi:
                x = F.sigmoid(x)
            else:
                x = F.log_softmax(x)
        elif self.y_range:
            x = F.sigmoid(x)
            x = x*(self.y_range[1] - self.y_range[0])
            x = x+self.y_range[0]
        return x<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13803131991051454, 'fraction_numerical': 0.013390859699584532, 'mean_word_length': 2.666198008201523, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '213029', 'license': 'MIT', 'n_tokens_mistral': 12654, 'n_tokens_neox': 12454, 'n_words': 2075, 'path': 'notebooks/toy_understanding_factorziation_machine.ipynb', 'repo_name': 'jmhsi/toy_datascience_notebooks', 'size': '31290'}