<jupyter_start><jupyter_text>Dataset Split based on displayed offers for target users and timeline<jupyter_code>user_list=np.unique(df.user_id)
test=[]
train=[]
for i in tqdm_notebook(range(len(user_list))):
    df_split = np.array_split(df[df.user_id==user_list[i]].sort_values(by='week'), 3)
    train.append(list(df_split[0].index))
    train.append(list(df_split[1].index))
    test.append(list(df_split[2].index))
    
    
train=list(itertools.chain(*train))
test=list(itertools.chain(*test))

X_train=df[df.index.isin(train)]
X_test=df[df.index.isin(test)]

X_train.set_index(['user_id','item_id'], inplace=True)
X_test.set_index(['user_id','item_id'], inplace=True)

Y_train_click=X_train.click
Y_train_book=X_train.book
Y_train_reply=X_train.reply
Y_test_click=X_test.click
Y_test_book=X_test.book
Y_test_reply=X_test.reply
col=[col for col in df.columns if col not in ['displayed','user_id','item_id',"click","book","reply","week"]]
X_train=X_train[col]
X_test=X_test[col]<jupyter_output><empty_output><jupyter_text>Dataset Normilization<jupyter_code>X_train = X_train.astype(np.float64)
X_test = X_test.astype(np.float64)

min_max_scaler = preprocessing.MinMaxScaler()
X_train_minmax = min_max_scaler.fit_transform(X_train.iloc[:,15:])
X_test_minmax = min_max_scaler.transform(X_test.iloc[:,15:])

X_train_minmax = pd.DataFrame(X_train_minmax)
X_test_minmax = pd.DataFrame(X_test_minmax)
X_train_minmax.columns = X_train.iloc[:,15:].columns
X_test_minmax.columns = X_test.iloc[:,15:].columns
X_train_minmax.index = X_train.iloc[:,15:].index
X_test_minmax.index = X_test.iloc[:,15:].index
X_test_minmax
sc=StandardScaler()
sc.fit(X_train_minmax)
X_train_std=sc.transform(X_train_minmax)
X_test_std=sc.transform(X_test_minmax)

X_train_std = pd.DataFrame(X_train_std)
X_test_std = pd.DataFrame(X_test_std)
X_train_std.columns = X_train.iloc[:,15:].columns
X_test_std.columns = X_test.iloc[:,15:].columns
X_train_std.index = X_train.iloc[:,15:].index
X_test_std.index = X_test.iloc[:,15:].index
X_test_std<jupyter_output><empty_output><jupyter_text>Union of extracted features + normolized contextual<jupyter_code>U=np.unique(X_train_std.reset_index().user_id)
I=np.unique(X_train_std.reset_index().item_id)
from random import randint
import random
import copy
def sample_floats(low, high, k=1):
    """ Return a k-length list of unique random floats
        in the range of low <= x <= high
    """
    result = []
    seen = set()
    for i in range(k):
        x = random.uniform(low, high)
        while x in seen:
            x = random.uniform(low, high)
        seen.add(x)
        result.append(x)
    return result
U_repr = {U[i]:[v for v in sample_floats(0,1,k=X_train.shape[1])] for i in range (len(U))}
I_repr= {I[j]:[w for w in sample_floats(0,1,k=X_train.shape[1])] for j in range (len(I))}

Y_train_click=Y_train_click.reset_index()
Y_train_book=Y_train_book.reset_index()
Y_train_reply=Y_train_reply.reset_index()
X_train_std=X_train_std.reset_index()
X_test_std=X_test_std.reset_index()
Y_test_click=Y_test_click.reset_index()
Y_test_book=Y_test_book.reset_index()
Y_test_reply=Y_test_reply.reset_index()
X_train_dict={}
for i in tqdm_notebook(xrange(len(X_train_std))):
    X_train_dict[X_train_std.user_id[i],X_train_std.item_id[i]]=X_train.iloc[i,:15].values.tolist()+\
    X_train_std.iloc[i,2:].values.tolist()
#     +U_repr[X_train_std.user_id[i]]+I_repr[X_train_std.item_id[i]]
Y_train_dict_click={}
Y_train_dict_book={}
Y_train_dict_reply={}
for i in tqdm_notebook(xrange(len(Y_train_click))):
    Y_train_dict_click[Y_train_click.user_id[i],Y_train_click.item_id[i]]=Y_train_click.iloc[i,2]
    Y_train_dict_book[Y_train_book.user_id[i],Y_train_book.item_id[i]]=Y_train_book.iloc[i,2]
    Y_train_dict_reply[Y_train_reply.user_id[i],Y_train_reply.item_id[i]]=Y_train_reply.iloc[i,2]<jupyter_output><empty_output><jupyter_text>Stochastic gradient descent<jupyter_code>indx_dict={}
for i in range(len(U)):
    indx_dict[U[i]]=X_train_std[X_train_std.user_id==U[i]].iloc[:,:2].values.tolist()
#Inputs:
eta=0.0001
eps=1e-3
lamb=1e-3
nb_runs = 600000
#Initialization:

global_loss = [1,0]
nb=1
while abs(global_loss[nb-1]-global_loss[nb])>eps:
    loss_local=0
    for j in tqdm_notebook(range(nb_runs)):
        u = randint(0,len(U)-1)
        
        target_samples=indx_dict[U[u]]
        target_lables=[Y_train_dict_click[tuple(elem)] for elem in target_samples]
        positive=[target_samples[m] for m in [l for l, e in enumerate(target_lables) if e != 0]]
        negative=[target_samples[m] for m in [l for l, e in enumerate(target_lables) if e == 0]]

        if (len(positive)!=0 and len(negative)!=0): 

            p=randint(0, len(positive)-1)
            n=randint(0, len(negative)-1)

            user=U_repr[U[u]]
            item_plus = I_repr[positive[p][1]]
            item_minus = I_repr[negative[n][1]]
            
            diff=(np.array(item_minus) - np.array(item_plus)).tolist()
            ex=np.exp(np.dot(user,diff))
            dl_4_user =  diff / ( 1 + (1. / ex ) )
            dl_4_item_plus = ( (-1) * np.array( user ) ) / ( 1 + (1. / ex ) )
            dl_4_item_minus = ( user ) / ( 1 + (1. / ex ) )

                
            user = user - eta * ( ( dl_4_user ) + ( 2 * lamb * np.sum( user ) ) ) 

            item_plus = item_plus - eta * ( ( dl_4_item_plus ) + ( 2 * lamb * np.sum( item_plus ) ) )

            item_minus = item_minus - eta * ( ( dl_4_item_minus ) + ( 2 * lamb *np.sum( item_minus ) ) )

            loss_local=loss_local+(np.log( 1. + ex))+\
            lamb*(np.linalg.norm(user,2)**2+np.linalg.norm(item_plus,2)**2+np.linalg.norm(item_minus,2)**2)

            U_repr[U[u]]=user.tolist()
            I_repr[positive[p][1]]=item_plus.tolist()
            I_repr[negative[n][1]]=item_minus.tolist()

    global_loss.append(loss_local/nb_runs)
    nb+=1
    print(global_loss)<jupyter_output><empty_output><jupyter_text>Ranking<jupyter_code>U_pandas=pd.DataFrame(U_repr.items())
I_pandas=pd.DataFrame(I_repr.items())
U_pandas=U_pandas.set_index(0)
U_pandas.index.name="user_id"
I_pandas=I_pandas.set_index(0)
I_pandas.index.name="item_id"
U_pandas=U_pandas[1].apply(pd.Series)
cols=['U_'+str(i) for i in range (31)]
U_pandas.columns=cols
I_pandas=I_pandas[1].apply(pd.Series)
cols=['I_'+str(i) for i in range (31)]
I_pandas.columns=cols
X_final=X_test_std
I_test=np.unique(X_test_std.item_id)
main_list = np.setdiff1d(I_test,I)
random_items={}
for i in range(len (main_list)):
    random_items[main_list[i]]=[v for v in sample_floats(0,1,k=31)]
    
    
    
pd_scores=[]
for i in tqdm_notebook(range(len(X_final))):
    try:
        pd_scores.append(np.dot(U_pandas[U_pandas.index==X_final.iloc[i,0]].values.tolist()[0],\
                    I_pandas[I_pandas.index==X_final.iloc[i,1]].values.tolist()[0]))
    except:
        pd_scores.append(np.dot(U_pandas[U_pandas.index==X_final.iloc[i,0]].values.tolist()[0],\
                    random_items[X_final.iloc[i,1]]))
X_final['scores']=pd_scores
X_final=pd.merge(X_final,Y_test_click,how='left',on=['user_id','item_id'])
k=5
total_map=[]
for i in tqdm_notebook(range (len(U))):
    target_list=X_final[X_final.user_id==U[i]].sort_values(by='scores',ascending=False)
    #     target_list=target_list[target_list.Y_test==1]
    up=1
    down=1
    precision=[]
    for j,score in enumerate(target_list.click[:k].values.tolist()):
        if score!=0:
            precision.append(float(up)/float(down+j))
            up+=1
        else:
            precision.append(0)
    total_map.append(np.sum(precision[0:k])/k)
np.mean(total_map)<jupyter_output><empty_output><jupyter_text>Classification<jupyter_code>Class_train={}
for i in tqdm_notebook(range(len(X_train_std))):
        Class_train[X_train_std.user_id[i],X_train_std.item_id[i]]=U_repr[X_train_std.user_id[i]]+\
        I_repr[X_train_std.item_id[i]]
#         +list(Y_train_click[(Y_train_click.user_id==X_train_std.user_id[i])&(Y_train_click.item_id==X_train_std.item_id[i])].click)

Class_train=pd.DataFrame(Class_train.items())
col=["U_repr_"+str(i) for i in range(31)]+["I_repr_"+str(i) for i in range(31)]
Class_train[['user_id','item_id']] = pd.DataFrame(Class_train[0].values.tolist(), index= Class_train.index)
Class_train[['user_id','item_id']] = pd.DataFrame(Class_train[0].values.tolist(), index= Class_train.index)
Class_train[col] = pd.DataFrame(Class_train[1].values.tolist(), index= Class_train.index)
Class_train.iloc[:,4:]
Class_train=pd.merge(Class_train,Y_train_click,how="left",on=["user_id","item_id"])
lr = LogisticRegressionCV(penalty='l2',class_weight='balanced')
lr.fit(Class_train.iloc[:,4:-1], Class_train.iloc[:,-1]) 
Y_test_click_dic={}
for i in tqdm_notebook(xrange(len(Y_test_click))):
    Y_test_click_dic[Y_test_click.user_id[i],Y_test_click.item_id[i]]=Y_test_click.iloc[i,2]
    
X_final=X_test_std
Class={}
for i in tqdm_notebook(range(len(X_final))):
    try:
        Class[X_final.user_id[i],X_final.item_id[i]]=U_repr[X_final.user_id[i]]+\
        I_repr[X_final.item_id[i]]+\
        [Y_test_click_dic[X_final.user_id[i],X_final.item_id[i]]]
    except:
        Class[X_final.user_id[i],X_final.item_id[i]]=U_repr[X_final.user_id[i]]+\
        random_items[X_final.item_id.values.tolist()[i]]+\
        [Y_test_click_dic[X_final.user_id[i],X_final.item_id[i]]]
       
col=col+['Y_test_click']
Class_final=pd.DataFrame(Class.items())
Class_final[['user_id','item_id']] = pd.DataFrame(Class_final[0].values.tolist(), index= Class_final.index)
Class_final[col] = pd.DataFrame(Class_final[1].values.tolist(), index= Class_final.index)
Class_final
Y_pred=lr.predict(Class_final.iloc[:,4:-1])
print(classification_report(Class_final.iloc[:,-1], Y_pred))
confusion_matrix(Class_final.iloc[:,-1], Y_pred)
false_positive_rate, recall, thresholds = roc_curve(Y_pred,Class_final.iloc[:,-1])
roc_auc = auc(false_positive_rate, recall)
plt.title('Receiver Operating Characteristic')
plt.plot(false_positive_rate, recall, 'b', label='AUC = %0.2f' % roc_auc)
plt.legend(loc='lower right')
plt.plot([0, 1], [0, 1], 'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.ylabel('Recall')
plt.xlabel('Fall-out')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16388288465317352, 'fraction_numerical': 0.015916683041854982, 'mean_word_length': 5.199147381242387, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '567394', 'license': 'MIT', 'n_tokens_mistral': 4374, 'n_tokens_neox': 4281, 'n_words': 490, 'path': '6_Representation_Learning .ipynb', 'repo_name': 'asarbaev/Multi-Target-learning', 'size': '10178'}