<jupyter_start><jupyter_text>Load amazon dataset<jupyter_code>import os
import sys
import pandas as pd
import gzip
import numpy as np

def parse(path):
    g = gzip.open(path, 'rb')
    for l in g:
        yield eval(l)

def getDF(path):
    i = 0
    df = {}
    for d in parse(path):
        df[i] = d
        i += 1
    return pd.DataFrame.from_dict(df, orient='index')

df = getDF(path+'reviews_Clothing_Shoes_and_Jewelry_5.json.gz')

df.head(3)
usernum = df.reviewerID.nunique()
itemnum = df.asin.nunique()
print('%sUsers %sItems'%(usernum, itemnum))<jupyter_output>39387Users 23033Items<jupyter_text>Raw_Id to Inner_Id(reviewerID, asin)<jupyter_code># dict 만들기
reviewerID_dict , asin_dict = {},{}

for n,i in enumerate(df.reviewerID.unique()):
    reviewerID_dict[i] = str(n)

for n,i in enumerate(df.asin.unique()):
    asin_dict[i] = str(n)
    
# one-hot
df.reviewerID = df.reviewerID.apply(lambda x: reviewerID_dict[x])
df.asin=df.asin.apply(lambda x : asin_dict[x])

df.head(4)
df1=df[['reviewerID','asin','overall','unixReviewTime']]
df1.head(3)<jupyter_output><empty_output><jupyter_text>Create implicit dataset  Split train, validation, test set- Most recent history --> test- next most --> val- else --> train<jupyter_code>train_dict, val_dict, test_dict = {},{},{}

for i in range(len(df1.reviewerID.unique())):
    test_dict[str(i)] = df1[df1.reviewerID.isin([str(i)])].sort_values(['unixReviewTime']).values[-1][1]#test
    val_dict[str(i)] = df1[df1.reviewerID.isin([str(i)])].sort_values(['unixReviewTime']).values[-2][1]#val
    train_dict[str(i)] = df1[df1.reviewerID.isin([str(i)])].sort_values(['unixReviewTime'])['asin'].values[:-2].tolist()#train
# print(test_dict)
"""
{'0': '1430','1': '19973',
 '2': '17307',
 '3': '6114',
 '4': '20713',
 '5': '22824',
 '6': '5434',
 '7': '2340',
 '8': '19467',
 '9': '4428',
 '10': '20594',
 '11': '14869',
 '12': '9325',
 '13': '9144',
 '14': '21224',
 '15': '22885',
 '16': '17786',
 '17': '22665',
 '18': '8731',
 '19': '14952',
  ....
  }

"""
"""
#print(train_dict)

{'0': ['0', '8557', '8094', '17184', '18246'],
 '1': ['2786', '0', '2791', '12429', '19284', '19698', '6974', '3875'],
 '2': ['9645','10311','721','9355','9696','0','20','4402','7974','2042'],
 '3': ['9131', '0', '395', '3997', '7424', '10292', '14194'],
 '4': ['465', '0', '5822'],
 '5': ['10769', '15418', '22856', '22053', '22122', '0', '1970'],
 '6': ['4238', '7012', '2495', '17488', '6056'],
 '7': ['0', '5005', '11262'],
 '8': ['42', '2327', '0'],
 '9': ['11980', '1497', '8773', '0'],
 '10': ['3091', '3239', '0', '3536', '8079', '15647', '8348'],
 '11': ['0', '2201', '7933'],
 '12': ['0', '9054', '11839', '12516'],
 '13': ['10929','11826','154','10535','0','4776','4858','8977','2889','7250','17032','19189']
  .....
  }
  
"""<jupyter_output><empty_output><jupyter_text>Save to json<jupyter_code>import json
from collections import OrderedDict

data=OrderedDict()

data['train'] = train_dict
data['val'] = val_dict
data['test'] = test_dict

user_dict, product_dict= {},{} # for user inner id to raw id
for k,v in reviewerID_dict.items():
    user_dict[v]=k
for k,v in asin_dict.items():
    product_dict[v]=k
    
data['user_dict'] = user_dict
data['product_dict'] = product_dict

# Print JSON
#print(json.dumps(data, ensure_ascii=False, indent="\t") )

with open(path+'/amazon_clothing_implicit.json', 'w', encoding="utf-8-sig") as make_file:
    json.dump(data, make_file, ensure_ascii=False, indent="\t")<jupyter_output>IOPub data rate exceeded.
The notebook server will temporarily stop sending output
to the client in order to avoid crashing it.
To change this limit, set the config variable
`--NotebookApp.iopub_data_rate_limit`.

Current values:
NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)
NotebookApp.rate_limit_window=3.0 (secs)<jupyter_text>Data preprocessing for Top-n model- reference : Neural Collaborative Filtering(2017), <NAME><jupyter_code># read json
dataset = open(path+'/amazon_clothing_explicit.json',encoding='utf-8-sig').read()
js=json.loads(dataset)
js.keys()
valNegatives,valRatings,testNegatives,testRatings=[],[],[],[]

# test,val Ratings 뽑기
for u,i in js['val'].items():
    valRatings.append([int(u),int(i)])
for u,i in js['test'].items():
    testRatings.append([int(u),int(i)])

# test,val Negatives 뽑기 (testNegative에 기존에 구매한것도 포함해야할까?). 난 그냥 테스트에만 포함되지 않게 만들었다.. 기존구매내역도 negative에 포함

for i in range(len(js['user_dict'])):
    val=[]
    for n in range(99):
        j=np.random.randint(len(js['product_dict']))
        while j in valRatings[i]: # valratings에 있으면 다시 뽑기
            j=np.random.randint(len(js['product_dict']))
        val.append(j)
    valNegatives.append(val)

for i in range(len(js['user_dict'])):
    test=[]
    for n in range(99):
        j=np.random.randint(len(js['product_dict']))
        while j in testRatings[i]: # testratings에 있으면 다시 뽑기
            j=np.random.randint(len(js['product_dict']))
        test.append(j
    testNegatives.append(test)
    

# val의 user np.full 해서, item은 sum_Ratings에 해당(Negatives와 Ratings를 합한 100개의 아이템 리스트)

val_full_user, test_full_user =  [],[]
for i in range(usernum):
    val_full_user.extend(np.full(100,i,dtype='int32').tolist())
for i in range(usernum):
    test_full_user.extend(np.full(100,i,dtype='int32').tolist())

# 이부분은 val에 해당 [[item....100개], [],,,,[]]    
sum_Ratings =[]
for n,i in enumerate(valRatings):
    sum_Ratings.append([i[1]]+valNegatives[n])
    #sum_Ratings.extend(i[1].extend
    
# 이부분은 test에 해당 [[item....100개], [],,,,[]]
sum_Ratings1 = []
for n,i in enumerate(testRatings):
    sum_Ratings1.append([i[1]]+testNegatives[n])

    
# sum_ratings들을 계산을 편하게 해주기위해 sum_Ratings들의 괄호들을 풀어줘서 1d array로 만들어주기 [1,1,1,1,1,1,...] 이런식으로
from itertools import chain

val_sum_Ratings, test_sum_Ratings=[],[]
val_sum_Ratings=list(chain.from_iterable(sum_Ratings))
test_sum_Ratings=list(chain.from_iterable(sum_Ratings1))



"""

저장

"""

from collections import OrderedDict

data=OrderedDict()

data['train'] = js['train']
data['valNegatives']=valNegatives
data['valRatings'] = valRatings
data['testNegatives'] = testNegatives
data['testRatings'] = testRatings

# for fast computaion 
data['val_full_user'] = val_full_user
data['test_full_user'] = test_full_user
data['val_sum_Ratings'] = sum_Ratings
data['test_sum_Ratings'] = sum_Ratings1 
data['flatten_val_sum_Ratings'] = val_sum_Ratings # 계산위해 flatten 시킨거
data['flatten_test_sum_Ratings'] = test_sum_Ratings

data['usernum'] = usernum
data['itemnum'] = itemnum

# Print JSON
print(json.dumps(data, ensure_ascii=False, indent="\t") )

with open(path+'/amazon_clothing_fast_implicit.json', 'w', encoding="utf-8-sig") as make_file:
    json.dump(data, make_file, ensure_ascii=False, indent="\t")<jupyter_output>IOPub data rate exceeded.
The notebook server will temporarily stop sending output
to the client in order to avoid crashing it.
To change this limit, set the config variable
`--NotebookApp.iopub_data_rate_limit`.

Current values:
NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)
NotebookApp.rate_limit_window=3.0 (secs)<jupyter_text>Create explicit dataset<jupyter_code>import os
import sys
import pandas as pd
import gzip
import numpy as np

def parse(path):
    g = gzip.open(path, 'rb')
    for l in g:
        yield eval(l)

def getDF(path):
    i = 0
    df = {}
    for d in parse(path):
        df[i] = d
        i += 1
    return pd.DataFrame.from_dict(df, orient='index')

df = getDF(path+'/reviews_Clothing_Shoes_and_Jewelry_5.json.gz')

df.head(3)
reviewerID_dict , asin_dict = {},{}

for n,i in enumerate(df.reviewerID.unique()):
    reviewerID_dict[i] = str(n)

for n,i in enumerate(df.asin.unique()):
    asin_dict[i] = str(n)
    
# one-hot
df.reviewerID = df.reviewerID.apply(lambda x: reviewerID_dict[x])
df.asin=df.asin.apply(lambda x : asin_dict[x])

df.head(4)
usernum = df.reviewerID.nunique()
itemnum = df.asin.nunique()
print('%s Users %s Items'%(usernum, itemnum))
df1=df[['reviewerID','asin','overall','unixReviewTime']]
df1.head(3)
#
df1[df1.reviewerID.isin(['0'])].sort_values(['unixReviewTime']).values
train_list, val_list, test_list =[],[],[]

for k in range(usernum):
    for i,j,R,_ in df1[df1.reviewerID.isin([str(k)])].sort_values(['unixReviewTime']).values : # train
        train_list.append((i,j,R))
        test_list.append(train_list.pop(-1))
        val_list.append(train_list.pop(-1))
  
  
"""
print(train_list)

[(user, item, rating),....(user M, item N, rating)]


"""

import json
from collections import OrderedDict

data=OrderedDict()

data['train'] = train_list
data['val'] = val_list
data['test'] = test_list
data['usernum'] = usernum
data['itemnum'] = itemnum

# Print JSON
#print(json.dumps(data, ensure_ascii=False, indent="\t") )

with open(path+'/amazon_clothing_explicit.json', 'w', encoding="utf-8-sig") as make_file:
    json.dump(data, make_file, ensure_ascii=False, indent="\t")<jupyter_output>IOPub data rate exceeded.
The notebook server will temporarily stop sending output
to the client in order to avoid crashing it.
To change this limit, set the config variable
`--NotebookApp.iopub_data_rate_limit`.

Current values:
NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)
NotebookApp.rate_limit_window=3.0 (secs)
{'fraction_non_alphanumeric': 0.18567552902875747, 'fraction_numerical': 0.060879001627780795, 'mean_word_length': 5.218623481781377, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '221854', 'license': 'Apache-2.0', 'n_tokens_mistral': 4065, 'n_tokens_neox': 3800, 'n_words': 725, 'path': 'Recommender-System/jupyter_notebook/amazon_dataset_split.ipynb', 'repo_name': 'Anukriti12/PersonalizedFashionStylist', 'size': '9220'}