<jupyter_start><jupyter_text>hours removed<jupyter_code>category['date'] = category.apply(lambda category: hourremover(category['date']),axis=1)
main['date'] = main.apply(lambda main: hourremover(main['date']),axis=1)
productDetail['date'] = productDetail.apply(lambda productDetail: hourremover(productDetail['date']),axis=1)
other['date'] = other.apply(lambda other: hourremover(other['date']),axis=1)
cart['date'] = cart.apply(lambda cart: hourremover(cart['date']),axis=1)
success['date'] = success.apply(lambda success: hourremover(success['date']),axis=1)
main=main.rename(columns={'category_click':'main_click'})
other=other.rename(columns={'main_click':'other_click'})<jupyter_output><empty_output><jupyter_text>merging<jupyter_code>cm=category.merge(main,on=['userid','sessionid','date'],how='outer')
po=productDetail.merge(other,on=['userid','sessionid','date'],how='outer')
cs = cart.merge(success,on=['userid','sessionid','date'],how='outer')
cmpo = cm.merge(po,on=['userid','sessionid','date'],how='outer')
cmpocs = cmpo.merge(cs,on=['userid','sessionid','date'],how='outer')
cmpocs=cmpocs.fillna(0)
cmpocs=cmpocs.sort_values(['userid','sessionid'])<jupyter_output><empty_output><jupyter_text>check<jupyter_code>df[df['userid']=='0001da844668450707f57217271faeb0']
# its working
cmpocs<jupyter_output><empty_output><jupyter_text>date to day number<jupyter_code>x='2020-01-14'
x[8:10]
def todaynumber(date):
    if date[6:7] == '1':
        return int(date[8:10])  
    elif date[6:7] == '2':
        return int(date[8:10])+31
    elif date [6:7] == '3':
        return int(date[8:10])+60
cmpocs['daynum']=cmpocs.apply(lambda cmpocs: todaynumber(cmpocs['date']),axis=1)<jupyter_output><empty_output><jupyter_text>Create points for every user<jupyter_code>#different from number of session creations by a user, this point includes the value of the date session has created
rstcmpocs=cmpocs.reset_index()
def pointation(day):
    if day<8:
        return 1
    elif day<16 and day>7:
        return 1.2
    elif day<24 and day>15:
        return 1.4
    elif day<32 and day>23:
        return 1.6
    elif day<42 and day>32:
        return 1.8
    elif day<50 and day>41:
        return 2
    else:
        return 2.2
rstcmpocs['points']=rstcmpocs.apply(lambda rstcmpocs: pointation(rstcmpocs['daynum']),axis=1)
rstcmpocs
pointofusers = rstcmpocs.groupby(['userid'])['points'].sum()
pointofusers=pointofusers.reset_index()
pd.set_option('display.max_rows', 1000)
pointso=pointofusers.sort_values('points')<jupyter_output><empty_output><jupyter_text>mapping points 0 to 1<jupyter_code>pointso.points.mean()
#3 ten yukarı puanlıları 1 olarak aldım
pointso[pointso['points']>2.46]
def mapper(point):
    if point > 3:
        return 1
    else:
        return ((point-1)*0.8)/2 + 0.2
pointso['mappedpoints']=pointso.apply(lambda pointso: mapper(pointso['points']),axis=1)
pointso['mappedpoints'].mean()
pointso=pointso.drop(columns={'points'})
pointso<jupyter_output><empty_output><jupyter_text>number of every user opened a session<jupyter_code>rstcmpocs['count']=1
numofsessionbyuser=rstcmpocs.groupby('userid')['count'].sum().reset_index()
numofsessionbyuser
cmpocs=cmpocs.reset_index()<jupyter_output><empty_output><jupyter_text>last arrangements<jupyter_code>cmpocs
cmpocs=cmpocs.merge(pointso,on='userid')
cmpocs=cmpocs.set_index(['userid','mappedpoints','sessionid'])<jupyter_output><empty_output><jupyter_text>Final dataframe of training<jupyter_code>finaltraining=cmpocs
fianltraining=finaltraining.drop(columns='daynum')
finaltraining.to_csv('finaltraining')
finaltraining=finaltraining.drop(columns='daynum')
finaltraining<jupyter_output><empty_output><jupyter_text>Applying same to the test data !!!<jupyter_code>df=pd.read_parquet('test.snappy.parquet', engine='pyarrow')

df=df.reset_index()

df['ones']=1<jupyter_output><empty_output><jupyter_text>creating columns of page types<jupyter_code>category=df[df['page_type']=='category']

category=category.groupby(['userid','sessionid',]).agg(aggregation)

category=category.rename(columns={'ones':'category_click'})

main=df[df['page_type']=='main']

main=main.groupby(['userid','sessionid',]).agg(aggregation)

main=main.rename(columns={'ones':'main_click'})

productDetail=df[df['page_type']=='productDetail']

productDetail=productDetail.groupby(['userid','sessionid',]).agg(aggregation)

productDetail=productDetail.rename(columns={'ones':'productDetail_click'})

other=df[df['page_type']=='other']

other=other.groupby(['userid','sessionid',]).agg(aggregation)

other=other.rename(columns={'ones':'other_click'})

cart=df[df['page_type']=='cart']

cart=cart.groupby(['userid','sessionid',]).agg(aggregation)

cart=cart.rename(columns={'ones':'cart_click'})

success=df[df['page_type']=='success']

success=success.groupby(['userid','sessionid',]).agg(aggregation)

success=success.rename(columns={'ones':'success_click'})
category['date'] = category.apply(lambda category: hourremover(category['date']),axis=1)

main['date'] = main.apply(lambda main: hourremover(main['date']),axis=1)

productDetail['date'] = productDetail.apply(lambda productDetail: hourremover(productDetail['date']),axis=1)

other['date'] = other.apply(lambda other: hourremover(other['date']),axis=1)

cart['date'] = cart.apply(lambda cart: hourremover(cart['date']),axis=1)

success['date'] = success.apply(lambda success: hourremover(success['date']),axis=1)

main=main.rename(columns={'category_click':'main_click'})

other=other.rename(columns={'main_click':'other_click'})
cm=category.merge(main,on=['userid','sessionid','date'],how='outer')

po=productDetail.merge(other,on=['userid','sessionid','date'],how='outer')

cs = cart.merge(success,on=['userid','sessionid','date'],how='outer')

cmpo = cm.merge(po,on=['userid','sessionid','date'],how='outer')

cmpocs = cmpo.merge(cs,on=['userid','sessionid','date'],how='outer')

cmpocs=cmpocs.fillna(0)

cmpocs=cmpocs.sort_values(['userid','sessionid'])
cmpocs['daynum']=cmpocs.apply(lambda cmpocs: todaynumber(cmpocs['date']),axis=1)
rstcmpocs=cmpocs.reset_index()
rstcmpocs
def pointation(day):
    if day<65 and day>60:
        return 1
    elif day<69 and day>64:
        return 1.2
    elif day<74 and day>69:
        return 1.4
    elif day<78 and day>73:
        return 1.6
    elif day<83 and day>78:
        return 1.8
    elif day<87 and day>83:
        return 2
    else:
        return 2.2
rstcmpocs['points']=rstcmpocs.apply(lambda rstcmpocs: pointation(rstcmpocs['daynum']),axis=1)

rstcmpocs

pointofusers = rstcmpocs.groupby(['userid'])['points'].sum()

pointofusers=pointofusers.reset_index()

pd.set_option('display.max_rows', 1000)

pointso=pointofusers.sort_values('points')
pointso.points.mean()
#3 ten yukarı puanlıları 1 olarak aldım

def mapper(point):
    if point > 3:
        return 1
    else:
        return ((point-1)*0.8)/2 + 0.2

pointso['mappedpoints']=pointso.apply(lambda pointso: mapper(pointso['points']),axis=1)

pointso['mappedpoints'].mean()

pointso=pointso.drop(columns={'points'})

pointso
cmpocs
cmpocs=cmpocs.merge(pointso,on='userid')
cmpocs=cmpocs.reset_index()

# last arrangements



cmpocs=cmpocs.set_index(['userid','mappedpoints','sessionid'])

finaltest=cmpocs.drop(columns={'level_0','index','daynum'})
testfinal=testfinal.drop(columns='daynum').reset_index().rename(columns={'mappedpoints_y':'mappedpoints'}).set_index(['userid','mappedpoints','sessionid'])
testfinal.to_csv('testfinal')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18018741633199464, 'fraction_numerical': 0.02436412315930388, 'mean_word_length': 8.456962025316455, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '601436', 'license': 'MIT', 'n_tokens_mistral': 2824, 'n_tokens_neox': 2669, 'n_words': 341, 'path': 'InsiderPythonCodes/insiderv2.ipynb', 'repo_name': 'batuhanovski/Internship_works', 'size': '7470'}