<jupyter_start><jupyter_text>for 5x5 grid<jupyter_code>import pandas as pd
import numpy as np
from datetime import datetime,timedelta
import pytz
import pygeohash as gh
from haversine import haversine
import time
import pickle as cPickle
import glob
import json

geohash_prec = 5
def geohash(lat,long):
    geo=gh.encode(float(lat), float(long), precision=geohash_prec)
    return geo
data_train_acc['geohash_five'] = data_train_acc.apply(lambda row: geohash(row['acc_lat'],row['acc_long']), axis=1)
data_train_acc
data_train_nonac['geohash_five'] = data_train_nonac.apply(lambda row: geohash(row['non_acclat'],row['non_acclong']), axis=1)
data_train_nonac
data_train_acc['count']=1
data_train_acc
data_train_acc=data_train_acc[['geohash','acc_lat','acc_long','UJAHR','UMONAT','UWOCHENTAG','USTUNDE','count']]
data_train_acc.columns=['geohash','lat','long','UJAHR','UMONAT','UWOCHENTAG','hour','count']
data_train_acc
#data_train_nonac['geohash'] = data_train_nonac.apply(lambda row: geohash(row['non_acclat'],row['non_acclong']), axis=1)
data_train_nonac['count']=0
data_train_nonac
data_train_nonac=data_train_nonac[['geohash','non_acclat','non_acclong','UJAHR','UMONAT','UWOCHENTAG','hour','count']]
data_train_nonac.columns=['geohash','lat','long','UJAHR','UMONAT','UWOCHENTAG','hour','count']
data_train_nonac
import geohash as gh
def geohash(lat,long):
    geo=gh.encode(float(lat), float(long), precision=6)
    return geo

data_train_nonac['geohash6'] = data_train_nonac.apply(lambda row: geohash(row['lat'],row['long']), axis=1)
data_train_nonac
data_train_nonac=data_train_nonac[['geohash6','lat','long','UJAHR','UMONAT','UWOCHENTAG','hour','count']]
data_train_nonac.columns=['geohash','lat','long','UJAHR','UMONAT','UWOCHENTAG','hour','count']
data_train_nonac
data_train_nonac.geohash.nunique()
frame=[data_train_acc,data_train_nonac]
merge_train=pd.concat(frame)
merge_train=merge_train.reset_index(drop=True)
merge_train
import geohash as gh
def geohash(lat,long):
    geo=gh.encode(float(lat), float(long), precision=6)
    return geo
#city='new_method/Nurmberg/5x5_Grid'
geohash7=merge_train[['geohash','lat','long']]
geohash7['cluster_id']=geohash7.apply(lambda row: geohash(row['lat'],row['long']), axis=1)
geohash7=geohash7[['cluster_id','geohash']]
geohash7.columns=['geohash','cluster_id']
geohash7=geohash7.drop_duplicates()
geohash7.to_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone/geohash_cluster5to6.csv",index=False)
geohash7
# # Helper functions for the parallel computing 
import pandas as pd
import numpy as np

import random
import os
import sys
import psutil

import matplotlib
import matplotlib.pyplot as plt
import math
from multiprocessing import cpu_count,Pool 
import multiprocessing
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
import pygeohash as gh
import pickle
from datetime import date

import pysolar
from pysolar.solar import *
from datetime import date
import datetime as dt
import pygeohash as gh
from pvlib import atmosphere
import pvlib.solarposition as pv
from pvlib.tools import datetime_to_djd, djd_to_datetime
# X=pd.read_csv('/home/optim7/Dropbox/Smart_Analytics_work/WeatherData_WOOPA/PV_data/Testing123.csv',index_col='Datetime')
# X.index = pd.to_datetime(X.index,unit='s')
# X.head()
import pytz
import calendar
import numpy as np



class WithExtraArgs(object):
     def __init__(self, func, **args):
         self.func = func
         self.args = args
     def __call__(self, df):
         return self.func(df, **self.args)

def applyParallel(data, func,pool,partition,kwargs):
     data_split = [data[i:i + partition] for i in range(0, len(data), partition)]
     #data_split = np.array_split(data, min(partitions,data.shape[0]))
     data =pool.map(WithExtraArgs(func, **kwargs), data_split)
     #data = pd.concat(pool.map(WithExtraArgs(func, **kwargs), data_split))
     return data
def caculatemean(ob):
    mid=sum(ob) / len(ob) 
    return mid
merge_train
join_hann_forlatcenter=merge_train[['lat','long','geohash']]
latlist=join_hann_forlatcenter.groupby('geohash')['lat'].apply(list).reset_index(name='latlist')
latlist['middlelat']= latlist.apply(lambda row: caculatemean(row['latlist']), axis=1)
latlist
longlist=merge_train.groupby('geohash')['long'].apply(list).reset_index(name='longlist')
longlist['middlelong']= longlist.apply(lambda row: caculatemean(row['longlist']), axis=1)
longlist
all_merge=pd.merge(latlist,longlist,on='geohash')
all_merge=all_merge[['geohash','middlelat','middlelong']]
all_merge.to_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone/oneCrossOneGrid_centroid_cluster.csv",index=False)
data_train_acc
join_hann_centroid=pd.merge(merge_train,all_merge,on='geohash')
join_hann_centroid=join_hann_centroid.reset_index(drop=True)
join_hann_centroid
train_generted=join_hann_centroid.copy()
train_generted
def getsolarpos(df,geomap):
    #chdf=df.copy
    avgliEl=[]
    avgliZe=[]
    avgliAz=[]
    for index, row in df.iterrows():
        liZe=[]
        liAz=[]
        liEl=[]
        year=row['UJAHR']
        month=row['UMONAT']
        day=row['UWOCHENTAG']
        #hour=row['sessiondivided']
        hour=int(row['USTUNDE'])
        #onekmhash=row['cluster_id']
        l=()
        if day==1:
            day=6
        else:
            day=day-2
        A=calendar.TextCalendar(day)
        lat=row['middlelat']
        long=row['middlelong']
        for k in A.itermonthdays(year,month):
            #print(year,month,k)
            if k!=0:
                day1=date(year,month,k)
                if day1.weekday()==day:
                    dtime=str(year)+"-"+str(month).zfill(2)+"-"+str(k).zfill(2)+" "+str(hour).zfill(2)+":"+"00"+":"+"00"
                    date_time_obj = dt.datetime.strptime(dtime, '%Y-%m-%d %H:%M:%S')
                    timezone = pytz.timezone('Europe/Berlin')
                    timezone_date_time_obj = timezone.localize(date_time_obj)
                    az=get_azimuth(lat,long,timezone_date_time_obj)
                    sol_pos=pv.get_solarposition(timezone_date_time_obj,lat,long)
                    getsolarpos=sol_pos[['apparent_zenith', 'zenith', 'apparent_elevation', 'elevation','azimuth', 'equation_of_time']].values
                    ele=getsolarpos[0][3]
                    ze=getsolarpos[0][1]
                    az=getsolarpos[0][4]
                    liEl.append(ele)
                    liZe.append(ze)
                    liAz.append(az)
        npEl=np.array(liEl)
        npZe=np.array(liZe)
        npAz=np.array(liAz)
        
        avgEl=np.mean(liEl)
        avgZe=np.mean(liZe)
        avgAz=np.mean(liAz)

        avgliEl.append(avgEl)
        avgliZe.append(avgZe)
        avgliAz.append(avgAz)
        
    return avgliEl,avgliZe,avgliAz
from multiprocessing import Pool
import numpy as np
def create_sequences(df):
    cores = 35 #Number of CPU cores on your system
    partitions = cores
    pool = Pool(cores)
    
    partition = int(np.ceil(float(len(df)/partitions)))    
    temp = applyParallel (df,getsolarpos,pool,partition,{'geomap':df.copy()})
    #todf=pd.DataFrame(temp)
    retres=temp
    print(retres)
    pool.close()
    pool.join()
    elevation = []
    zenith = []
    azimuth=[]
    for set_ in retres:
        elevation.extend(set_[0])
        zenith.extend(set_[1])
        azimuth.extend(set_[2])

    elevation, zenith,azimuth = np.array(elevation), np.array(zenith),np.array(azimuth)   
    return elevation,zenith,azimuth
    #return retres
df_accCount=train_generted.copy()
train_generted_final=train_generted.copy()
train_generted_final['is_present']=1
train_generted_final=train_generted_final.sort_values(['geohash','UJAHR','UMONAT','UWOCHENTAG','hour'])
train_generted_final=train_generted_final.reset_index(drop=True)
train_generted_final
train_generted_final.columns
df_midlle_latlong=df_accCount[['geohash','middlelat','middlelong']]
df_midlle_latlong=df_midlle_latlong.drop_duplicates().reset_index().drop(['index'],axis=1)
df_midlle_latlong.columns=['geohash','lat_mid','long_mid']
df_midlle_latlong
def ispresent(middlelat,middlelong):
    is_present=-1
    if  middlelat==0 and middlelong==0:
        is_present=0
    else:
        is_present=1
    return is_present
import numpy as np
#def add_features(train_generted_final):
df_final = pd.pivot_table(train_generted_final,
                       index=['geohash','UJAHR','UMONAT','UWOCHENTAG','hour'],
                       values=['middlelat', 'middlelong', 'count'],                            
                       fill_value = 0,
                       dropna=False,
                       aggfunc=np.mean).reset_index()

df_final=df_final.sort_values(['geohash','UJAHR','UMONAT','UWOCHENTAG','hour'])
df_final['is_present']=df_final.apply(lambda row: ispresent(row['middlelat'],row['middlelong']), axis=1)
df_final_train=df_final.loc[((df_final['UJAHR']==2017) & (df_final['UMONAT']<=12)|(df_final['UJAHR']==2018) & (df_final['UMONAT']<=12) | ((df_final['UJAHR']==2019) & (df_final['UMONAT']<=5)))]
df_final_train=df_final_train.reset_index(drop=True)    
df_final_train
def findpast(row):
    df_filter=pd.DataFrame(columns=['geohash', 'UJAHR', 'UMONAT', 'UWOCHENTAG', 'hour', 'middlelat',
       'middlelong', 'count', 'is_present'])
    df_index = df_final_train.loc[
      (df_final_train['UJAHR'] == row['UJAHR']) & (df_final_train['UMONAT'] == row['UMONAT']) & (
                df_final_train['geohash'] == row['geohash'])
       & (df_final_train['hour'] == row['hour']) & (df_final_train['UWOCHENTAG'] == row['UWOCHENTAG'])
    &(df_final_train['is_present'] == row['is_present'])].index[0]
    i=int(df_index)-8
    for i in range(i,df_index):
        df_filter=df_filter.append((df_final_train.iloc[[i]]),ignore_index=True)
    return df_filter
def getdata(df_1,geomap):
    df=pd.DataFrame()
    for index, row in df_1.iterrows(): 
        #print(index)
        temp=findpast(row)
        temp['is_p2']=0
        df=df.append(temp)
        temp1=row
        temp1['is_p2']=2
        df=df.append(temp1)        
    return df
                                                                                                               
from multiprocessing import Pool
import numpy as np
def create_sequences_1(df):
    cores = 30#Number of CPU cores on your system
    partitions = cores
    pool = Pool(cores)
    partition = int(np.ceil(float(len(df)/partitions)))    
    temp = applyParallel (df,getdata,pool,partition,{'geomap':df.copy()})
    retres=temp
    print(retres)
    pool.close()
    pool.join()
    final = []
    final.extend(retres)
    return final

df=create_sequences_1(train_generted_final)
df
df_final=pd.DataFrame()
for df1 in df:
    df_final=df_final.append(df1)
df_final=df_final.reset_index(drop=True)
df_final

df_final.columns
join_middle_latlong=pd.merge(df_final,df_midlle_latlong,on='geohash',how='left')
join_middle_latlong.drop(['middlelat','middlelong'],axis=1,inplace=True)
join_middle_latlong=join_middle_latlong[['geohash', 'UJAHR', 'UMONAT', 'UWOCHENTAG', 'hour', 'count', 'is_present', 'is_p2'
                                        ,'lat_mid','long_mid']]
join_middle_latlong.columns=['geohash', 'UJAHR', 'UMONAT', 'UWOCHENTAG', 'USTUNDE',
       'counts', 'is_present', 'is_p2','middlelat', 'middlelong']
join_middle_latlong
elevation,zenith,azimuth=create_sequences(join_middle_latlong)
join_middle_latlong['elevaion']=elevation
join_middle_latlong['zenith']=zenith
join_middle_latlong['azimuth_solar']=azimuth
join_middle_latlong['concate_year']='20172018'
join_middle_latlong.columns
join_middle_latlong.to_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone/train_final_withSolarConvareyear_geohash.csv",index=False)
l=[]
year=[]
i=0
parts1=[]
with open("../../../data_preprocessing/data/regions/"+city+"/oneXone/train_final_withSolarConvareyear_geohash.csv", 'r') as file:
    header = True
    for line in file:
        if not header:
            header = True
            continue
        parts = line.replace('\r', '').replace('\n', '').split(',')
        if [parts[0],parts[13]] not in l:
            m=[parts[0],parts[13]]
            l.append(m)
            i=0
            parts.append(i)
        else:
            i=i+1
            parts.append(i) 
        parts1.append(parts)
import csv

def writeCsvFile(fname, data, *args, **kwargs):
    """
    @param fname: string, name of file to write
    @param data: list of list of items

    Write data to file
    """
    mycsv = csv.writer(open(fname, 'w'), *args, **kwargs)
    for row in data:
        mycsv.writerow(row)

writeCsvFile(r"../../../data_preprocessing/data/regions/"+city+"/oneXone//datawithazimuthUTPYE3OhannwithIndextrainGeohash.csv", parts1)
import pandas as pd
hann_indexed=pd.read_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone//datawithazimuthUTPYE3OhannwithIndextrainGeohash.csv")
hann_indexed.columns=['geohash', 'year', 'month', 'day', 'twohoursession',
       'acc_count', 'is_present',  'is_p2','middlelat', 'middlelong', 'elevaion', 'zenith',
       'azimuth_solar', 'concate_year', 'timestep']

hann_indexed.to_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone/datawithazimuthUTPYE3OhannwithIndextrainGeohash.csv",index=False)

hann_indexed.geohash.nunique()
import pandas as pd
import numpy as np

import random
import os
import sys
import psutil

import matplotlib
import matplotlib.pyplot as plt
import math
from multiprocessing import cpu_count,Pool 
import multiprocessing

from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

from sklearn.preprocessing import MinMaxScaler
import pickle

def daynight(month,hour):
    if hour>=22 or (hour>=0 and hour<4):
        return 0
    elif hour>=8 and hour<=15:
        return 1
    if month==1:
        if (hour>=4 and hour<=8) or (hour>=16 and hour<=22):
            return 0
        else:
            return 1
    if month==2:
        if (hour>=4 and hour<=8) or (hour>=18 and hour<=22):
            return 0
        if hour>=16 and hour <=18:
            return 1
    if month==3:
        if (hour>=4 and hour<=6) or  (hour>=19 and hour<=22):
            return 0
        if (hour>= 7 or hour <=8) or (hour>=16 and hour <=18):
            return 1
    if month==4:
        if (hour>=4 and hour<=5) or (hour>=20 and hour<=22):
            return 0
        if (hour>= 6 and hour <=8) or (hour>= 16 and hour <=20):
            return 1
    if month==5 or month==6 or month==7 or month==8:
        if (hour>=4 and hour<=5) or (hour==22):
            return 0
        if (hour>=6 and hour<=8) or (hour>=16 and hour<=21):
            return 1 
    if month==9 or month==10:
        if (hour>=4 and hour<=7 )or (hour>=20 and hour<=22):
            return 0
        if  (hour>=16 and hour<=19) :
            return 1 
    if month==11 or month==12:
        if(hour>=4 and hour<=8) or (hour>=16 and hour<=22):
            return 0 
        
                 
def time_interval_two(HOD):        
    if HOD>=6 and HOD<=10 :
        return 0
    if  HOD>=11 and HOD<=15 :
        return 1
    if  HOD>=16 and HOD<=17:
        return 2;
    if HOD>=18 and HOD<=22:
        return 3
    if HOD>22 or (HOD>=0 and HOD<6):
        return 4
    else:
        return 5;
#Officially spring is during the months of March, April and May in Germany. Summer is from June through to August. 
#Autumn is during the months of September, October and November and winter is from December to February. 
def season(month):
    if month>=3 and month<=5:
        return 0
    if month>=6 and month<=8:
        return 1
    if month>=9 and month<=11:
        return 2
    if month==12 or month==1 or month== 2:
        return 3
    
import pandas as pd

df_final=pd.read_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone/datawithazimuthUTPYE3OhannwithIndextrainGeohash.csv",header=0)
df_final_train=df_final.loc[((df_final['year']==2017) & (df_final['month']<=12)|(df_final['year']==2018) & (df_final['month']<=12) | ((df_final['year']==2019) & (df_final['month']<=1)))]
df_final_valisdation=df_final.loc[((df_final['year']==2019) & (df_final['month']>1)& (df_final['month']<=5))]

df_final_train.columns=['geohash','year', 'month', 'day', 'twohoursession',
       'acc_count', 'is_present','is_p2', 'middlelat', 'middlelong', 'elevaion',
       'zenith', 'azimuth_solar', 'concate_year', 'timestep']
df_final_train.to_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone/df_finalTrain.csv",index=False)
df_final_valisdation.columns=['geohash','year', 'month', 'day', 'twohoursession',
       'acc_count', 'is_present','is_p2', 'middlelat', 'middlelong', 'elevaion',
       'zenith', 'azimuth_solar', 'concate_year', 'timestep']
df_final_valisdation.to_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone/df_finalVald.csv",index=False)

geo_dict = dict(zip((hann_indexed.geohash.unique()), range(len(hann_indexed.geohash.unique()))))
print(geo_dict)
f = open"../../../data_preprocessing/data/regions/"+city+"/oneXone/geo_dict.pkl","wb")
pickle.dump(geo_dict,f)
f.close()
def create_temporaldata(filepath,city):
    df = pd.read_csv(filepath)
    f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone/geo_dict.pkl", "rb")
    geo_dict = pickle.load(f)
    f.close()

    def fun_hash(geohash):
        return geo_dict[geohash]

    df['geohash_code'] = df.apply(lambda row: fun_hash(row['geohash']), axis=1)
    def week_day(DOW):
        if DOW > 1 and DOW <= 6:
            return 1
        elif DOW == 1 or DOW == 7:
            return 0

    def make_binary(d):
        if d > 0:
            return 1
        else:
            return 0

    def year_ternary(d):
        if d == 2017:
            return 0
        if d==2018:
            return 1
        else:
            return 2

    df['DOW_cat'] = df.apply(lambda row: week_day(row['day']), axis=1)
    df['HOD_cat'] = df.apply(lambda row: time_interval_two(row['twohoursession']), axis=1)
    df['DayLight'] = df.apply(lambda row: daynight(row['month'], row['twohoursession']), axis=1)
    df['season'] = df.apply(lambda row: season(row['month']), axis=1)
    df['year'] = df.apply(lambda row: year_ternary(row['year']), axis=1)
    df['predicted_accident']=df.apply(lambda row: make_binary(row['acc_count']), axis=1)
    mergespeed_tempdf = df[[u'timestep', u'predicted_accident',u'is_p2',u'is_present', u'geohash', u'geohash_code',u'year', u'month', u'day',u'twohoursession', 
                  u'HOD_cat', u'DOW_cat', u'DayLight', u'season', u'elevaion', u'zenith', u'azimuth_solar', 'acc_count'
                           ]]
    mergespeed_tempdf.to_hdf("../../../data_preprocessing/data/regions/"+city+"/oneXone/"+"ng_train.h5",
                   key='set3')
    display(mergespeed_tempdf.head())

create_temporaldata("../../../data_preprocessing/data/regions/"+city+"/oneXone/df_finalTrain.csv", city)<jupyter_output><empty_output><jupyter_text>train and test<jupyter_code>import pandas as pd
import numpy as np

import random
import os
import sys
import psutil

import matplotlib
import matplotlib.pyplot as plt
import math
from multiprocessing import cpu_count,Pool 
import multiprocessing
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
from collections import Counter

import pickle<jupyter_output><empty_output><jupyter_text>hannover (1x1,5x5,clustering)<jupyter_code>latlong=pd.read_csv("../../../data_preprocessing/data/regions/"+city+"/oneXone/"+"oneCrossOneGrid_centroid_cluster.csv")
latlong['middlelong']=scalarval(latlong['middlelong'])
latlong['middlelat']=scalarval(latlong['middlelat'])
latlong
col = latlong.iloc[:, 1:].values
NLP_dict_mid = {}
for index, row in latlong.iterrows():
    NLP_dict_mid[row.geohash] = np.array(col[index])
f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone/hann_centerCord.pkl", "wb")
pickle.dump(NLP_dict_mid, f)
f=open("../../../data_preprocessing/data/regions/"+city+"/oneXone//geodata//clustered_data_poi_acc_merged.pkl","rb")
geohash_dict = pickle.load(f)
f.close()
print(geohash_dict)

f=open("../../../data_preprocessing/data/regions/"+city+"/oneXone//geo_dict.pkl","rb")
geo_dict = pickle.load(f)
f.close()
#print(geo_dict)

f=open("../../../data_preprocessing/data/regions/"+city+"/oneXone//geodata//uartTrainOnly.pkl","rb")
NLP_dict_uart = pickle.load(f)
f.close()
#print(geo_dict)

f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone//geodata//zustandTrainOnly.pkl","rb")
NLP_dict_zustand = pickle.load(f)
f.close()

f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone//geodata//utypeTrainOnly.pkl","rb")
NLP_dict_utype1 = pickle.load(f)
f.close()


f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone/geodata//acc_countTrainOnly.pkl","rb")
NLP_dict_accCount = pickle.load(f)
f.close()
#print((NLP_dict_accCount))


f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone//geodata//htype.pkl","rb")
NLP_dict_htype = pickle.load(f)
f.close()

f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone//geodata//maxspeed.pkl","rb")
NLP_dict_avg_max_speed = pickle.load(f)
f.close()

f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone///hann_centerCord.pkl","rb")
NLP_dict_centerCord = pickle.load(f)
f.close()

f = open("../../../data_preprocessing/data/regions/"+city+"/oneXone//geodata/clusterToGeohashCountTrainOnly.pkl","rb")
NLP_dict_clustertoGeohash = pickle.load(f)
f.close()


# # Helper functions for the parallel computing 

cores = cpu_count() #Number of CPU cores on your system
partitions = 25

class WithExtraArgs(object):
    def __init__(self, func, **args):
         self.func = func
         self.args = args
    def __call__(self, df):
         return self.func(df, **self.args)

def applyParallel(data, func,pool,partition, kwargs):
    data_split = [data[i:i + partition] for i in range(0, len(data), partition)]
    data =pool.map(WithExtraArgs(func, **kwargs), data_split)
    return data
def onhot_enoceder_month(train):
    myEncoder = OneHotEncoder(sparse=False)
    myEncoder.fit(train['month'].values.reshape(-1, 1))

    onehot_encode = pd.concat([train.reset_index().drop('month',1),
                 pd.DataFrame(myEncoder.transform(train['month'].values.reshape(-1, 1)),
                              columns=['m_en0','m_en1','m_en2','m_en3','m_en4','m_en5','m_en6','m_en7','m_en8','m_en9','m_en10','m_en11'])], axis=1).reindex()
    return onehot_encode.drop('index',1)
def onhot_enoceder_day(train):
    myEncoder = OneHotEncoder(sparse=False)
    myEncoder.fit(train['day'].values.reshape(-1, 1))

    onehot_encode = pd.concat([train.reset_index().drop('day',1),
                 pd.DataFrame(myEncoder.transform(train['day'].values.reshape(-1, 1)),
                              columns=['d_en0','d_en1','d_en2','d_en3','d_en4','d_en5','d_en6'])], axis=1).reindex()
    return onehot_encode.drop('index',1)
def onhot_enoceder(train):
    myEncoder = OneHotEncoder(sparse=False)
    myEncoder.fit(train['HOD_cat'].values.reshape(-1, 1))

    onehot_encode = pd.concat([train.reset_index().drop('HOD_cat',1),
                 pd.DataFrame(myEncoder.transform(train['HOD_cat'].values.reshape(-1, 1)),
                              columns=['HOD_en0','HOD_en1','HOD_en2','HOD_en3','HOD_en4'])], axis=1).reindex()
    return onehot_encode.drop('index',1)
def scalarval(df):
    scaler = MinMaxScaler(feature_range=(0, 1))
    scaler.fit(df.values.reshape(-1,1)) 
    scaled_values = scaler.transform(df.values.reshape(-1,1)) 
    df = scaled_values
    return df
def onhot_enoceder_season(train):
    myEncoder = OneHotEncoder(sparse=False)
    myEncoder.fit(train['season'].values.reshape(-1, 1))

    onehot_encode = pd.concat([train.reset_index().drop('season',1),
                 pd.DataFrame(myEncoder.transform(train['season'].values.reshape(-1, 1)),
                              #columns=['m_en0','m_en1','m_en2','m_en3','m_en4'])], axis=1).reindex()
                              columns=['season_0','season_1','season_2','season_3'])], axis=1).reindex()
    return onehot_encode.drop('index',1)
def onhot_enoceder_year(train):
    myEncoder = OneHotEncoder(sparse=False)
    myEncoder.fit(train['year'].values.reshape(-1, 1))

    onehot_encode = pd.concat([train.reset_index().drop('year',1),
                 pd.DataFrame(myEncoder.transform(train['year'].values.reshape(-1, 1)),
                              #columns=['m_en0','m_en1','m_en2','m_en3','m_en4'])], axis=1).reindex()
                              columns=['year_0','year_1','year_2'])], axis=1).reindex()
    return onehot_encode.drop('index',1)
def create_train_set_aug_geo(frame_list,geomap):
    process_name = str(multiprocessing.current_process())
    id = int(process_name.split(',')[0].split('-')[1])
    print("process ",id," started")
    
    X_train = []
    y_train = []
    print ("process list with length of ",len(frame_list))
    for frame in frame_list:
        training_set = frame.values
        geo_code = geo_dict[frame.geohash.iloc[0]]
        htype_count = 0
        for key, values in NLP_dict_htype.items():
            htype_count = len(values)
             break
        
        try:
            NLP_center_cord=NLP_dict_centerCord[frame.geohash.iloc[0]]
        except:
            NLP_center_cord = np.zeros(2)             
        NLP_cluster_geohash = np.zeros(1) 
        try:
            NLP_htype = NLP_dict_htype[frame.geohash.iloc[0]]
        except:
            NLP_htype = np.zeros(htype_count)
        try:
            geo_vec = geomap[frame.geohash.iloc[0]]
        except:
            geo_vec = np.zeros(10) 

        try:
            NLP_code_uart = NLP_dict_uart[frame.geohash.iloc[0]]
        except:
            NLP_code_uart = np.zeros(10)

        try:
            NLP_code_uytpe1 = NLP_dict_utype1[frame.geohash.iloc[0]]
        except:
            NLP_code_uytpe1 = np.zeros(7) 
        try:
            NLP_code_zustand = NLP_dict_zustand[frame.geohash.iloc[0]]
        except:
            NLP_code_zustand = np.zeros(3) 
        
        try:
            NLP_code_max_speed = NLP_dict_avg_max_speed[frame.geohash.iloc[0]]
        except:
            NLP_code_max_speed=[0.24]
             
        try:
            NLP_code_acc_count = NLP_dict_accCount[frame.geohash.iloc[0]]
        except:
            NLP_code_acc_count = np.zeros(1) 

        
        for i in range(8, training_set.shape[0]):
            if training_set[i, 2] > 1 : 
                if training_set[i, 1] > 0:
                    a = np.concatenate((training_set[i-8:i,7:].flatten(),geo_vec),axis=0)
                    a = np.concatenate((a,NLP_code_uart),axis=0)
                    a= np.concatenate((a,NLP_code_uytpe1),axis=0)
                    a=np.concatenate((a,NLP_code_zustand),axis=0)
                    a=np.concatenate((a,NLP_code_acc_count),axis=0)
                    a=np.concatenate((a,NLP_code_max_speed),axis=0)
                    a=np.concatenate((a,NLP_htype),axis=0)
                    a=np.concatenate((a,NLP_center_cord),axis=0)
                    a=np.concatenate((a,NLP_cluster_geohash),axis=0)
                    a = np.append(a, geo_code)
                    X_train.append(a)
                    y_train.append(1)
                else:
                    a = np.concatenate((training_set[i-8:i,7:].flatten(),geo_vec),axis=0)
                    a = np.concatenate((a,NLP_code_uart),axis=0)
                    a= np.concatenate((a,NLP_code_uytpe1),axis=0)
                    a=np.concatenate((a,NLP_code_zustand),axis=0)
                    a=np.concatenate((a,NLP_code_acc_count),axis=0)
                    a=np.concatenate((a,NLP_code_max_speed),axis=0)
                    a=np.concatenate((a,NLP_htype),axis=0)
                    a=np.concatenate((a,NLP_center_cord),axis=0)
                    a=np.concatenate((a,NLP_cluster_geohash),axis=0)
                    a = np.append(a, geo_code)
                    X_train.append(a)
                    y_train.append(0)
    return X_train, y_train
def create_sequences(df,geohash_dict):
    frame_list=[]
    for idx, frame in df.groupby(df.geohash):        
        frame_list.append(frame)
    pool = Pool(cores)
    partition = int(np.ceil(float(len(frame_list))/partitions))    
    train_set = applyParallel (frame_list,create_train_set_aug_geo,pool,partition,{'geomap':geohash_dict.copy()})
    pool.close()
    pool.join()
    X_train = []
    y_train = []
    for set_ in train_set:
        X_train.extend(set_[0])
        y_train.extend(set_[1])

    X_train, y_train = np.array(X_train), np.array(y_train)   
    return X_train,y_train
def train_data(filename):
    df=pd.read_hdf("../../../data_preprocessing/data/regions/"+city+"/oneXone/"+"ng_train.h5",
                   key='set3')
    df=df[[u'timestep', u'predicted_accident',u'is_p2',u'is_present', u'geohash', u'geohash_code', u'month',u'day', u'twohoursession',u'year', 
                  u'HOD_cat', u'DOW_cat', u'DayLight', u'season', u'elevaion', u'zenith', u'azimuth_solar'
                           ]]
    display(df.head())
    df = onhot_enoceder_month(df)
    df= onhot_enoceder_day(df)
    df=onhot_enoceder(df)
    df=onhot_enoceder_season(df)
    df=onhot_enoceder_year(df)
    df['previous_acc']=df['predicted_accident']
    df['elevaion'] = scalarval(df['elevaion'])
    df['zenith'] = scalarval(df['zenith'])
    df['azimuth_solar'] = scalarval(df['azimuth_solar'])
    
    print(df.columns)
    df_normalize = df.copy()
    X_train, y_train = create_sequences(df,geohash_dict)
    np.save("../../../data_preprocessing/data/regions/"+city+"/oneXone/traindata/X_train",X_train) 
    print (X_train.shape)
    np.save("../../../data_preprocessing/data/regions/"+city+"/oneXone/traindata/y_train",y_train)
    print (y_train.shape)
train_data('hann')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1547477155343663, 'fraction_numerical': 0.016189908621374653, 'mean_word_length': 3.470840734162226, 'pattern_counts': {'":': 2, '<': 52, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '630226', 'license': 'MIT', 'n_tokens_mistral': 11166, 'n_tokens_neox': 10781, 'n_words': 1638, 'path': 'data_preprocessing/1_data_generation/5_create_trainTestData/oneXone_grid_train.ipynb', 'repo_name': 'Rajjat/ACAP', 'size': '30204'}