<jupyter_start><jupyter_text>Hour of Day<jupyter_code>df = get_data(name='avg_dur_hod')
fig = plt.figure(figsize=(10,3))
ax = plt.gca()
ax.set_ylim([0, 1400])
for year in df.keys():
    x=df[year].iloc[:,0]
    y=df[year].iloc[:,1]
    plt.scatter(x,y,label=year)
    plt.legend(loc=1,bbox_to_anchor=(1.2,1.045))
plt.title('Avg Duration vs Hour of the Day',fontweight='bold')
plt.xlabel('HOD')
plt.ylabel('Avg Duration')
plt.show()<jupyter_output><empty_output><jupyter_text>For year 2015, hour 0001 and 0007 have expentionally small avergae duration. Although one can see the pattern but limiting the y-axis range will show it clearer.<jupyter_code>fig = plt.figure(figsize=(10,3))
ax = plt.gca()
ax.set_ylim([550, 1400])
for year in df.keys():
    x=df[year].iloc[:,0]
    y=df[year].iloc[:,1]
    plt.scatter(x,y,label=year)
    plt.legend(loc=1,bbox_to_anchor=(1.2,1.045))
plt.title('Avg Duration vs Hour of the Day',fontweight='bold')
plt.xlabel('HOD')
plt.ylabel('Avg Duration')
plt.show()<jupyter_output><empty_output><jupyter_text>There is defintely a pattern here. The interval between 2 PM and 5 PM has the highest average duration. After 5 PM, average duration starts to gradually decline. This pattern at hourly level seems like a good candidate for a new feature. I'll impute median average duration for the hour of the day.<jupyter_code># Calculating median from a list of avg duration from 2009 to 2021
df_joined = df['2009'].merge(df['2010'],on='H_O_D',suffixes=('2009','2010'))
for year in list(df.keys())[2:]:
    df_joined=df_joined.merge(df[year],on='H_O_D',suffixes=('',year))
df_joined.sort_values('H_O_D',inplace=True)
df_joined.reset_index(drop=True,inplace=True)
df_joined['median']=df_joined.iloc[:,1:].median(axis=1)
df_joined.loc[:,['H_O_D','median']].head()
x=df_joined.loc[:,'H_O_D']
y=df_joined.loc[:,'median']
plt.scatter(x,y)
plt.title('Median Duration vs HOD',fontweight='bold')
plt.xlabel('Hour of Day')
plt.ylabel('Median Duration(All years)')
plt.show()<jupyter_output><empty_output><jupyter_text>Day of Week<jupyter_code>df = get_data(name='avg_dur_dow')
# Sorting df on Day of Week for plotting purpose
dic = {'Thu':4, 'Fri':5, 'Sat':6, 'Wed':3, 'Sun':7, 'Tue':2, 'Mon':1}
def convert(x):
    return dic[x]
for year in df.keys():
    df[year].iloc[:,0]=df[year]['D_O_W'].apply(convert)
    df[year] = df[year].sort_values('D_O_W')
ax = plt.gca()
ax.set_ylim([600, 1250])
for year in df.keys():
    x=df[year].iloc[:,0]
    y=df[year].iloc[:,1]
    plt.scatter(x,y,label=year)
plt.title('Avg Duration vs DoW',fontweight='bold')
plt.xlabel('Day of the Week')
plt.ylabel('Avg Duration')
plt.show()<jupyter_output><empty_output><jupyter_text>This pattern can also be used for creating a new feature.<jupyter_code># Calculating median from a list of avg duration from 2009 to 2021
df_joined = df['2009'].merge(df['2010'],on='D_O_W',suffixes=('2009','2010'))
for year in list(df.keys())[2:]:
    df_joined=df_joined.merge(df[year],on='D_O_W',suffixes=('',year))
df_joined.sort_values('D_O_W',inplace=True)
df_joined.reset_index(drop=True,inplace=True)
df_joined['median']=df_joined.iloc[:,1:].median(axis=1)
df_joined.loc[:,['D_O_W','median']].head()
x=df_joined.loc[:,'D_O_W']
y=df_joined.loc[:,'median']
plt.scatter(x,y)
plt.title('Median Duration vs DOW',fontweight='bold')
plt.xlabel('Day of Week')
plt.ylabel('Median Duration(All years)')
plt.show()<jupyter_output><empty_output><jupyter_text>Month<jupyter_code>df_pickup = get_data(name='daily_pickup_cnt')
df_dur = get_data(name='avg_dur_doy')
df_joined = {}
for year in df_pickup.keys():
        df_joined[year] = df_pickup[year].merge(df_dur[year],on='D_O_Y')
        df_joined[year]['duration'] = df_joined[year].loc[:,'avg_trip_duration']*df_joined[year].loc[:,'daily_count']
df_joined['2009'].head()
# Converting day of the year to the month it belongs to
for year in df_joined.keys():
    def doy_to_month(day_num):
        day_num = str(day_num)
        day_num.rjust(3 + len(day_num), '0')
        month = datetime.strptime(year + "-" + day_num, "%Y-%j").strftime("%m")
        return month 
    df_joined[year]['month'] = df_joined[year].loc[:,'D_O_Y'].apply(doy_to_month)
    df_joined[year].drop(columns=['avg_trip_duration'],inplace=True)
df_joined['2009'].head()
df_new = {}
for year in df_joined.keys():
    temp = df_joined[year].iloc[:,1:].groupby('month').sum().reset_index()
    temp['avg_dur_month'] = temp['duration']/temp['daily_count']
    df_new[year] = temp.loc[:,['month','avg_dur_month']]
df_new['2009'].head()
fig = plt.figure(figsize=(15,10))
t=1
i=1
limit = {1:[650,800],2:[450,1250],3:[600,1200]}
for year in df_new.keys():
    if t % 4 == 0 and t<=8:
        i+=1
    if i ==1:
        plt.title('Avg Duration vs Month',fontweight='bold')
        plt.ylabel('Avg Duration')
    elif i ==3:
        plt.xlabel('Month')
    x=df_new[year].iloc[:,0]
    y=df_new[year].iloc[:,1]
    plt.subplot(3, 1, i)
    plt.plot(x,y,label=year)
    ax = plt.gca()
    ax.set_ylim(limit[i])
    plt.legend(loc=1,bbox_to_anchor=(1.2,1.045))
    t+=1
plt.show()<jupyter_output><empty_output><jupyter_text>Total Revenue<jupyter_code>df = get_data('total_rev')
df = pd.concat(df)
df.reset_index(inplace=True)
df.head()
plt.bar(x=df['level_0'],height=df['total_revenue'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1873231465761177, 'fraction_numerical': 0.03489907564610451, 'mean_word_length': 6.069333333333334, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '294407', 'license': 'MIT', 'n_tokens_mistral': 2310, 'n_tokens_neox': 2149, 'n_words': 374, 'path': 'EDA/query-viz.ipynb', 'repo_name': 'agam001/nyc-taxi-trip', 'size': '5301'}