<jupyter_start><jupyter_text>Deep Gravity<jupyter_code>big_res_df1 = pd.DataFrame(columns=['cpc', 'nrmse', 'nmae', 'smape'])
OD = np.load('./data/3d_daily.npy').sum(axis=2)[:48, :48]
OD_max = OD.max(axis=1).reshape(-1, 1)
OD_max_pred = OD_max[-14:]

labels = np.load('./res/dg and g/labels.npy')[-14*48:]
labels = (labels.reshape(14, 48) * OD_max_pred).reshape(-1, 1)

for i in range(25):
    run = i + 1
    path = './runs/run%i/'%run
    pred_run = np.load(path+'pred.npy')[-14*48:]
    pred_run = (pred_run.reshape(14, 48) * OD_max_pred).reshape(-1, 1)
    cpc = dgu.get_CPC(pred_run, labels)
    nrmse = dgu.nrmse_loss_func(pred_run, labels, 0)
    nmae = dgu.nmae_loss_func(pred_run, labels, 0)
    smape = dgu.smape_loss_func(pred_run, labels, 0)
    big_res_df1.loc[run, :] = [cpc, nrmse, nmae, smape]
    
big_res_df1.loc['mean', :] = big_res_df1.mean().values
big_res_df1.loc['std', :] = big_res_df1.std().values

big_res_df1.to_csv('./res/dg and g/results_dg_25_runs.csv')
big_res_df1.loc[['mean', 'std']]
OD = np.load('./data/3d_daily.npy').sum(axis=2)[:48, :48]
OD_max = OD.max(axis=1).reshape(-1, 1)
OD_max_pred = OD_max[-14:]
run = 11
path = './runs/run%i/'%run
pred = np.load(path+'pred.npy')[-14*48:]
labels = np.load('./res/dg and g/labels.npy')[-14*48:]

pred = (pred.reshape(14, 48) * OD_max_pred).reshape(-1, 1)
labels = (labels.reshape(14, 48) * OD_max_pred).reshape(-1, 1)

labels_df = pd.DataFrame(labels).sort_values(by=0, ascending=False)
l_ind = labels_df.index
labels_df.index = range(labels_df.shape[0])
pred_df = pd.DataFrame(pred).loc[l_ind]
pred_df.index = range(pred_df.shape[0])

fig_res = plt.figure()
ax0 = fig_res.add_subplot(1, 1, 1)
ax0.plot(labels_df[0], '.', label='ground truth', ms=10)
ax0.plot(pred_df[0], '.', label='pred', ms=10)
ax0.set_xlabel('OD Pair', fontsize=12)
ax0.set_ylabel('# Annual Trips', fontsize=12)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.legend(fontsize=12)
plt.grid(ls='--')
m = 0.0

print('The mae loss is %.4f'%dgu.mae_loss_func(pred, labels, m))
print('The mape loss is %.4f'%dgu.mape_loss_func(pred, labels, m))
print('The smape loss is %.4f'%dgu.smape_loss_func(pred, labels, m))
print('The nrmse loss is %.4f'%dgu.nrmse_loss_func(pred, labels, m))
print('The nmae loss is %.4f'%dgu.nmae_loss_func(pred, labels, m))
print('The CPC is %.4f'%dgu.get_CPC(pred, labels))<jupyter_output>The mae loss is 32711.9884
The mape loss is 4.7522
The smape loss is 0.8279
The nrmse loss is 0.0875
The nmae loss is 0.0458
The CPC is 0.7160<jupyter_text>Gravity<jupyter_code>g_pred_df = pd.read_csv('./res/dg and g/g_pred.csv', index_col=0)
g_pred = g_pred_df['0'].values[-14*48:].reshape(-1, 1)
g_pred.shape
labels = np.load('./res/dg and g/labels.npy')[-14*48:]

labels = (labels.reshape(14, 48) * OD_max_pred).reshape(-1, 1)

labels_df = pd.DataFrame(labels).sort_values(by=0, ascending=False)
l_ind = labels_df.index
labels_df.index = range(labels_df.shape[0])
g_pred_df = pd.DataFrame(g_pred).loc[l_ind]
g_pred_df.index = range(g_pred_df.shape[0])

fig_res = plt.figure()
ax0 = fig_res.add_subplot(1, 1, 1)
ax0.plot(labels_df[0], '.', label='ground truth', ms=10)
ax0.plot(g_pred_df[0], '.', label='g_pred', ms=10)
ax0.set_xlabel('OD Pair', fontsize=12)
ax0.set_ylabel('# Annual Trips', fontsize=12)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.legend(fontsize=12)
plt.grid(ls='--')
m = 0.0

print('The mae loss is %.4f'%dgu.mae_loss_func(g_pred, labels, m))
print('The mape loss is %.4f'%dgu.mape_loss_func(g_pred, labels, m))
print('The smape loss is %.4f'%dgu.smape_loss_func(g_pred, labels, m))
print('The nrmse loss is %.4f'%dgu.nrmse_loss_func(g_pred, labels, m))
print('The nmae loss is %.4f'%dgu.nmae_loss_func(g_pred, labels, m))
print('The CPC is %.4f'%dgu.get_CPC(g_pred, labels))<jupyter_output>The mae loss is 55596.9690
The mape loss is 17.3799
The smape loss is 1.1917
The nrmse loss is 0.1396
The nmae loss is 0.0778
The CPC is 0.3505<jupyter_text>Two models in one graph<jupyter_code>run = 11
path = './runs/run%i/'%run
pred = np.load(path+'pred.npy')[-14*48:]
labels = np.load('./res/dg and g/labels.npy')[-14*48:]

pred = (pred.reshape(14, 48) * OD_max_pred).reshape(-1, 1)
labels = (labels.reshape(14, 48) * OD_max_pred).reshape(-1, 1)

labels_df = pd.DataFrame(labels).sort_values(by=0, ascending=False)
l_ind = labels_df.index
labels_df.index = range(labels_df.shape[0])
g_pred_df = pd.DataFrame(g_pred).loc[l_ind]
g_pred_df.index = range(g_pred_df.shape[0])
pred_df = pd.DataFrame(pred).loc[l_ind]
pred_df.index = range(pred_df.shape[0])

ms = 8
fs = 14
fig_res = plt.figure(figsize=[12, 3])
ax0 = fig_res.add_subplot(1, 2, 1)
ax0.plot(pred_df[0], '.', label='DG', ms=ms, c='red')
ax0.plot(labels_df[0], '.', label='Ground Truth', ms=ms, c='blue')
ax0.set_xlabel('Sorted OD Pair', fontsize=fs)
ax0.set_ylabel('Number of Annual Trips', fontsize=fs)
plt.xticks(fontsize=fs)
plt.yticks(fontsize=fs)
ax0.legend(fontsize=fs)
ax0.grid(ls='--')

ax1 = fig_res.add_subplot(1, 2, 2)
ax1.plot(g_pred_df[0], '.', label='GM', ms=ms, c='coral')
ax1.plot(labels_df[0], '.', label='Ground Truth', ms=ms, c='blue')
ax1.set_xlabel('Sorted OD Pair', fontsize=fs)
ax1.set_ylabel('Number of Annual Trips', fontsize=fs)
plt.xticks(fontsize=fs)
plt.yticks(fontsize=fs)
ax1.legend(fontsize=fs)
ax1.grid(ls='--')

plt.tight_layout()
fig_res.savefig('./res/dg and g/res.png', dpi=500)<jupyter_output><empty_output><jupyter_text>Spatial Visualization<jupyter_code>import geopandas as gpd
data_X_all = gpd.read_file('./data/data_X_all.shp')
stops = pd.read_csv('./data/stops_order.csv', index_col=0).iloc[:48, :]
data_X_all.head(2)
bart_coor = pd.read_csv('./data/station-coor.csv', index_col=0)
bart_coor.head(2)
fs = 12

fig = plt.figure(figsize=[20, 10], dpi=50)
ax0 = fig.add_subplot(133)
plt.xticks(rotation=30, fontsize=fs)
plt.yticks(fontsize=fs)
# ax0.set_title('DG Generation')
ax1 = fig.add_subplot(131)
plt.xticks(rotation=30, fontsize=fs)
plt.yticks(fontsize=fs)
# ax1.set_title('Ground Truth')
ax2 = fig.add_subplot(132)
plt.xticks(rotation=30, fontsize=fs)
plt.yticks(fontsize=fs)
# ax2.set_title('GM Generation')

data_X_all.plot(column='TotPop', ax=ax0, cmap='RdPu')
data_X_all.plot(column='TotPop', ax=ax1, cmap='RdPu')
data_X_all.plot(column='TotPop', ax=ax2, cmap='RdPu')

# plot trips
num_pred_stations = pred.shape[0] // 50  # 50 means that there are 50 stations in total
pred_stations = stops.iloc[-num_pred_stations:, 0]

low_lon_pred_dest = []
low_lat_pred_dest = []
mid_lon_pred_dest = []
mid_lat_pred_dest = []
high_lon_pred_dest = []
high_lat_pred_dest = []
low_lon_pred_origin = []
low_lat_pred_origin = []
mid_lon_pred_origin = []
mid_lat_pred_origin = []
high_lon_pred_origin = []
high_lat_pred_origin = []

low_lon_g_pred_dest = []
low_lat_g_pred_dest = []
mid_lon_g_pred_dest = []
mid_lat_g_pred_dest = []
high_lon_g_pred_dest = []
high_lat_g_pred_dest = []
low_lon_g_pred_origin = []
low_lat_g_pred_origin = []
mid_lon_g_pred_origin = []
mid_lat_g_pred_origin = []
high_lon_g_pred_origin = []
high_lat_g_pred_origin = []

low_lon_labels_dest = []
low_lat_labels_dest = []
mid_lon_labels_dest = []
mid_lat_labels_dest = []
high_lon_labels_dest = []
high_lat_labels_dest = []
low_lon_labels_origin = []
low_lat_labels_origin = []
mid_lon_labels_origin = []
mid_lat_labels_origin = []
high_lon_labels_origin = []
high_lat_labels_origin = []

for i in pred_stations.index:
    station_dest = pred_stations.loc[i]
    for j in stops.index:
        station_origin = stops.loc[j, 'stop']
        
        lon_origin = bart_coor.loc[bart_coor['abbr']==station_origin, 'lon'].values[0]
        lat_origin = bart_coor.loc[bart_coor['abbr']==station_origin, 'lat'].values[0]
        lon_dest = bart_coor.loc[bart_coor['abbr']==station_dest, 'lon'].values[0]
        lat_dest = bart_coor.loc[bart_coor['abbr']==station_dest, 'lat'].values[0]
        
        dest_ind = i-34
        origin_ind = j+1
        trip_ind = dest_ind*origin_ind - 1
        
        if pred[trip_ind]<0.3*labels.max():
            low_lon_pred_dest.append(lon_dest)
            low_lat_pred_dest.append(lat_dest)
            low_lon_pred_origin.append(lon_origin)
            low_lat_pred_origin.append(lat_origin)
#             ax0.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'b', lw=0.1)
        elif pred[trip_ind]>=0.3*labels.max() and pred[trip_ind]<0.6*labels.max():
            mid_lon_pred_dest.append(lon_dest)
            mid_lat_pred_dest.append(lat_dest)
            mid_lon_pred_origin.append(lon_origin)
            mid_lat_pred_origin.append(lat_origin)
#             ax0.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'red', lw=0.6*labels.max())
        elif pred[trip_ind]>=0.6*labels.max():
            high_lon_pred_dest.append(lon_dest)
            high_lat_pred_dest.append(lat_dest)
            high_lon_pred_origin.append(lon_origin)
            high_lat_pred_origin.append(lat_origin)
#             ax0.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'yellow', lw=2)
        
        if g_pred[trip_ind]<0.3*labels.max():
            low_lon_g_pred_dest.append(lon_dest)
            low_lat_g_pred_dest.append(lat_dest)
            low_lon_g_pred_origin.append(lon_origin)
            low_lat_g_pred_origin.append(lat_origin)
#             ax0.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'b', lw=0.1)
        elif g_pred[trip_ind]>=0.3*labels.max() and pred[trip_ind]<0.6*labels.max():
            mid_lon_g_pred_dest.append(lon_dest)
            mid_lat_g_pred_dest.append(lat_dest)
            mid_lon_g_pred_origin.append(lon_origin)
            mid_lat_g_pred_origin.append(lat_origin)
#             ax0.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'red', lw=0.6*labels.max())
        elif g_pred[trip_ind]>=0.6*labels.max():
            high_lon_g_pred_dest.append(lon_dest)
            high_lat_g_pred_dest.append(lat_dest)
            high_lon_g_pred_origin.append(lon_origin)
            high_lat_g_pred_origin.append(lat_origin)
#             ax0.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'yellow', lw=2)
        
        if labels[trip_ind]<0.3*labels.max():
            low_lon_labels_dest.append(lon_dest)
            low_lat_labels_dest.append(lat_dest)
            low_lon_labels_origin.append(lon_origin)
            low_lat_labels_origin.append(lat_origin)
#             ax1.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'b', lw=0.1)
        elif labels[trip_ind]>=0.3*labels.max() and labels[trip_ind]<0.6*labels.max():
            mid_lon_labels_dest.append(lon_dest)
            mid_lat_labels_dest.append(lat_dest)
            mid_lon_labels_origin.append(lon_origin)
            mid_lat_labels_origin.append(lat_origin)
#             ax1.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'red', lw=0.6*labels.max())
        elif labels[trip_ind]>=0.6*labels.max():
            high_lon_labels_dest.append(lon_dest)
            high_lat_labels_dest.append(lat_dest)
            high_lon_labels_origin.append(lon_origin)
            high_lat_labels_origin.append(lat_origin)
#             ax1.plot([lon_origin, lon_dest], [lat_origin, lat_dest], 'yellow', lw=2)

# dg pred
for i in range(len(low_lon_pred_dest)):
    ax0.plot([low_lon_pred_origin[i], low_lon_pred_dest[i]], [low_lat_pred_origin[i], low_lat_pred_dest[i]], 'b', lw=0.3)
    
for i in range(len(mid_lon_pred_dest)):
    ax0.plot([mid_lon_pred_origin[i], mid_lon_pred_dest[i]], [mid_lat_pred_origin[i], mid_lat_pred_dest[i]], 'red', lw=1)
    
for i in range(len(high_lon_pred_dest)):
    ax0.plot([high_lon_pred_origin[i], high_lon_pred_dest[i]], [high_lat_pred_origin[i], high_lat_pred_dest[i]], 'yellow', lw=1.5)

# gm pred
for i in range(len(low_lon_g_pred_dest)):
    ax2.plot([low_lon_g_pred_origin[i], low_lon_g_pred_dest[i]], [low_lat_g_pred_origin[i], low_lat_g_pred_dest[i]], 'b', lw=0.3)
    
for i in range(len(mid_lon_g_pred_dest)):
    ax2.plot([mid_lon_g_pred_origin[i], mid_lon_g_pred_dest[i]], [mid_lat_g_pred_origin[i], mid_lat_g_pred_dest[i]], 'red', lw=1)
    
for i in range(len(high_lon_g_pred_dest)):
    ax2.plot([high_lon_g_pred_origin[i], high_lon_g_pred_dest[i]], [high_lat_g_pred_origin[i], high_lat_g_pred_dest[i]], 'yellow', lw=1.5)

# labels
for i in range(len(low_lon_labels_dest)):
    ax1.plot([low_lon_labels_origin[i], low_lon_labels_dest[i]], [low_lat_labels_origin[i], low_lat_labels_dest[i]], 'b', lw=0.3)
    
for i in range(len(mid_lon_labels_dest)):
    ax1.plot([mid_lon_labels_origin[i], mid_lon_labels_dest[i]], [mid_lat_labels_origin[i], mid_lat_labels_dest[i]], 'red', lw=1)
    
for i in range(len(high_lon_labels_dest)):
    ax1.plot([high_lon_labels_origin[i], high_lon_labels_dest[i]], [high_lat_labels_origin[i], high_lat_labels_dest[i]], 'yellow', lw=1.5)



# plot stations
for i in bart_coor.index:
    lon = bart_coor.loc[i, 'lon']
    lat = bart_coor.loc[i, 'lat']
    ax0.plot(lon, lat, 'r.')
    ax1.plot(lon, lat, 'r.')
    ax2.plot(lon, lat, 'r.')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16188714153561518, 'fraction_numerical': 0.03276287388220783, 'mean_word_length': 4.689912280701755, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '667305', 'license': 'MIT', 'n_tokens_mistral': 5704, 'n_tokens_neox': 5452, 'n_words': 790, 'path': 'Result_Analysis_dg_g.ipynb', 'repo_name': 'HaTT2018/Deep_Gravity', 'size': '12972'}