<jupyter_start><jupyter_text>Importing Datasets<jupyter_code>ROAD = "<NAME>."
YEAR = "2015"
EXT = ".csv"<jupyter_output><empty_output><jupyter_text>Traffic Dataset<jupyter_code>TRAFFIC_WINDOWSIZE = 1
TRAFFIC_FILENAME = "mmda_" + ROAD + "_" + YEAR +  "_transformed"
traffic_raw_data = pd.read_csv("../../../datasets/mmda/2015/transformed/" + TRAFFIC_FILENAME + EXT, skipinitialspace=True)
traffic_raw_data = traffic_raw_data.fillna(0)
traffic_dataset = traffic_raw_data

cols_to_remove =  [1, 2]

traffic_dataset.drop(traffic_raw_data.columns[[cols_to_remove]], axis=1, inplace=True)
traffic_dataset.head()<jupyter_output><empty_output><jupyter_text>Weather Dataset<jupyter_code>WEATHER_WINDOWSIZE = 2
WEATHER_FILENAME = "wwo_" + YEAR 
weather_dataset = pd.read_csv("../../../datasets/wwo/all-features/" + WEATHER_FILENAME + EXT, skipinitialspace=True)
weather_dataset = weather_dataset.fillna(0)

cols_to_remove = [0]
cols_to_remove += [2, 3, 13, 15, 17, 18, 21] #Remove Fahr and Miles features
weather_dataset = weather_dataset.drop(weather_dataset.columns[[cols_to_remove]], axis=1)
weather_dataset.head()
merged_dataset = pd.merge(traffic_dataset, weather_dataset, left_index=True, right_index=True)
merged_dataset.head()<jupyter_output><empty_output><jupyter_text>Visualize Datasets  Visualize Traffic<jupyter_code>startMonth = 14496
endMonth = 29184

#96 rows in one day
#January 0 - 2976
#February - 2976 - 5665
#March - 5664 - 8640
#April - 8640 -  11520
#May - 11520 - 14496
#June - 14496 - 17376
#July - 17376 - 20352
#August - 20352 - 23330
#September = 23330 to 26209
#October - 26209 to 29184
#November - 29184 to 32064
#December - 32064 to 35040

td = traffic_dataset
td.statusN = normalize(traffic_dataset.statusN)
td.statusS = normalize(traffic_dataset.statusS)

plt.figure(figsize=(20, 4))
# for w in weather_dataset.columns:
#     plt.plot(td.iloc[startMonth:endMonth,:].dt, weather_dataset[w][startMonth-95:endMonth-95])

xaxis = range(9, 12)
# plt.plot(weather_dataset['precipMM'][startMonth:endMonth].rolling(96).mean())
plt.plot(td.iloc[startMonth:endMonth,:].statusS, alpha=0.2, c='b')
plt.plot(td.iloc[startMonth:endMonth,:].statusS.rolling(96).mean(), c='black')

plt.plot(weather_dataset.iloc[startMonth:endMonth,:].precipMM, alpha=0.2,c='r')
plt.plot(weather_dataset.iloc[startMonth:endMonth,:].precipMM.rolling(60).mean(), c='green')

plt.show()
wd = weather_dataset.iloc[startMonth:endMonth,:].rolling(2880).mean()
d1 = td.iloc[startMonth:endMonth,:].rolling(2880).mean()
d2 = wd.rolling(672).mean()
df = pd.merge(d1, d2, left_index=True, right_index=True)
print("===============================\nWith Rolling Mean")
cor = df.corr(method='spearman')
print(cor.iloc[:, 0:2])
print()
print("Highly Correlated Features")
getAllHighCorr(cor, 0.1)


d1 = td
d2 = wd
df = pd.merge(d1, d2, left_index=True, right_index=True)
print("===============================\nWithout Rolling Mean")
cor = df.corr(method='spearman')
print(cor.iloc[:, 0:2])
print()
print("Highly Correlated Features")
getAllHighCorr(cor, 0.1)<jupyter_output>===============================
With Rolling Mean
                statusN   statusS
statusN        1.000000 -0.662335
statusS       -0.662335  1.000000
tempC          0.009394 -0.233156
windspeedKmph  0.702029 -0.535947
winddirDegree  0.689786 -0.553360
cond          -0.412431  0.024961
precipMM       0.426986 -0.420583
humidity      -0.385555  0.428139
visibility    -0.407632  0.494033
pressure      -0.738908  0.680440
cloudcover     0.715724 -0.523944
heatIndexC    -0.241106 -0.003743
dewPointC     -0.717916  0.569628
windChillC     0.008849 -0.233033
windGustKmph   0.684479 -0.518719
feelsLikeC    -0.241106 -0.003743

Highly Correlated Features
                statusS
statusN       -0.662335
statusS        1.000000
tempC         -0.233156
windspeedKmph -0.535947
winddirDegree -0.553360
precipMM      -0.420583
humidity       0.428139
visibility     0.494033
pressure       0.680440
cloudcover    -0.523944
dewPointC      0.569628
windChillC    -0.233033
windGustKmph  -0.518719





   [...]<jupyter_text>Correlate<jupyter_code># correlate weather_dataset
tc = traffic_dataset.corr(method='spearman')
tc
# correlate weather_dataset
wc = weather_dataset.iloc[14496:29184,:].corr(method='spearman')

#CorrelationPlot
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(wc, vmin=-1, vmax=1)
fig.colorbar(cax)
ticks = np.arange(0,len(wc.columns),1)
ax.set_xticks(ticks)
ax.set_yticks(ticks)
ax.set_yticklabels(wc.columns)
plt.show()<jupyter_output><empty_output><jupyter_text>Analyzing the relationship of traffic to weather<jupyter_code>#96 rows in one day
#January 0 - 2976
#February - 2976 - 5665
#March - 5664 - 8640
#April - 8640 -  11520
#May - 11520 - 14496
#June - 14496 - 17376
#July - 17376 - 20352
#August - 20352 - 23330
#September = 23330 to 26209
#October - 26209 to 29184
#November - 29184 to 32064
#December - 32064 to 35040
#Analyzing correlation of month of June to October (Rainy Season)
weather_lag = 0
startMonth = 5664
endMonth = 17376

traffic = traffic_dataset.iloc[startMonth:endMonth,:]
traffic.statusS = traffic.statusS.rolling(96).mean()
weather = weather_dataset.iloc[startMonth-weather_lag:endMonth-weather_lag,:]
merged_dataset = pd.merge(traffic, weather, left_index=True, right_index=True)
mdc_rainy = merged_dataset.corr(method='spearman')
mdc = mdc_rainy.iloc[0:2, :]
#CorrelationPlot
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(mdc, vmin=-1, vmax=1)
fig.colorbar(cax)
ticks = np.arange(0,len(mdc.columns[0:2]),1)
ax.set_yticks(ticks)
ticks = np.arange(0,len(mdc.columns),1)
ax.set_xticks(ticks)
ax.set_yticklabels(mdc.columns[0:2])
plt.title("Rainy Season")

#Analyzing correlation of month of November to May (Dry Season)
weather_lag = 0
traffic = [traffic_dataset.iloc[0:14496,:], traffic_dataset.iloc[29184:35040,:]]# January to May, November to December
traffic = pd.concat(traffic)
traffic.statusS = traffic.statusS.rolling(96).mean()
weather = weather_dataset.iloc[startMonth-weather_lag:endMonth-weather_lag,:]
merged_dataset = pd.merge(traffic, weather, left_index=True, right_index=True)
mdc_dry = merged_dataset.corr(method='spearman')
mdc = mdc_dry.iloc[0:2, :]
#CorrelationPlot
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(mdc, vmin=-1, vmax=1)
fig.colorbar(cax)
ticks = np.arange(0,len(mdc.columns[0:2]),1)
ax.set_yticks(ticks)
ticks = np.arange(0,len(mdc.columns),1)
ax.set_xticks(ticks)
ax.set_yticklabels(mdc.columns[0:2])
plt.title("Dry Season")

# Correlating one whole year
merged_dataset = pd.merge(traffic_dataset.iloc[:,:], weather_dataset.iloc[:,:], left_index=True, right_index=True)
merged_dataset.statusS =merged_dataset.statusS.rolling(96).mean()
mdc_year = merged_dataset.corr(method='spearman')
mdc = mdc_year.iloc[0:2, :]
#CorrelationPlot
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(mdc, vmin=-1, vmax=1)
fig.colorbar(cax)
ticks = np.arange(0,len(mdc.columns[0:2]),1)
ax.set_yticks(ticks)
ticks = np.arange(0,len(mdc.columns),1)
ax.set_xticks(ticks)
ax.set_yticklabels(mdc.columns[0:2])
plt.title("Whole Year")
plt.show()
def getAllHighCorr(dataset, limit):
    features = dataset.columns
    hc_features = []
    for w in range(0, len(features)):
        #print('Highly correlated with ' + w)
        col_name = features[w]
        col = dataset[col_name]
        temp = col[((col <= -limit) | (col >= limit))]
        hc_features.append(temp)
        #print(col[col>=0.5])
        #print('\n')
        
        
    for w in range(1, len(features)-1):
        hc_features[w] = hc_features[w].to_frame()
        print(hc_features[w])
        print('\n\n\n\n')
        
    return hc_features
# Get all Highly Correlated with Traffic of Dry Season
hc_features = getAllHighCorr(mdc_dry, 0.3)
# Get all Highly Correlated with Traffic of Wet Season
hc_features = getAllHighCorr(mdc_rainy, 0.5)
# Get all Highly Correlated with Traffic of the whole year
hc_features = getAllHighCorr(mdc_year, 0.1)<jupyter_output>statusS
statusN        0.204000
statusS        1.000000
tempC         -0.331357
windspeedKmph -0.155130
winddirDegree -0.172971
cond          -0.139468
precipMM      -0.102356
humidity       0.312691
heatIndexC    -0.295486
windChillC    -0.331377
windGustKmph  -0.123451
feelsLikeC    -0.295486



                  tempC
statusS       -0.331357
tempC          1.000000
winddirDegree  0.386547
cond           0.512631
precipMM       0.126571
humidity      -0.813179
pressure      -0.320333
cloudcover    -0.125661
heatIndexC     0.966170
dewPointC      0.311011
windChillC     0.999923
feelsLikeC     0.966172



               windspeedKmph
statusS            -0.155130
windspeedKmph       1.000000
humidity           -0.226920
pressure            0.106337
dewPointC          -0.332497
windGustKmph        0.966191



               winddirDegree
statusS            -0.172971
tempC               0.386547
winddirDegree       1.000000
cond                0.152191
precipMM           [...]<jupyter_text>Auto CorrelationI have no idea kung tama ginagawa ko :D<jupyter_code>traffic_dataset.iloc[0].autocorr(lag=100)
#Auto Correlation of statusS
plt.figure(figsize=(10, 6))
#plt.acorr(traffic_dataset.iloc[23330:24002,:].statusS, maxlags=97)
plt.acorr(traffic_dataset.iloc[23330:24002,:].statusN, maxlags=45)
plt.show()
plt.figure(figsize=(10, 6))
autocorrelation_plot(traffic_dataset.iloc[23330:24002,:].statusS);
plt.show()<jupyter_output><empty_output><jupyter_text>Miscellaneous Visualizations<jupyter_code>traffic_dataset.iloc[23330:24002,:].statusS.diff().plot(figsize=(20,10), linewidth=5, fontsize=20)
weather_dataset.iloc[23330:24002,:].tempC.diff().plot(figsize=(20,10), linewidth=5, fontsize=20)
plt.xlabel('Year', fontsize=20);
plt.show()
traffic_dataset.iloc[20352:26209,:].statusS.diff().plot(figsize=(20,10), linewidth=5, fontsize=20)

plt.show()
#Weekly
statusS = traffic_dataset.iloc[20352:26209,:].statusS
plt.figure(figsize=(10, 6))
plt.plot(traffic_dataset.iloc[20352:26209,:].dt, statusS.rolling(96).mean())
plt.show()<jupyter_output><empty_output><jupyter_text>Export Correlated as CSV<jupyter_code>merged_dataset.corr().to_csv("stats/cross-correlation7_" + ROAD + "_" + YEAR + EXT, encoding='utf-8')
merged_dataset.to_csv("stats/merged_dataset_" + ROAD + "_" + YEAR + EXT, encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1440317276068872, 'fraction_numerical': 0.10582317662990907, 'mean_word_length': 4.531835205992509, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '471312', 'license': 'MIT', 'n_tokens_mistral': 4764, 'n_tokens_neox': 3990, 'n_words': 696, 'path': '.ipynb_checkpoints/CorrelationAnalysis-checkpoint.ipynb', 'repo_name': 'fuouo/TrafficBato', 'size': '10340'}