<jupyter_start><jupyter_text>Settings<jupyter_code># colors

sns.set_style("white")
flatui = ['#1b9e77','#d95f02','#7570b3','#e7298a','#66a61e','#e6ab02']
#flatui = ['#e41a1c','#377eb8','#4daf4a','#984ea3','#ff7f00','#ffff33']
#flatui = ['#7fc97f','#beaed4','#fdc086','#ffff99','#386cb0','#f0027f']
sns.set_palette(flatui)

# workday, weekend
workdays_colors=["#426287","#a3a3a3"]

#good, bad
#weather_colors = ["#5f5961", "#c25050"]
#weather_colors = ["#5f5961", "#c28282"]
#weather_colors = ["#5f5961", "#c28282"]
weather_colors = ["#5f5961", "#c27474"]



x_size, y_size = 12,8
plt.rcParams.update({'font.size': 16})
use_external_weather = True<jupyter_output><empty_output><jupyter_text>Read the data<jupyter_code>df = pd.read_csv("data\\google\\Podatki o prometu v Ljubljani 5 oktober - 8 november 2020.csv", encoding="utf8", skiprows=1)
df[df.route_id == 4].tail(100)
df[df.route_id == 4].describe()
df = df[["timestamp", "route_id", "t_duration_in_traffic_[s]", "t_distance_[m]", "w_description", "w_temperature"]]
df.head()
df["timestamp"] = pd.to_datetime(df["timestamp"])
df["date"] = df["timestamp"].dt.date
df['time'] = pd.DatetimeIndex(df['timestamp']).hour + pd.DatetimeIndex(df['timestamp']).minute/60# + pd.DatetimeIndex(df['timestamp']).second/3600
df["hour"] = df["timestamp"].dt.hour
#df["hour"] = df["timestamp"].dt.time.astype(str).map(lambda x: float(x.split(":")[0]) + float(x.split(":")[1])/60)
df["weekday"] = df["timestamp"].dt.weekday
df["workday"] = df["weekday"].map(lambda x: 1 if x < 5 else 0)
df["speed"] = df["t_distance_[m]"]/df["t_duration_in_traffic_[s]"]
df["pace"] = 1/df["speed"]<jupyter_output><empty_output><jupyter_text>Weather data<jupyter_code>if not use_external_weather:
    weather = {'clear sky': 'dry', 
           'broken clouds': 'dry',
           'scattered clouds': 'dry', 
           'few clouds': 'dry',
           'moderate rain': 'wet',
           'light rain': 'humid',
           'heavy intensity rain': 'wet',
           'overcast clouds': 'humid', 
           'heavy intensity shower rain': 'wet',
           np.nan: '',
           'shower rain': 'wet',
           'light intensity shower rain': 'wet',
           'fog': 'humid', 
           'mist': 'humid',
           'light intensity drizzle': 'humid', 
           'drizzle': 'humid',
           'heavy intensity drizzle': 'wet'}
    df['weather'] = df["w_description"].map(lambda x: weather[x])
    
    for date in df['date'].unique():
        locs = date == df['date']

        avg_weather = df.loc[locs, 'weather'].mode()[0] # the most frequent entry in a given day
        avg_t = np.nanmean(df.loc[locs, 'w_temperature'])

        df.loc[locs, "avg daily weather"] = avg_weather
        df.loc[locs, "avg daily temp"] = avg_t

        weather = "normal" if (avg_weather == 'dry' and avg_t >= 5) or (avg_weather == 'humid' and avg_t >= 10) else 'adverse'
        #weather = "normal" if (avg_weather == 'dry' and avg_t >= 2) or (avg_weather == 'humid' and avg_t >= 4) else 'adverse'

        df.loc[locs, "weather"] = weather
else:
    df_weather = pd.read_csv('data\\weather.csv')
    df_weather['date'] = pd.to_datetime(df_weather['date']).dt.date
    for date in df['date'].unique():
        weather = df_weather.loc[df_weather['date'] == date, 'weather'].iloc[0]
        locs = date == df['date']
        df.loc[locs, "weather"] = weather<jupyter_output><empty_output><jupyter_text>Example export<jupyter_code>df[['timestamp', 'route_id', 't_duration_in_traffic_[s]', 't_distance_[m]', 'speed', 'pace', 'w_description', 'w_temperature','weather']].head(25).to_excel("data\\google_data_example.xlsx", index=False)
df = df[['timestamp', 
         'date', 
         'time', 
         'hour', 
         'route_id', 
         'speed',
         'pace',
         'workday',
         'weather', 
         't_duration_in_traffic_[s]', 
         't_distance_[m]',
         'weekday']]
df_filtered=df[['timestamp', 
                'date', 
                'time', 
                'hour', 
                'route_id', 
                'speed',
                'pace',
                'workday',
                'weather']]
df_filtered.to_csv("data\\google_data_preprocessed.csv", index=False)<jupyter_output><empty_output><jupyter_text>Validation  Circadian<jupyter_code>#sns.scatterplot(data=df.iloc[:1000,:], y='dt', x='time', hue='route_id', alpha=0.75)
#sns.lineplot(data=df.iloc[:1000,:], y='dt', x='time', hue='route_id', alpha=0.75)

sns.scatterplot(data=df, y='pace', x='time', hue='route_id', alpha=0.5, palette = flatui, s=20)
#sns.lineplot(data=df, y='dt', x='time', hue='route_id', alpha=0.75)


f = plt.gcf()
#f.set_size_inches(15,10)
plt.xlabel("Time of day")
plt.ylabel("Pace [s/m]")
plt.xticks(np.arange(0,25, 4), np.arange(0, 25, 4))

#plt.xticks(np.arange(25), np.arange(25))
#plt.savefig("time.pdf", bbox_inches="tight")
#plt.legend(routes.astype(str))
plt.show()
routes = df.route_id.unique()
fig, axes = plt.subplots(3,2)

for i, (route, color) in enumerate(zip(routes,flatui)):
    ax = axes.flat[i]
    sns.scatterplot(data=df[df.route_id == route], y='pace', x='time', alpha=0.75, color=color, ax = ax)
    ax.set_title("route " + str(route))
    ax.set_xlabel("Time of day")
    ax.set_ylabel("Pace [s/m]")
    ax.set_xticks(np.arange(0, 25, 4))
    ax.set_xticklabels(np.arange(0, 25, 4))
    

    

f = plt.gcf()
f.set_size_inches(x_size,2*y_size)
plt.savefig("figs\\circadian_pace.pdf", bbox_inches="tight")
plt.savefig("figs\\circadian_pace.png", bbox_inches="tight")
plt.savefig("figs\\circadian_pace.svg", bbox_inches="tight")
plt.show()<jupyter_output><empty_output><jupyter_text>Violinplots and boxplots<jupyter_code>sns.violinplot(data=df, y='pace', x='route_id')
f = plt.gcf()
f.set_size_inches(x_size, y_size)
plt.xlabel("Route")
plt.ylabel("Pace [s/m]")
plt.xticks
plt.savefig("figs\\distribution_pace.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_pace.svg", bbox_inches="tight")
plt.savefig("figs\\distribution_pace.png", bbox_inches="tight")
#plt.ylim([0.01,0.3])
plt.show()
sns.violinplot(data=df, y='speed', x='route_id')
f = plt.gcf()
f.set_size_inches(x_size, y_size)
plt.xlabel("Route")
plt.ylabel("Speed [m/s]")
plt.xticks
plt.savefig("figs\\distribution_speed.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_speed.svg", bbox_inches="tight")
plt.savefig("figs\\distribution_speed.png", bbox_inches="tight")
plt.show()
sns.boxplot(data=df, y='pace', x='route_id')
f = plt.gcf()
f.set_size_inches(x_size, y_size)
plt.xlabel("Route")
plt.ylabel("Pace [s/m]")
plt.xticks
plt.savefig("figs\\distribution_pace_box.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_pace_box.svg", bbox_inches="tight")
plt.savefig("figs\\distribution_pace_box.png", bbox_inches="tight")
plt.show()
sns.boxplot(data=df, y='speed', x='route_id')
f = plt.gcf()
f.set_size_inches(x_size, y_size)
plt.xlabel("Route")
plt.ylabel("Speed [m/s]")
plt.xticks
plt.savefig("figs\\distribution_speed_box.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_speed_box.svg", bbox_inches="tight")
plt.savefig("figs\\distribution_speed_box.png", bbox_inches="tight")
plt.show()
df_work = df.copy()
df_work['day'] = ["workday" if x == 1 else "weekend" for x in df_work['workday']] 

sns.violinplot(data=df_work, y='pace', x='route_id', hue = 'day', palette=workdays_colors[::-1])
f = plt.gcf()
f.set_size_inches(x_size, y_size)
plt.xlabel("Route")
plt.ylabel("Pace [s/m]")
plt.xticks
plt.savefig("figs\\distribution_workday.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_workday.svg", bbox_inches="tight")
plt.savefig("figs\\distribution_workday.png", bbox_inches="tight")
plt.show()
df_work = df.copy()
df_work['day'] = ["workday" if x == 1 else "weekend" for x in df_work['workday']] 

sns.boxplot(data=df_work, y='pace', x='route_id', hue = 'day', palette=workdays_colors[::-1])
f = plt.gcf()
f.set_size_inches(x_size, y_size)
plt.xlabel("Route")
plt.ylabel("Pace [s/m]")
plt.xticks
plt.savefig("figs\\distribution_workday_box.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_workday_box.svg", bbox_inches="tight")
plt.savefig("figs\\distribution_workday_box.png", bbox_inches="tight")
plt.show()
sns.violinplot(data=df, y='pace', x='route_id', hue = 'weather', palette=weather_colors)
f = plt.gcf()
f.set_size_inches(x_size, y_size)
plt.xlabel("Route")
plt.ylabel("Pace [s/m]")
plt.xticks
#plt.savefig("violin.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_pace_weather.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_pace_weather.svg", bbox_inches="tight")
plt.savefig("figs\\distribution_pace_weather.png", bbox_inches="tight")
plt.show()
sns.boxplot(data=df, y='pace', x='route_id', hue = 'weather', palette=weather_colors)
f = plt.gcf()
f.set_size_inches(x_size, y_size)
plt.xlabel("Route")
plt.ylabel("Pace [s/m]")
plt.xticks
#plt.savefig("violin.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_pace_weather_box.pdf", bbox_inches="tight")
plt.savefig("figs\\distribution_pace_weather_box.svg", bbox_inches="tight")
plt.savefig("figs\\distribution_pace_weather_box.png", bbox_inches="tight")
plt.show()<jupyter_output><empty_output><jupyter_text>Data separation  Routes<jupyter_code>df1 = df[['time', 'pace', 'route_id']].copy()
df1.columns = ['X', 'Y', 'route_id']
df1.to_csv("data\\01_routes.csv", index=False)<jupyter_output><empty_output><jupyter_text>Workdays / weekends<jupyter_code>df21 = df.loc[df.workday == 0, ['time', 'pace', 'route_id']].copy()
df21.columns = ['X', 'Y', 'route_id']
df21.to_csv("data\\02_routes_weekends.csv", index=False)

df22 = df.loc[df.workday == 1, ['time', 'pace', 'route_id']].copy()
df22.columns = ['X', 'Y', 'route_id']
df22.to_csv("data\\02_routes_workdays.csv", index=False)<jupyter_output><empty_output><jupyter_text>Weather<jupyter_code>df31 = df.loc[df["weather"] == "adverse", ['time', 'pace', 'route_id']].copy()
df31.columns = ['X', 'Y', 'route_id']
df31.to_csv("data\\03_routes_adverse_weather.csv", index=False)

df32 = df.loc[df["weather"] == "normal", ['time', 'pace', 'route_id']].copy()
df32.columns = ['X', 'Y', 'route_id']
df32.to_csv("data\\03_routes_normal_weather.csv", index=False)<jupyter_output><empty_output><jupyter_text>Saving the data in CosinorPy format<jupyter_code>from CosinorPy import file_parser
df_cosinor1 = df1.copy()
df_cosinor1.columns = ['x', 'y', 'test']
df_cosinor1['test'] = df_cosinor1['test'].map(lambda x: "route_" + str(x))
file_parser.export(df_cosinor1, "data\\google_traffic.xlsx")
df_cosinor21 = df21.copy()
df_cosinor21.columns = ['x', 'y', 'test']
df_cosinor21['test'] = df_cosinor21['test'].map(lambda x: "route_" + str(x) + "_weekend")

df_cosinor22 = df22.copy()
df_cosinor22.columns = ['x', 'y', 'test']
df_cosinor22['test'] = df_cosinor22['test'].map(lambda x: "route_" + str(x) + "_workday")

df_cosinor2 = pd.concat([df_cosinor21, df_cosinor22])


file_parser.export(df_cosinor2, "data\\google_traffic_workdays.xlsx")

df_cosinor31 = df31.copy()
df_cosinor31.columns = ['x', 'y', 'test']
df_cosinor31['test'] = df_cosinor31['test'].map(lambda x: "route_" + str(x) + "_adverse_weather")

df_cosinor32 = df32.copy()
df_cosinor32.columns = ['x', 'y', 'test']
df_cosinor32['test'] = df_cosinor32['test'].map(lambda x: "route_" + str(x) + "_normal_weather")

df_cosinor3 = pd.concat([df_cosinor31, df_cosinor32])


file_parser.export(df_cosinor3, "data\\google_traffic_weather.xlsx")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20257375470541888, 'fraction_numerical': 0.026788059178849688, 'mean_word_length': 5.417977528089888, 'pattern_counts': {'":': 2, '<': 42, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '346359', 'license': 'MIT', 'n_tokens_mistral': 4976, 'n_tokens_neox': 4582, 'n_words': 680, 'path': 'traffic_google_preprocess.ipynb', 'repo_name': 'SusTra/NSAPub', 'size': '11423'}