<jupyter_start><jupyter_text>Pennsylvania, USA **Source of original dataset:** https://pennshare.maps.arcgis.com/apps/webappviewer/index.html?id=8fdbf046e36e41649bbfd9d7dd7c7e7e**Location of accidents:** Latitude, Longitude**Date of accidents:** Date**Outcome of accidents:** Fatality, Suspected Serious Injury, Injury, Possible Injury, Suspected Minor Injury, , PDO<jupyter_code>import pandas as pd
pd.set_option('max_columns', None)
pd.set_option('display.max_colwidth', -1)
import numpy as np
from plotly import graph_objects as go
import plotly.express as px
from itertools import chain
import matplotlib.pyplot as plt
import glob<jupyter_output><empty_output><jupyter_text>Setup input files<jupyter_code>data_dir = "../data/pennsylvania/"

files = glob.glob(data_dir+'CRASH*.csv')<jupyter_output><empty_output><jupyter_text>Read original data<jupyter_code>data_aux = []
for x in files:
    print(x)
    aux = pd.read_csv(x, encoding = "ISO-8859-1")
    data_aux.append(aux)
list_of_dfs = data_aux
list_of_dicts = [cur_df.T.to_dict().values() for cur_df in list_of_dfs]    
data = pd.DataFrame(list(chain(*list_of_dicts)))<jupyter_output><empty_output><jupyter_text>Create Datetime column<jupyter_code>data['Date'] = data['CRASH_YEAR'].astype('str').astype('str') + '/' +data['CRASH_MONTH'].astype('str')+ '/01 00:00'
data['Date'] = pd.to_datetime(data['Date'])<jupyter_output><empty_output><jupyter_text>Setup Longitude & Latitude columns<jupyter_code>data['Longitude'] = data['DEC_LONG']
data['Latitude'] = data['DEC_LAT']<jupyter_output><empty_output><jupyter_text>Some key statistics<jupyter_code>total_accidents = data.shape[0]
print("There are a total of "+str(total_accidents)+" accidents.")

fatalities = data['FATAL_COUNT'].sum()
print("There are a total of "+str(fatalities)+" fatalities.")

serious_injuries = data['MAJ_INJ_COUNT'].sum()
print("There are a total of "+str(serious_injuries)+" suspected seriously injured.")

injuries = data['INJURY_COUNT'].sum()
print("There are a total of "+str(injuries)+" injured.")

min_inj = data['MIN_INJ_COUNT'].sum()
print("There are a total of "+str(min_inj)+" Possible Injury.")

mod = data['MOD_INJ_COUNT'].sum()
print("There are a total of "+str(mod)+" Suspected Minor Injury.")

bicycles = data[data['BICYCLE_COUNT']>0].shape[0]
print("There are a total of "+str(bicycles)+" bicycles involved in all the accidents.")<jupyter_output><empty_output><jupyter_text>Slice all bicycle accidents<jupyter_code>data_bicycles = data[data['BICYCLE_COUNT']>0]
data_bicycles.head()<jupyter_output><empty_output><jupyter_text>Save to file<jupyter_code>print(data_bicycles.shape)
data_bicycles.to_csv('cycling_safety_pennsylvania.csv')
print('Wrote file to: cycling_safety_pennsylvania.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14642082429501085, 'fraction_numerical': 0.011569052783803326, 'mean_word_length': 8.60763888888889, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '376491', 'license': 'MIT', 'n_tokens_mistral': 1037, 'n_tokens_neox': 995, 'n_words': 185, 'path': 'dataset_curation/pennsylvania.ipynb', 'repo_name': 'U-Shift/cyclands', 'size': '2766'}