<jupyter_start><jupyter_text>General cleaning until this point, remapping after<jupyter_code>cleaned['isCitySpeed'] = cleaned.apply(lambda row: (row.Crash_Speed_Limit in ['60 km/h', '0 - 50 km/h']), axis=1)
cleaned['Crash_Lighting_Condition'] = cleaned['Crash_Lighting_Condition'].replace(['Darkness - Lighted', 'Darkness - Not lighted'], 'Darkness')
cleaned['combined_street'] = cleaned.apply(lambda row: ("%s - %s - %s" % (row.Loc_Suburb, row.Crash_Street, row.Crash_Street_Intersecting) ), axis=1)
cleaned.columns
cleaned.corr()
cleaned['Crash_Lighting_Condition'].value_counts()
shuffle(cleaned).to_csv(os.path.join(data_path,'cyclist_any_address_align.csv'),
                          index=False,
                         columns=['Crash_Month', 'Crash_Day_Of_Week', 'Crash_Hour',
                                 'combined_street', 'isCitySpeed',
                                 'Crash_Road_Surface_Condition', 'Crash_Atmospheric_Condition',
                                 'Crash_Lighting_Condition',
                                 'Crash_Road_Horiz_Align', 'Crash_Road_Vert_Align',
                                 'Cyclist_FSI'
                                 ])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12344656172328086, 'fraction_numerical': 0.00579950289975145, 'mean_word_length': 2.6495468277945617, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '112821', 'license': 'MIT', 'n_tokens_mistral': 389, 'n_tokens_neox': 371, 'n_words': 51, 'path': '04-cyclists_any_no_geolocation.ipynb', 'repo_name': 'sandra-arato/ARRBHack-2018', 'size': '1207'}