<jupyter_start><jupyter_text>Basic visualizations<jupyter_code>nyc_rs_pd[(nyc_rs_pd["tip_amount"] <15) & (nyc_rs_pd["tip_amount"] > 0) ]["tip_amount"].plot(kind='hist', bins=50)
plt.title('Tip amount distribution')
plt.xlabel('Tip Amount ($)'); plt.ylabel('Counts');
plt.figure(figsize=(6,6));

nyc_rs_pd_f = nyc_rs_pd[ (nyc_rs_pd["tip_amount"] <15) & (nyc_rs_pd["tip_amount"] > 0)]
nyc_rs_pd_f.boxplot(column="tip_amount", by='passenger_count')
# plt.title("tip amount by passenger count")
plt.figure(figsize=(7,7))

trip_duration = nyc_rs_pd["dropoff_datetime"] - nyc_rs_pd["pickup_datetime"] 
nyc_rs_pd["trip_duration_m"] = trip_duration / np.timedelta64(60, 's')
nyc_rs_pd[(nyc_rs_pd["trip_duration_m"] > 0) & (nyc_rs_pd["trip_duration_m"] < 70)]["trip_duration_m"].plot(kind="hist", bins=50)
plt.title("Histogram of trip duration in minutes")
plt.xlabel("trip duration in minutes")
nyc_rs_pd_f = nyc_rs_pd[(nyc_rs_pd["trip_duration_m"] > 0) 
                        & (nyc_rs_pd["trip_duration_m"] < 70)
                       & (nyc_rs_pd["tip_amount"] <15) 
                        & (nyc_rs_pd["tip_amount"] > 0)]
plt.scatter(nyc_rs_pd_f["trip_duration_m"], nyc_rs_pd_f["tip_amount"], c='blue', alpha = 0.10,)
plt.xlabel("trip duration in minutes")
plt.ylabel("tip amount")
plt.scatter(nyc_rs_pd_f["fare_amount"], nyc_rs_pd_f["tip_amount"], c='blue', alpha = 0.10,)
plt.xlabel("fare amount")
plt.ylabel("tip amount")<jupyter_output><empty_output><jupyter_text>Cardinalities of categorical data We will treat vendor ID, ratecode ID and payment ID as categorical data<jupyter_code>print(nyc_rs_pd["vendor_id"].unique())
print(nyc_batch_pd["ratecode_id"].unique())
print(nyc_batch_pd["payment_type"].unique())
print(nyc_batch_pd["store_and_fwd_flag"].unique())

print(len(nyc_batch_pd["pu_location_id"].unique()))
print(len(nyc_batch_pd["do_location_id"].unique()))<jupyter_output>223
253
{'fraction_non_alphanumeric': 0.17326994189117803, 'fraction_numerical': 0.020602218700475437, 'mean_word_length': 7.270742358078603, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '461545', 'license': 'MIT', 'n_tokens_mistral': 805, 'n_tokens_neox': 750, 'n_words': 101, 'path': 'notebooks/01-analysis-visualization.ipynb', 'repo_name': 'mailshanx/nyc_data', 'size': '1893'}