<jupyter_start><jupyter_text>Dataset OverviewHow much data do we have?  What ages?<jupyter_code>import pandas as pd
import plotly.io as pio; pio.renderers.default='notebook'
import plotly.express as px
import pickle


df_train = pickle.load(open("../data/selfie_train.pkl", "rb"))
df_test = pickle.load(open("../data/selfie_test.pkl", "rb"))

df_train.head()
import matplotlib.pyplot as plt

# the histogram of the data
n, bins, patches = plt.hist(df_train.age.to_numpy(), 20, density=False, facecolor='g', alpha=0.75)

plt.xlabel('Age')
plt.ylabel('Image count')
#plt.grid(True)
plt.savefig('../selfies_by_age.png', dpi=100)
plt.show()
print(f"We have {df_train.shape[0] + df_test.shape[0]} images total.")<jupyter_output>We have 21070 images total.
{'fraction_non_alphanumeric': 0.15733333333333333, 'fraction_numerical': 0.02, 'mean_word_length': 5.827272727272727, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '230578', 'license': 'MIT', 'n_tokens_mistral': 307, 'n_tokens_neox': 290, 'n_words': 64, 'path': 'src/summary.ipynb', 'repo_name': 'EdwardDixon/selfie-fv', 'size': '750'}