<jupyter_start><jupyter_text><jupyter_code>#@title ## Initialization + Evaluation
#@markdown Hit the play button on the left to initialize this notebook.
#@markdown Listen to each sound file and give it an age bracket.
#@markdown We recommend doing this with headphones.
#@markdown Do as many as you can (you don't have to do them all). 
#@markdown When done, go to the bottom to evaluate your results.
#@markdown Save a copy of your results and send it back to the evaluator.
import os
import pandas as pd
import ipywidgets as widgets
from IPython.display import display
from IPython.display import Audio

print('initializing...')
if not os.path.isdir('cv_balanced/'):
  print("Downloading data...")
  !curl -L -o cv_balanced.zip "https://www.dropbox.com/s/uee4r9s3w7imj9k/cv_balanced.zip?dl=1"
  !unzip -o -q cv_balanced.zip
print('done!')

total_samples = 100
metadata = pd.read_csv('cv_balanced/cv_metadata_balanced.csv')
shuffled_metadata = metadata.sample(n=len(metadata), random_state=1337)

age_categories=['twenties', 'thirties', 'fourties', 'fifties', 'sixties']
assert set(age_categories) == set(shuffled_metadata.age.unique())
filenames = shuffled_metadata['path'].iloc[0:total_samples]
all_widgets = {}

for i, fname in enumerate(filenames):
  print(f'{i+1}. \n')
  current_widget =  widgets.RadioButtons(
        options=age_categories, 
        description='Choose Age:', 
        value=None,
        disabled=False,
  )
  all_widgets[fname] = {}
  all_widgets[fname]['widget'] = current_widget
  all_widgets[fname]['gt'] = shuffled_metadata[shuffled_metadata.path == fname]['age'].values[0]
  display(Audio('cv_balanced/clips/' + fname))
  display(current_widget)
  print("\n"*4)
#@title ## Results
#@markdown Hit the play button on this cell to compute your stats. 
#@markdown Copy-paste and send your results to your evaluator.

from sklearn.metrics import confusion_matrix, accuracy_score

# gather results
y_true = []
y_pred = []
for fname in all_widgets:
  if all_widgets[fname]['widget'].value is None:
    continue
  y_true.append(all_widgets[fname]['gt'])
  y_pred.append(all_widgets[fname]['widget'].value)

print("Number of samples analyzed:", len(y_pred))
print("Confusion Matrix:")
display(confusion_matrix(y_true, y_pred, labels=age_categories))
print("\n")
print(f"Accuracy: {accuracy_score(y_true, y_pred)*100} %")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12842105263157894, 'fraction_numerical': 0.008421052631578947, 'mean_word_length': 5.545454545454546, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '357999', 'license': 'MIT', 'n_tokens_mistral': 823, 'n_tokens_neox': 779, 'n_words': 207, 'path': 'voicemd/colab/Manual_Age_Evaluation.ipynb', 'repo_name': 'jerpint/voicemd', 'size': '2375'}