<jupyter_start><jupyter_text>Standard Test<jupyter_code>df_train = enron.get_dataframe('skilling-j', received_only=True)
np.log(1+features.words_in_body(df_train))
from willireply.features import features

my_common_words = ['ASAP', 'please', 'could you']
subject_common_words_feature = lambda df: features.common_words_subject(df, ['?' '!'])
body_common_words_feature = lambda df: features.common_words_body(df, ['?', '!'])

fe = FeatureExtractor(
      subject_common_words_feature,
      body_common_words_feature,
      #features.thread_length,
      lambda df: np.log(1+features.words_in_body(df)),
      features.words_in_subject
                     )
#users = [f.stem for f in enron.ENRON_INDEX_FOLDER.iterdir() if enron.is_labeled(f.stem)]
users = ['skilling-j']
model = SGDRegressor(max_iter=100)
for user in tqdm(users[:20]):
    df_train = enron.get_dataframe(user, received_only=True)
    if len(df_train) == 0:
        print(user,"has no emails")
        continue
    X = fe.extract(df_train)
    y_true = fe.get_labels(df_train)
    if len(np.where(y_true)[0]) == 0:
        print(user,"has no replies")
        continue

    model.fit(X, y_true)
#df_train = enron.get_dataframe('may-l', received_only=True)
df_validate = enron.get_dataframe('skilling-j', received_only=True)
y_pred = model.predict(fe.extract(df_validate)) > 0# Not sure best place to set this predict...
y_true = fe.get_labels(df_validate)

print(classification_report(y_true, y_pred, target_names=["no reply", "reply"]))
print('f_2 = %s' % fbeta_score(y_true, y_pred, 2, labels=['no reply', 'reply'], pos_label=1))
df_all = enron.get_dataframe(users[0])
df_all.loc[61].body.split('-----Original Message-----')[0]
df2 = df_all.copy()
df2['body'] = df_all['body'].apply(lambda x: x.split('-----Original Message-----')[0])
print(df2.loc[61].body)
missed = df_validate.iloc[np.where(y_true > y_pred)[0]]

labels = []
left = []
right = []

for idx, row in missed.iterrows():
    left.append(row.body)
    labels.append(f'({idx}) {row.subject}')
    right.append(df_all.loc[row.reply_id].body)
    
accordion = widgets.Accordion(children=[
    widgets.Box( [widgets.Textarea(l, layout={'height': '500px'}), widgets.Textarea(r, layout={'height': '500px'})] )
    for (l, r) in zip(left, right)
])
for i, l in enumerate(labels):
    accordion.set_title(i, l)
accordion<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16596461668070767, 'fraction_numerical': 0.013058129738837404, 'mean_word_length': 4.778588807785888, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '649713', 'license': 'MIT', 'n_tokens_mistral': 901, 'n_tokens_neox': 872, 'n_words': 153, 'path': 'notebooks/standard_tests.ipynb', 'repo_name': 'jondoesntgit/email-response-predictor', 'size': '2374'}