<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/gdrive')
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import plotly.express as px
import os
project_dir = '/gdrive/My Drive/!Colab Notebooks/expert/'
os.listdir(project_dir)
question = pd.read_csv(f'{project_dir}/question-difficulty.csv')
apsa = pd.read_csv(f'{project_dir}/APSA_ExPERT_2021-07-25-04.csv')
apsa = apsa[(apsa['Stage'] == 'Question Editing') & (~apsa['Content Name'].str.contains('COPY')) & (~apsa['Content Name'].str.contains('Practicing Surgeon'))]
test = pd.read_csv(f'{project_dir}/test-difficulty.csv')
question.sample(25)
apsa.shape
_apsa = apsa.copy()# Copy the df
au_apsa = apsa.copy()
au_apsa.drop(au_apsa.iloc[:, 12:85], inplace=True,  axis = 1)<jupyter_output><empty_output><jupyter_text>create an author column from all tags<jupyter_code>au_apsa.loc[au_apsa['All Tags'].str.contains("au:CS"), 'author']= 'snyder'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:CL"), 'author']= 'lillehei'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:PJ"), 'author']= 'jarozki'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:EB"), 'author']= 'beierle'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:SI"), 'author']= 'islam'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:RC"), 'author']= 'cusick'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:GM"), 'author']= 'mcgahren'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:DP"), 'author']= 'powell'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:ES"), 'author']= 'skarsgard'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:MA"), 'author']= 'arca'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:JW"), 'author']= 'waldhausen'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:KH"), 'author']= 'heiss'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:KA"), 'author']= 'azarow'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:RM"), 'author']= 'meyers'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:MH"), 'author']= 'harmon'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:HR"), 'author']= 'rice'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:SL"), 'author']= 'lee'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:DL"), 'author']= 'ledbetter'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:\?"), 'author']= 'unknown'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:RR"), 'author']= 'ricca'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:ME"), 'author']= 'edwards'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:DL"), 'author']= 'ledbetter'
au_apsa.loc[au_apsa['All Tags'].str.contains("au:TP"), 'author']= 'ponsky'
crap = au_apsa.iloc[0:4, [0,1,2,11]]

crap.loc[crap['All Tags'].str.contains("au:SI"), 'author']= 'islam'
crap.loc[crap['All Tags'].str.contains("au:\?"), 'author']= 'fu'

crap.drop(crap.iloc[:, 1:3], inplace = True, axis = 1)
crap.head()
au_apsa.head() 
# This works
df_merge = question_clean.merge(au_apsa, how="inner", left_on="title", right_on="Content Name")
df_merge.head()
_apsa = _apsa.drop(columns=['Owner', 'Format', 'Due', 'Last Changed', 'Version', 'credit value', 'All Tags'])
_apsa = pd.melt(_apsa, id_vars=["Unique ID", "Content Name","Arrived in stage", "UPub Link"])

_apsa['value'] = _apsa['value'].fillna(0)
_apsa['value'].replace('x', 1, inplace=True)
_apsa.to_csv( "/gdrive/My Drive/!Colab Notebooks/expert/apsa_melt.csv", index=False, encoding='utf-8-sig')
_apsa.shape
# _apsa[(_apsa['variable'] == "au:CS") & (_apsa['value'] == 1)]
_apsa[(_apsa['variable'] == "au:CL") & (_apsa['value'] == 1)].value.count()
_apsa.groupby('variable', as_index = False)['value'].sum()
summary_apsa = _apsa.groupby('variable' , as_index = False)['value'].sum()
summary_apsa.to_csv( "/gdrive/My Drive/!Colab Notebooks/expert/apsa_summary.csv", index=False, encoding='utf-8-sig')
#test['average'].sort_values(ascending = False)
round(question['average'].mean(),2)
px.histogram(x='average', data_frame = question, nbins = 90, title = "Histogram of % correct first answers")
# question_clean = question[~(question['title'].str.contains('Practicing')) & ~(question['title'].str.contains('COPY')) ]
# drop col experimenting
# question_clean = question_clean.drop(columns=["B"], axis = 1)<jupyter_output><empty_output><jupyter_text>Create categories from the question titles<jupyter_code># Exclude copies and  practicing surgeon
question_clean = question[~(question['title'].str.contains('Practicing')) & ~(question['title'].str.contains('COPY')) ]

# question_clean['title'].str.contains('|'.join(searchfor)).sum()
# question_clean = question_clean.drop(columns=["A"], axis = 1)
question_clean.shape
#question.shape
onc_terms = ['umor', 'euroblastoma', 'ilm', 'habdo' , 'epatoblastoma', 'elanoma', 'Mass', 'eratoma', 'arcoma', 'Chemo', 'ymphoma', 'arcinoma']
trauma_terms = ['rauma', 'njury', 'urn']
fetal_terms = ['Prenatal', 'ulmon']
ecmo_terms = ['xtracorporeal', 'yperten']
anesthesia_terms = ['nesthesia', 'nalgesia']
arm_terms = ['irschsprung', 'loaca', 'rectal', 'rarino']
appy_terms = ['ppendicitis']
fen_terms = ['luid', 'utrition', 'nteral']
hepbil_terms = ['iver', 'ancrea', 'oledochal', 'iliary', 'allbladd', 'ortal', 'pleen', 'plenic']
gi_terms = ['soph', 'yloric', 'ntestin', 'ileal', 'lcerative', 'rohn', 'owel', 'uodenal',  'chala', 'eutropenic', 'tussus', 'ariatric']
hematol_terms = ['oagulat', 'fusion']
id_terms = ['fection', 'epsis']
vasc_terms = ['ascular', 'uctus', 'eart']
resp_terms = ['espir']
abd_wall_terms = ['nguina', 'mbilic', 'astrosch', 'mphalocele']
cdh_terms = ['iaphragm']
systems_terms = ['rrors', 'afety', 'olicy', 'thics', 'nformed', 'COVID', 'mprovement','orkforce', 'anagement', 'Board']
neo_terms = ['eonate', 'ecrotiz']
breast_terms = ['reast']
endo_terms = ['hyroid', 'drenal', 'eochrom', 'nsulin', 'exual']
gugyn_terms = ['varian', 'vary', 'estic', 'orchid','umcision']
question_clean.loc[question_clean['title'].str.contains('|'.join(onc_terms)), 'category'] = 'oncology'  
question_clean.loc[question_clean['title'].str.contains('|'.join(trauma_terms)), 'category'] = 'trauma' 
question_clean.loc[question_clean['title'].str.contains('|'.join(fetal_terms)), 'category'] = 'trauma' 
question_clean.loc[question_clean['title'].str.contains('|'.join(ecmo_terms)), 'category'] = 'fetal' 
question_clean.loc[question_clean['title'].str.contains('|'.join(anesthesia_terms)), 'category'] = 'anesthesia' 
question_clean.loc[question_clean['title'].str.contains('|'.join(arm_terms)), 'category'] = 'anorectal' 
question_clean.loc[question_clean['title'].str.contains('|'.join(appy_terms)), 'category'] = 'appendicitis' 
question_clean.loc[question_clean['title'].str.contains('|'.join(fen_terms)), 'category'] = 'fluid electrolytes and nutrition' 
question_clean.loc[question_clean['title'].str.contains('|'.join(hepbil_terms)), 'category'] = 'hepatobiliary and pancreas' 
question_clean.loc[question_clean['title'].str.contains('|'.join(gi_terms)), 'category'] = 'gastrointestinal' 
question_clean.loc[question_clean['title'].str.contains('|'.join(hematol_terms)), 'category'] = 'hematology' 
question_clean.loc[question_clean['title'].str.contains('|'.join(id_terms)), 'category'] = 'infection' 
question_clean.loc[question_clean['title'].str.contains('|'.join(vasc_terms)), 'category'] = 'vascular' 
question_clean.loc[question_clean['title'].str.contains('|'.join(resp_terms)), 'category'] = 'respiratory' 
question_clean.loc[question_clean['title'].str.contains('|'.join(abd_wall_terms)), 'category'] = 'abdominal wall' 
question_clean.loc[question_clean['title'].str.contains('|'.join(cdh_terms)), 'category'] = 'diaphragmatic hernia' 
question_clean.loc[question_clean['title'].str.contains('|'.join(systems_terms)), 'category'] = 'systems and ethics' 
question_clean.loc[question_clean['title'].str.contains('|'.join(neo_terms)), 'category'] = 'neonatal' 
question_clean.loc[question_clean['title'].str.contains('|'.join(endo_terms)), 'category'] = 'endocrine'
question_clean.loc[question_clean['title'].str.contains('|'.join(breast_terms)), 'category'] = 'breast'
question_clean.loc[question_clean['title'].str.contains('|'.join(gugyn_terms)), 'category'] = 'gu and gyn'

question_clean.loc[question_clean['category'].isnull(), 'category'] = 'other'
# question_clean['category1'] = 'unknown'
# question_clean['A'] = np.where(question_clean['title'].str.contains('|'.join(searchfor)), 'oncology', 'unknown')
# np.where(question_clean['title'].str.contains('|'.join(searchfor)))
question.count()
question_clean.sample(10)
# How many of the categories are still empty
print(question_clean['category'].isnull().sum())

# calculates what fraction of questions have no category assigned
(question_clean['category'].isnull().sum()/question_clean['title'].count() *100).round(2)

# print the actual questions with missing categories
# question_clean[question_clean['category'].isnull()].head(25)
# note the 'as_index = False' - this fixes col names
question_clean.groupby('category', as_index = False)['average'].mean().sort_values(by = 'average', ascending = False) 
# cat_analy = question_clean.groupby('category', as_index = False)['average'].mean().sort_values(by = 'average', ascending = False) 
fig = px.box(data_frame=question_clean, x = 'category', y = 'average')
fig.update_xaxes(categoryorder='mean ascending')
question_clean.groupby('category', as_index = False)['firstAttemptCount'].sum().sort_values(by = 'firstAttemptCount', ascending = False)
counts = question_clean.groupby('category', as_index = False)['firstAttemptCount'].sum()
fig = px.bar(data_frame=counts, x = 'category', y = 'firstAttemptCount')
#fig.update_xaxes(categoryorder='total ascending')
fig

question_clean.groupby('category', as_index = False).agg({ 'firstAttemptCount': 'sum', 
                                                          'average': 'mean'}) \
.rename(columns = {'firstAttemptCount': 'count'}) \
.sort_values(by = 'count', ascending = False)
counts_avg = question_clean.groupby('category', as_index = False).agg({ 'firstAttemptCount': 'sum', 
                                                          'average': 'mean'}) \
.rename(columns = {'firstAttemptCount': 'count'}) 
fig = px.bar(data_frame=counts_avg, x = 'category', y = 'count', color = 'average', text='count', \
             title="Number of question attempts by category (color represents % correct on 1st attempt)")
fig.update_layout(xaxis={'categoryorder':'total descending'})
fig.show()
authors = ['au:CS', 'au:CL', 'au:SI', 'au:DP', 'au:MA', 'au:GM', 'au:RC', 'au:JW', 'au:EB', 'au:RM', "au:KA", "au:CM" ]

apsa_au = _apsa[(_apsa['variable'].isin(authors)) & (_apsa['value'] == 1)]
apsa_au.head()<jupyter_output><empty_output><jupyter_text>Merge the questions with the au_apsa data to get the authors attached to the questions<jupyter_code>df_merge = question_clean.merge(apsa_au, how="inner", left_on="title", right_on="Content Name")
df_merge.head()
df_merge.groupby('author', as_index = False)['average'].mean().sort_values(by='average', ascending = False)
x = df_merge.groupby('author', as_index = False).agg({ 'firstAttemptCount': 'sum',
                                                      'title': 'count' ,
                                                          'average': 'mean'}) \
.rename(columns = {'firstAttemptCount': 'attempts', 'title':'tally'}) \
.sort_values(by = 'tally', ascending = False)
print(x)
x['tally'].sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20878254009992112, 'fraction_numerical': 0.00429485493908318, 'mean_word_length': 7.075017692852088, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '405777', 'license': 'CC0-1.0', 'n_tokens_mistral': 4235, 'n_tokens_neox': 4047, 'n_words': 662, 'path': 'expert.ipynb', 'repo_name': 'clsnyder/knowledge', 'size': '11409'}