<jupyter_start><jupyter_text>__Labels:__* MNCHN = 1* Adolescent = 2* Geriatrics = 3* Special Populations = 4Reviewer feedback is integrated via weights.Need for sampling of documents not included in list as assumed label 0 with 0.5 weight.<jupyter_code>seed = 427
# MNCHN
df_train_mnchn = pd.DataFrame()
df_train_mnchn['body'] = df_mnchn['body'].append(df_mnchn['cycle_2_keywords'])
df_train_mnchn['label'] = np.ceil(df_mnchn['FLAG'].append(df_mnchn['FLAG']))
df_train_mnchn['weights'] = df_mnchn['FLAG'].append(df_mnchn['FLAG'])
df_train_mnchn = df_train_mnchn.dropna()
df_train_mnchn = df_train_mnchn[df_train_mnchn['label'] != 0]
df_train_mnchn['label'] = 1

# Adolescent
df_train_adolescent = pd.DataFrame()
df_train_adolescent['body'] = df_adolescent['body'].append(df_adolescent['cycle_2_keywords'])
df_train_adolescent['label'] = np.ceil(df_adolescent['FLAG'].append(df_adolescent['FLAG']))
df_train_adolescent['weights'] = df_adolescent['FLAG'].append(df_adolescent['FLAG'])
df_train_adolescent = df_train_adolescent.dropna()
df_train_adolescent = df_train_adolescent[df_train_adolescent['label'] != 0]
df_train_adolescent['label'] = 2 

# Geriatrics
df_train_geriatrics = pd.DataFrame()
df_train_geriatrics['body'] = df_geriatric['body'].append(df_geriatric['cycle_2_keywords'])
df_train_geriatrics['label'] = np.ceil(df_geriatric['FLAG'].append(df_geriatric['FLAG']))
df_train_geriatrics['weights'] = df_geriatric['FLAG'].append(df_geriatric['FLAG'])
df_train_geriatrics = df_train_geriatrics.dropna()
df_train_geriatrics = df_train_geriatrics[df_train_geriatrics['label'] != 0]
df_train_geriatrics['label'] = 3 

# Spec Pop
df_train_specpop = pd.DataFrame()
df_train_specpop['body'] = df_specpop['body'].append(df_specpop['cycle_2_keywords'])
df_train_specpop['label'] = np.ceil(df_specpop['FLAG'].append(df_specpop['FLAG']))
df_train_specpop['weights'] = df_specpop['FLAG'].append(df_specpop['FLAG'])
df_train_specpop = df_train_specpop.dropna()
df_train_specpop = df_train_specpop[df_train_specpop['label'] != 0]
df_train_specpop['label'] = 4

df_train = df_train_mnchn.append(df_train_adolescent).append(df_train_geriatrics).append(df_train_specpop)
# seed = 427
# # MNCHN
# df_train_mnchn = pd.DataFrame()
# df_train_mnchn['body'] = df_mnchn['body'].append(df_mnchn['cycle_2_keywords'])
# df_train_mnchn['label'] = np.ceil(df_mnchn['FLAG'].append(df_mnchn['FLAG']).replace('', 0))
# df_train_mnchn['weights'] = df_mnchn['FLAG'].append(df_mnchn['FLAG']).replace('', 1)
# df_train_mnchn = df_train_mnchn.dropna()
# df_train_mnchn = df_train_mnchn.append(pd.DataFrame({
#     'body': df[~df.title.isin(df_mnchn.AO)].sample(30, random_state=seed).body.values,
#     'label': 0,
#     'weights': 0.5
# }))


# # Geriatrics
# df_train_geriatrics = pd.DataFrame()
# df_train_geriatrics['body'] = df_geriatric['body'].append(df_geriatric['cycle_2_keywords'])
# df_train_geriatrics['label'] = np.ceil(df_geriatric['FLAG'].append(df_geriatric['FLAG']).replace('', 0))
# df_train_geriatrics['weights'] = df_geriatric['FLAG'].append(df_geriatric['FLAG']).replace('', 1)
# df_train_geriatrics = df_train_geriatrics[df_train_geriatrics.body != '']
# df_train_geriatrics = df_train_geriatrics.dropna()
# df_train_geriatrics = df_train_geriatrics.append(pd.DataFrame({
#     'body': df[~df.title.isin(df_geriatric.AO)].sample(30, random_state=seed).body.values,
#     'label': 0,
#     'weights': 0.5
# }))

# # Spec Pop
# df_train_specpop = pd.DataFrame()
# df_train_specpop['body'] = df_specpop['body'].append(df_specpop['cycle_2_keywords'])
# df_train_specpop['label'] = np.ceil(df_specpop['FLAG'].append(df_specpop['FLAG']).replace('', 0))
# df_train_specpop['weights'] = df_specpop['FLAG'].append(df_specpop['FLAG']).replace('', 1)
# df_train_specpop = df_train_specpop.dropna()
# df_train_specpop = df_train_specpop.append(pd.DataFrame({
#     'body': df[~df.title.isin(df_specpop.AO)].sample(30, random_state=seed).body.values,
#     'label': 0,
#     'weights': 0.5
# }))


# # Adolescent
# df_train_adolescent = pd.DataFrame()
# df_train_adolescent['body'] = df_adolescent['body'].append(df_adolescent['cycle_2_keywords'])
# df_train_adolescent['label'] = np.ceil(df_adolescent['FLAG'].append(df_adolescent['FLAG']).replace('', 0))
# df_train_adolescent['weights'] = df_adolescent['FLAG'].append(df_adolescent['FLAG']).replace('', 1)
# df_train_adolescent = df_train_adolescent.dropna()
# df_train_adolescent = df_train_adolescent.append(pd.DataFrame({
#     'body': df[~df.title.isin(df_adolescent.AO)].sample(30, random_state=seed).body.values,
#     'label': 0,
#     'weights': 0.5
# }))<jupyter_output><empty_output><jupyter_text>Classification Pipeline<jupyter_code>from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline

tfid_params = {
    'stop_words':stopwords.words(), 
    'ngram_range': (1,4), 
    'strip_accents':'ascii',
    
}

text_clf = Pipeline([('vect_tfid', TfidfVectorizer(**tfid_params)),
                     ('clf', MultinomialNB()),
                     ])
model_cycle_2 = text_clf.fit(df_train.body, df_train.label.values, 
                     **{'clf__sample_weight': df_train.weights})
results = pd.DataFrame(model_cycle_2.predict_proba(df.body), columns=themes)
results['AO'] = df.title
for theme in themes:
    r = results.sort_values(by=theme, ascending=False)[:60][['AO', theme]]
    a = set.difference(set(r.AO),
                        set(keywords_final[theme].index))
    r[r.AO.isin(a)].to_csv(theme+'_cycle2_results.csv', index=False)
# mnchn = text_clf.fit(df_train_mnchn.body, df_train_mnchn.label.values, 
#                      **{'clf__sample_weight': df_train_mnchn.weights})
# specpop = text_clf.fit(df_train_specpop.body, df_train_specpop.label, 
#                        **{'clf__sample_weight': df_train_specpop.weights})
# adolescent = text_clf.fit(df_train_adolescent.body, df_train_adolescent.label, 
#                           **{'clf__sample_weight': df_train_adolescent.weights})
# geriatrics = text_clf.fit(df_train_geriatrics.body, df_train_geriatrics.label, 
#                           **{'clf__sample_weight': df_train_geriatrics.weights})
# df_mnchn_res = pd.DataFrame(mnchn.predict_proba(df.body), columns=['irrelevance', 'relevance'])
# df_mnchn_res['AO'] = df.title
# df_specpop_res = pd.DataFrame(specpop.predict_proba(df.body), columns=['irrelevance', 'relevance'])
# df_specpop_res['AO'] = df.title
# df_adolescent_res = pd.DataFrame(adolescent.predict_proba(df.body), columns=['irrelevance', 'relevance'])
# df_adolescent_res['AO'] = df.title
# df_geriatric_res = pd.DataFrame(geriatrics.predict_proba(df.body), columns=['irrelevance', 'relevance'])
# df_geriatric_res['AO'] = df.title
# a = df_mnchn_res.sort_values('relevance', ascending=False)[:40]
# # set.intersection(set(a.AO),set(df_mnchn_flag.AO))
# a
df_mnchn.drop(['body', 'FLAG'], axis=1).to_csv('mnchn_cycle2_keywords.csv', index=False)
df_geriatric.drop(['body', 'FLAG'], axis=1).to_csv('geriatric_cycle2_keywords.csv', index=False)
df_adolescent.drop(['body', 'FLAG'], axis=1).to_csv('adolescent_cycle2_keywords.csv', index=False)
df_specpop.drop(['body', 'FLAG'], axis=1).to_csv('specpop_cycle2_keywords.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16832900669490367, 'fraction_numerical': 0.010110670856674409, 'mean_word_length': 6.930660888407368, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '467565', 'license': 'MIT', 'n_tokens_mistral': 2873, 'n_tokens_neox': 2729, 'n_words': 306, 'path': 'analytics/Classification_cycle_2.ipynb', 'repo_name': 'hadrianpaulo/project_deathstar', 'size': '7319'}