<jupyter_start><jupyter_text>---  INSTEAD OF STORING BUNCH OF HUGE FILES IN RAM, PROCESS ONE BY ONE SO BROWSER DOES NOT REFRESH<jupyter_code>#if going url route, grab v8 .json using below call
#methods work; HTML path here is broken

# jsonurl = urllib.request.urlopen("https://storage.cloud.google.com/coronawhy/NLPDatasets/v7_preprocessed/v7_text.json")
# text = json.loads(jsonurl.read()) 

# text
#Another route: .json into pandas DF from Google Storage - This took a LONG time
# NLPData = pd.read_json('gs://coronawhy/NLPDatasets/v6_preprocessed/v6_text.json')
# NLPData 
#Put all files in a list to iterate through when reading .pkl as Pandas DF
pubs = ["v6_text_0.pkl", "v6_text_1.pkl", "v6_text_2.pkl", "v6_text_3.pkl", "v6_text_4.pkl", "v6_text_5.pkl", "v6_text_6.pkl", "v6_text_7.pkl", "v6_text_8.pkl", "v6_text_9.pkl", "v6_text_10.pkl", "v6_text_11.pkl", "v6_text_12.pkl", "v6_text_13.pkl", "v6_text_14.pkl", "v6_text_15.pkl", "v6_text_16.pkl", "v6_text_17.pkl", "v6_text_18.pkl", "v6_text_19.pkl"]
len(pubs)
#From each pub that is read in create a pandas data frame for each one
#Took around 10-15 min

def total_pubs(pubs):
   
  i = 0  
  while i < len(pubs):
    yield pd.read_pickle(pubs[i], 'gzip')    
    i += 1 
        
all_pubs = [i for i in total_pubs(pubs)] 
len(all_pubs) #Check that it is the length of original no. of files
#Combine all DF's

all_DFs = []
for i in range(0,20):
    all_DFs.append(all_pubs[i])

comb_DFs = pd.concat(all_DFs, ignore_index=True)
comb_DFs
#Get relevant columns
NLPData = comb_DFs[['paper_id','sentence']]
NLPData
#Group rows by paper_id creating txt at doc level then renaming "sentence" column to pub_text
NLPData = NLPData.dropna()
NLPData = NLPData.groupby('paper_id')['sentence'].apply(' '.join).reset_index()
NLPData.columns = ['paper_id', 'paper_txt']
NLPData
#NLPData['paper_txt'][0] #Check.
#Pull in annotation dataset with timestamp of **04.12.20, rename columns, drop unnecessary columns.
df2 = pd.read_csv('Study_Design_Annotation_Snapshot_4_12_20.csv')
df2.columns = ['Assignee',
 'cord_uid',
 'sha',
 'title',
 'url',
 'in silico',
 'in vitro',
 'in vivo',
 'Systematic review or meta-analysis of RCTs',
 'RCT',
 'Non-randomized controlled trial',
 'Comparative study',
 'Descriptive study',
 'Systematic review or meta-analysis of studies other than RCTs',
 'Other ',
 'General Notes']
df2 = df2.drop(columns=[
 'Assignee',
 'cord_uid',
 'title',
 'url',
'General Notes'])
df2.head()
### Drop all rows with 0 annotations; clean up annotations by converting all NaN's to 0
df2 = df2.dropna(how='all',subset=
[
 'sha',
 'in silico',
 'in vitro',
 'in vivo',
 'Systematic review or meta-analysis of RCTs',
 'RCT',
 'Non-randomized controlled trial',
 'Comparative study',
 'Descriptive study',
 'Systematic review or meta-analysis of studies other than RCTs',
 'Other ',
])
df2
#Merge NLPData with Annotations
cw_final = NLPData.merge(df2, how='inner', left_on='paper_id', right_on='sha')
cw_final
#Initial work of for text preprocessing fxn

#Lowercase all letters, get rid of punctuation, get rid of extra spaces on left and right side of string
# NLPData2['text'] = NLPData2['text'].map(lambda x: x.lower().translate(str.maketrans('', '', string.punctuation)).strip())
# NLPData2['text'][0] #got rid of abbreviations as well!

#Remove non-ASCII characters
# NLPData2['text'].replace({r'[^\x00-\x7F]+':''}, regex=True, inplace=True)
# NLPData2['text'][0]

#Tokenize
# NLPData2['text'] = NLPData2['text'].map(lambda x: x.split())
# NLPData2['text']

# Remove stop words
# stop_words = stopwords.words('english')
# NLPData2['text'] = NLPData2['text'].apply(lambda x: [item for item in x if item not in stop_words])
# NLPData2['text']

#Turn lists into string to lemmatize on string
# NLPData2['text'] = NLPData2['text'].apply(', '.join)
# NLPData2['text']

#Lemmatize
# def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
#     """https://spacy.io/api/annotation"""
#     texts_out = []
    
#     doc = nlp(texts) 
#     texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
#     return texts_out
# NLPData2['text']=NLPData2['text'].map(lambda x: lemmatization(x))
# NLPData2['text']

#Rejoin words --> list of words to huge string with words
# from itertools import chain
# NLPData2['text']=NLPData2.text.map(lambda x: list(chain.from_iterable(x)))
# NLPData2['text']=NLPData2['text'].apply(' '.join)
# NLPData2['text'][0]
#All pre-Processings steps into function to encapsulate logic

def preprocess(textCol):
    textCol=textCol.map(lambda x: x.lower().translate(str.maketrans('', '', string.punctuation)).strip())
    textCol.replace({r'[^\x00-\x7F]+':''}, regex=True, inplace=True)
    textCol=textCol.map(lambda x: x.split())
    stop_words = stopwords.words('english')
    textCol = textCol.apply(lambda x: [item for item in x if item not in stop_words])
    textCol = textCol.apply(' '.join) 
    
    
    def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
        """https://spacy.io/api/annotation"""
        texts_out = []
    
        doc = nlp(texts) 
        texts_out.append(token.lemma_ for token in doc if token.pos_ in allowed_postags)
        return texts_out

    textCol = textCol.map(lambda x: lemmatization(x))


    textCol=textCol.map(lambda x: list(chain.from_iterable(x)))
    textCol=textCol.apply(' '.join) 
    return textCol

cw_final['paper_txt'] = preprocess(cw_final['paper_txt'])
cw_final['paper_txt']
#cw_final.paper_txt[0] #Check
#Fill NaN's to 0 and check out class distribution
cw_final.isnull().sum()
cw_final = cw_final.fillna(0)
cw_final.isnull().sum()
cw_final.info()
cw_final.to_csv("cwclasses.csv", index=False)<jupyter_output><empty_output><jupyter_text>Two columns from annotation set are obj columns. These need to be converted to float to enable computation. Let's do this by separating out the annotations.<jupyter_code>annotations = cw_final.iloc[:,3:]
cols = annotations.select_dtypes(exclude=['float']).columns

annotations[cols] = annotations[cols].apply(pd.to_numeric, downcast='float', errors='coerce')
annotations.to_csv("cwclasses.csv", index=False)
#annotations.info() #Check
#Due to columns with obj's can't compute sum aggregation so will separate these col's out, remove non-int items and then sum
an_cols = list(annotations)
for i in an_cols:
    print((i),annotations[i].sum())<jupyter_output>in silico 26.0
in vitro 119.0
in vivo 77.0
Systematic review or meta-analysis of RCTs 7.0
RCT 7.0
Non-randomized controlled trial 10.0
Comparative study 30.0
Descriptive study 58.0
Systematic review or meta-analysis of studies other than RCTs 21.0
Other  25.0<jupyter_text>There is **class imbalance** from **380** CoronaWhy annotations. Let us take a look at a Kaggle's annotation dataset distribution as a comparison.<jupyter_code>#Kaggle datset labels

# 0 - Other
# 1 - Meta analysis
# 2 - Randomized control trial
# 3 - Non-randomized trial
# 4 - Prospective cohort
# 5 - Time-series analysis
# 6 - Retrospective cohort
# 7 - Cross-sectional
# 8 - Case control
# 9 - Case study
# 10 - Simulation


kaggleLabels = pd.read_csv("design.csv")

labels = range(0,11)
columns = ['Other', 'Meta analysis', 'Randomized control trial', 'Non-randomized trial', 'Prospective cohort', 'Time-series analysis', 'Retrospective cohort', 'Cross-sectional', 'Case control', 'Case study', 'Simulation']
mapped = dict(zip(labels,columns))
kaggleLabels['label'] = kaggleLabels['label'].map(mapped) 
kaggleLabels.head()
kaggleclasses = NLPData.merge(kaggleLabels, how='inner', left_on='paper_id', right_on='id')
kaggleclasses
kaggleclasses.to_csv("kaggleclasses.csv", index=False)
kaggleclasses.groupby('label').size().sort_values(ascending=False)<jupyter_output><empty_output><jupyter_text>____ Kaggle classes are more distinct in regards to classifying study type per publication. At the same time, there are less matches between NLPData and Kaggle dataset. For now, will work with cw_final to build classification model.  Mapping CoronaWhy Annotations to Kaggle Annotations 1. Map and One-Hot Encode Kaggle dataset to CW dataset2. See how many pub's are shared between CW and Kaggle datasets3. If there is significant amount pub's shared between cw and kaggle datasets then augment cw dataset<jupyter_code>list(df2)
df3 = kaggleLabels
df4 = df3.explode('label')
df4['label'].unique()
dic_map = {'Other':'Other',
         'Meta analysis':'Systematic review or meta-analysis of RCTs',
         'Randomized control trial':'RCT',
         'Non-randomized trial':'Non-randomized controlled trial',
         'Prospective cohort':'Comparative study',
         'Time-series analysis':'Comparative Study',
         'Retrospective cohort':'Comparative Study',
         'Cross-sectional':'Descriptive Study',
         'Case control': 'Compartative Study',
         'Case study': 'Descriptive Study',
         'Simulation': 'in silico'}
df3['mapped'] = df3.apply(lambda x: dic_map[x.label], axis = 1) 
df3.head()
#One hot encoding on mapped
one_hot = pd.get_dummies(df3['mapped'])
#drop mapped column as it is encoded now
df3 = df3.drop('mapped',axis = 1)
# Join the encoded mapped to df3
df3 = df3.join(one_hot)
df3
#See how many pub's are shared between cw and kaggle annotation sets
shared = cw_final.merge(df3, how='inner', left_on='paper_id', right_on='id')
shared #only 8, so need to merge kaggle dataset with original corpus of >7 million pub's
#From 739, 8 are shared.
#Next step would be to take out the 8, to merge same labeled columns, make sure column values only have 0/1 and union on both cw and kaggle df's.<jupyter_output><empty_output><jupyter_text>For now, will work with one annotation set: cw_final  Split Dataset into Training and Test Set (80/20 Split) and Feature Engineering<jupyter_code>cw_final.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 939 entries, 0 to 938
Data columns (total 13 columns):
 #   Column                                                         Non-Null Count  Dtype  
---  ------                                                         --------------  -----  
 0   paper_id                                                       939 non-null    object 
 1   paper_txt                                                      939 non-null    object 
 2   sha                                                            939 non-null    object 
 3   in silico                                                      939 non-null    float64
 4   in vitro                                                       939 non-null    float64
 5   in vivo                                                        939 non-null    float64
 6   Systematic review or meta-analysis of RCTs                     939 non-null    float64
 7   RCT                                                           [...]<jupyter_text>Be careful with obj columns in annotations set...Will transform them before training model.<jupyter_code># cols = y_train.select_dtypes(exclude=['float']).columns

# y_train[cols] = y_train[cols].apply(pd.to_numeric, downcast='float', errors='coerce')
train, test = train_test_split(cw_final, random_state=42, test_size=0.20, shuffle=True)<jupyter_output><empty_output><jupyter_text>Dataset Split<jupyter_code>x_train = train["paper_txt"]
y_train = train.iloc[:,3:]
x_test = test["paper_txt"]
y_test = test.iloc[:,3:]
x_train.shape, y_train.shape, x_test.shape, y_test.shape<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>vectorizer = TfidfVectorizer(analyzer='word', ngram_range=(1,3), norm='l2', max_features = 30000)
X_train = vectorizer.fit_transform(x_train)
X_test = vectorizer.fit_transform(x_test)
X_train.shape, X_test.shape
cols = y_train.select_dtypes(exclude=['float']).columns

y_train[cols] = y_train[cols].apply(pd.to_numeric, downcast='float', errors='coerce')
cols = y_test.select_dtypes(exclude=['float']).columns

y_test[cols] = y_test[cols].apply(pd.to_numeric, downcast='float', errors='coerce')
y_train.info() #Check
y_test.info() #Check
y_train.isnull().sum()
y_test.isnull().sum()
y_train = y_train.fillna(0)
y_test = y_test.fillna(0)
y_train.isnull().sum() #Check
y_test.isnull().sum() #Check<jupyter_output><empty_output><jupyter_text>Train Model<jupyter_code>#Binary Relevance
br_classifier = BinaryRelevance(classifier = SVC(), require_dense = [False, True])
br_classifier.fit(X_train, y_train)
br_predictions = br_classifier.predict(X_test)
print("Accuracy = ",accuracy_score(y_test,br_predictions.toarray()))
print("F1 score = ",f1_score(y_test,br_predictions, average="micro"))
print("Hamming loss = ",hamming_loss(y_test,br_predictions))
#Label Powerset
lp_classifier = LabelPowerset(LogisticRegression())
lp_classifier.fit(X_train, y_train)
lp_predictions = lp_classifier.predict(X_test)
print("Accuracy = ",accuracy_score(y_test,lp_predictions.toarray()))
print("F1 score = ",f1_score(y_test,lp_predictions, average="micro"))
print("Hamming loss = ",hamming_loss(y_test,lp_predictions))
#Classifier Chain
classifier = ClassifierChain(LogisticRegression())
#For classifier chain to run, need to take out all labels = 0 for y_train and y_test. 

selected_labels = y_train.columns[y_train.sum(axis = 0, skipna = True) > 0].tolist() #only >0 label values in each label column

y_train = y_train.filter(selected_labels, axis=1) 
y_test = y_test.filter(selected_labels, axis=1)  

classifier.fit(X_train, y_train)
# Run Model on Test Set

predictions = classifier.predict_proba(X_test)<jupyter_output><empty_output><jupyter_text>Assessing Model Performance<jupyter_code># Assess Performance of Model (Accuracy (mention class imbalance if there is), F1-Score, Hamming Loss) and Report Results to Imran
#for plotting metrics as a function of threshold
th = []
f = []
ham = []
ac = []
for t in range(5,60): # threshold value
    y_pred_new = (predictions >= t/100).astype(int)
    print("t =" ,t/100)
    print("Accuracy = ",accuracy_score(y_test,y_pred_new))
    print("F1 = ",f1_score(y_test,y_pred_new, average="micro")) #f1_score @ micro means to calculate metrics globally by counting the total true positives, false negatives and false positives
    print("Hamming loss = ",hamming_loss(y_test,y_pred_new))
    th.append(t)
    ac.append(accuracy_score(y_test,y_pred_new))
    f.append(f1_score(y_test,y_pred_new, average="micro"))
    ham.append(hamming_loss(y_test,y_pred_new))
plt.rcParams["figure.figsize"] = (12,6)
with plt.style.context('ggplot'):
    plt.plot(th, f)
    plt.plot(th, ham)
    plt.plot(th, ac)
    plt.legend(['F1', 'Hamming loss', 'Accuracy'], loc='center left', fontsize = 14)
    plt.ylabel("metrics", fontsize = 14)
    plt.xlabel("threshold", fontsize = 14)
    plt.title("Classifier Chain Model", fontsize = 18)
plt.show()<jupyter_output>t = 0.05
Accuracy =  0.0
F1 =  0.1747269890795632
Hamming loss =  0.28138297872340423
t = 0.06
Accuracy =  0.010638297872340425
F1 =  0.1858736059479554
Hamming loss =  0.23297872340425532
t = 0.07
Accuracy =  0.026595744680851064
F1 =  0.18233618233618235
Hamming loss =  0.15265957446808512
t = 0.08
Accuracy =  0.0797872340425532
F1 =  0.16974169741697417
Hamming loss =  0.1196808510638298
t = 0.09
Accuracy =  0.09574468085106383
F1 =  0.17358490566037738
Hamming loss =  0.11648936170212766
t = 0.1
Accuracy =  0.11702127659574468
F1 =  0.17760617760617758
Hamming loss =  0.11329787234042553
t = 0.11
Accuracy =  0.3404255319148936
F1 =  0.1717171717171717
Hamming loss =  0.08723404255319149
t = 0.12
Accuracy =  0.5957446808510638
F1 =  0.08
Hamming loss =  0.04893617021276596
t = 0.13
Accuracy =  0.6436170212765957
F1 =  0.05063291139240507
Hamming loss =  0.0398936170212766
t = 0.14
Accuracy =  0.6436170212765957
F1 =  0.025641025641025647
Hamming loss =  0.04042553191489362
t = 0.15
[...]
{'fraction_non_alphanumeric': 0.12790918891495973, 'fraction_numerical': 0.05377639672775699, 'mean_word_length': 4.254915028323892, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 44, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '340377', 'license': 'MIT', 'n_tokens_mistral': 6075, 'n_tokens_neox': 5473, 'n_words': 1386, 'path': 'classifier/Supervised_Classification_PaperStudy_ClassifierChain_BinaryRelevance_LabelPowerset.ipynb', 'repo_name': 'CoronaWhy/Classy', 'size': '15769'}