<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Read Dataset<jupyter_code>dataset=pd.read_csv("/content/drive/MyDrive/Machine Learning/ML DATASET/COVID-19 AR Clinical Correlates July202020_text.csv")
print(dataset.shape)
dataset.head()
dataset.columns=dataset.iloc[0]
dataset=dataset[1:]
dataset
dataset=dataset[['patient_id','Radiologist KEY FINDINGS']]
dataset = dataset.reset_index(drop=True)
dataset
# df.rename(columns={'Image Study Description' : 'Image_Study_Description', 'study modality': 'study_modality', 'Radiologist KEY FINDINGS': 'Radiologist_KEY_FINDINGS' }, inplace=True)
# df
dataset.dropna(inplace=True)
print(dataset.shape)
dataset
dataset['Radiologist KEY FINDINGS'] = dataset.groupby(['patient_id'])['Radiologist KEY FINDINGS'].transform(lambda x : ' '.join(x))
  
# drop duplicate data
dataset = dataset.drop_duplicates()
dataset = dataset.reset_index(drop=True)
dataset.shape
dataset<jupyter_output><empty_output><jupyter_text>Tokenize, lemmatization and stopwords remove<jupyter_code># removing punctuations
import re
dataset['Radiologist KEY FINDINGS'] = [re.sub(r'[^\w\s]','',s) for s in dataset['Radiologist KEY FINDINGS']]
dataset
# Tokenize
import nltk
nltk.download('punkt')
nltk.download('wordnet')
dataset['Radiologist KEY FINDINGS'] = [nltk.word_tokenize(s) for s in dataset['Radiologist KEY FINDINGS']]
dataset
#removing stopwords
from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = stopwords.words('english')
print(stop_words)
# removing stopwords
def remove_stopword(word_list):
    new_word_list = []
    for word in word_list:
        if word.lower() not in stop_words:
            new_word_list.append(word)
    return new_word_list

dataset['Radiologist KEY FINDINGS'] = [remove_stopword(this_list) for this_list in dataset['Radiologist KEY FINDINGS']]
dataset
# Lemmatization
from nltk.stem.wordnet import WordNetLemmatizer

lemmatizer  = WordNetLemmatizer()
lemmatized_dataset = []

for this_str in dataset['Radiologist KEY FINDINGS']:
    lemmatized_str = []
    for word in this_str:
        lemmatized_str.append(lemmatizer.lemmatize(word))
    lemmatized_dataset.append(lemmatized_str)

dataset['Radiologist KEY FINDINGS'] = lemmatized_dataset
dataset<jupyter_output><empty_output><jupyter_text>TFIDF vectorization<jupyter_code>dataset["Radiologist KEY FINDINGS"]= dataset["Radiologist KEY FINDINGS"].str.join(" ")
dataset
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd

vectorizer = TfidfVectorizer()
# vectors = vectorizer.fit_transform(dataset['Radiologist KEY FINDINGS'])
vectors = vectorizer.fit_transform(list(dataset['Radiologist KEY FINDINGS']))
feature_names = vectorizer.get_feature_names()
matrix = vectors.todense()

tfidf_df = pd.DataFrame(matrix, columns = feature_names)

tfidf_df<jupyter_output>/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.
  warnings.warn(msg, category=FutureWarning)<jupyter_text>Checking for Patients are in same sequence or not<jupyter_code>patient_1 = dataset['patient_id']  #Text Dataframe
patient_1 = list(patient_1)
print(patient_1)
# df contains the pathology dataframe
filepath =  r"/content/drive/MyDrive/Machine Learning/ML DATASET/COVID-19 AR Clinical Correlates July202020 (csv).csv"
df = pd.read_csv(filepath, header = 1)
print(df.tail(3))

patient_2 = list(df['PATIENT_ID'][:-2])
print(patient_2)
miss_match = 0
for idx, name in enumerate(patient_1):
    if name != patient_2[idx]:
        miss_match += 1
print(miss_match) 
# as miss_match is 0, we can say in both dataframe text_data and the pathology data 
# the patients_ID are in same Sequence<jupyter_output>0<jupyter_text>Adding Mortality (Label) into tfidf dataset<jupyter_code>mortality_truth = list(df['MORTALITY'])[:-2]

from sklearn import preprocessing
le = preprocessing.LabelEncoder()
mortality_truth = le.fit_transform(mortality_truth)

tfidf_df['MORTALITY'] = mortality_truth
X = tfidf_df.iloc[:,:-1]
y = tfidf_df['MORTALITY']

print(X.shape)
print(y.shape)<jupyter_output>(105, 310)
(105,)<jupyter_text>Train_test split<jupyter_code>from sklearn.model_selection import train_test_split
 
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 10)
print(y_test.value_counts())
print(y_train.value_counts())
print(f"X_train shape {X_train.shape}")
print(f"y_train shape {y_train.shape}")
print(f"X_test shape {X_test.shape}")
print(f"x_train shape {y_test.shape}")<jupyter_output>0    28
1     4
Name: MORTALITY, dtype: int64
0    67
1     6
Name: MORTALITY, dtype: int64
X_train shape (73, 310)
y_train shape (73,)
X_test shape (32, 310)
x_train shape (32,)<jupyter_text>Oversampling using SMOTE<jupyter_code># Before Oversampling
print(y_test.value_counts())
print(y_train.value_counts())
print("\n\n")


from imblearn.over_sampling import SMOTE, SVMSMOTE, SMOTEN
sm = SMOTE(random_state = 42)
X_train, y_train = sm.fit_resample(X_train, y_train)

# from imblearn.combine import SMOTETomek
# smt = SMOTETomek(random_state=42)
# X_train, y_train = smt.fit_resample(X_train, y_train)

# After Oversampling
print(y_test.value_counts())
print(y_train.value_counts())
y_train.value_counts().plot.bar()
# from imblearn.over_sampling import RandomOverSampler
# ros = RandomOverSampler(random_state=42)
# X_train_res, y_train_res = ros.fit_resample(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Applying PCA<jupyter_code>from sklearn.decomposition import PCA
pca = PCA(n_components=0.95)
X_train = pca.fit_transform(X_train)
X_test = pca.transform(X_test)

print(X_train.shape)
print(X_test.shape)<jupyter_output>(134, 49)
(32, 49)<jupyter_text>Random Forest Classifier<jupyter_code>from sklearn.ensemble import RandomForestClassifier
RF_clf =RandomForestClassifier(n_estimators=100)
RF_clf.fit(X_train,y_train)
y_pred=RF_clf.predict(X_test)

from sklearn import metrics
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))

from sklearn.metrics import confusion_matrix
print(confusion_matrix(y_test, y_pred))

from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred))<jupyter_output>Accuracy: 0.875
[[28  0]
 [ 4  0]]
              precision    recall  f1-score   support

           0       0.88      1.00      0.93        28
           1       0.00      0.00      0.00         4

    accuracy                           0.88        32
   macro avg       0.44      0.50      0.47        32
weighted avg       0.77      0.88      0.82        32<jupyter_text>RUSBoostClassifier<jupyter_code>from imblearn.ensemble import RUSBoostClassifier
clf = RUSBoostClassifier(random_state=42)
clf.fit(X_train, y_train)  
y_pred = clf.predict(X_test) 

from sklearn import metrics
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))

from sklearn.metrics import confusion_matrix
print(confusion_matrix(y_test, y_pred))

from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred))<jupyter_output>Accuracy: 0.875
[[28  0]
 [ 4  0]]
              precision    recall  f1-score   support

           0       0.88      1.00      0.93        28
           1       0.00      0.00      0.00         4

    accuracy                           0.88        32
   macro avg       0.44      0.50      0.47        32
weighted avg       0.77      0.88      0.82        32<jupyter_text>KNN CLassifier Model<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
from sklearn import metrics
for i in range(1,20):
    knn = KNeighborsClassifier(n_neighbors=i)
    knn.fit(X_train, y_train)
    y_pred = knn.predict(X_test)
    print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
knn = KNeighborsClassifier(n_neighbors=3)
knn.fit(X_train, y_train)
y_pred = knn.predict(X_test)

from sklearn import metrics
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))

from sklearn.metrics import confusion_matrix
print(confusion_matrix(y_test, y_pred))

from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10463287770654409, 'fraction_numerical': 0.0279424216765453, 'mean_word_length': 5.230595327807084, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '419285', 'license': 'MIT', 'n_tokens_mistral': 3108, 'n_tokens_neox': 2943, 'n_words': 591, 'path': 'old_and_other_codes/Text_preprocessing_TFIDF.ipynb', 'repo_name': 'R4HUL-ROY/Multimodal_feature_extraction', 'size': '8267'}