<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd
dataset=pd.read_csv("/content/drive/MyDrive/Machine Learning/ML DATASET/COVID-19 AR Clinical Correlates July202020_text.csv")
print(dataset.shape)
dataset.head()
dataset.columns=dataset.iloc[0]
dataset=dataset[1:]
dataset
dataset=dataset[['patient_id','Radiologist KEY FINDINGS']]
dataset = dataset.reset_index(drop=True)
dataset
dataset.dropna(inplace=True)
print(dataset.shape)
dataset
dataset['Radiologist KEY FINDINGS'] = dataset.groupby(['patient_id'])['Radiologist KEY FINDINGS'].transform(lambda x : ' '.join(x))
  
# drop duplicate data
dataset = dataset.drop_duplicates()
dataset = dataset.reset_index(drop=True)
dataset.shape
# removing punctuations
import re
dataset['Radiologist KEY FINDINGS'] = [re.sub(r'[^\w\s]','',s) for s in dataset['Radiologist KEY FINDINGS']]
dataset
# Tokenize
import nltk
nltk.download('punkt')
nltk.download('wordnet')
dataset['Radiologist KEY FINDINGS'] = [nltk.word_tokenize(s) for s in dataset['Radiologist KEY FINDINGS']]
dataset
#removing stopwords
from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = stopwords.words('english')
print(stop_words)
# removing stopwords
def remove_stopword(word_list):
    new_word_list = []
    for word in word_list:
        if word.lower() not in stop_words:
            new_word_list.append(word)
    return new_word_list

dataset['Radiologist KEY FINDINGS'] = [remove_stopword(this_list) for this_list in dataset['Radiologist KEY FINDINGS']]
dataset
# Lemmatization
from nltk.stem.wordnet import WordNetLemmatizer

lemmatizer  = WordNetLemmatizer()
lemmatized_dataset = []

for this_str in dataset['Radiologist KEY FINDINGS']:
    lemmatized_str = []
    for word in this_str:
        lemmatized_str.append(lemmatizer.lemmatize(word))
    lemmatized_dataset.append(lemmatized_str)

dataset['Radiologist KEY FINDINGS'] = lemmatized_dataset
dataset
bigger_list = list(dataset["Radiologist KEY FINDINGS"])
print(bigger_list)
token_words = []
for nested_list in bigger_list:
    token_words += nested_list

for idx, val in enumerate(token_words):
    token_words[idx] = token_words[idx].lower()

print(len(token_words))
print(token_words)    
# generating WordCloud
 
from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
import pandas as pd
 
comment_words = ''
stopwords = set(STOPWORDS)
comment_words += " ".join(token_words)+" "
 
wordcloud = WordCloud(width = 800, height = 800,
                background_color ='white',
                stopwords = stopwords,
                min_font_size = 10).generate(comment_words)
 
# plot the WordCloud image                      
plt.figure(figsize = (8, 8), facecolor = None)
plt.imshow(wordcloud)
plt.axis("off")
plt.tight_layout(pad = 0)
 
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11201413427561838, 'fraction_numerical': 0.007420494699646643, 'mean_word_length': 5.277161862527716, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '204482', 'license': 'MIT', 'n_tokens_mistral': 974, 'n_tokens_neox': 933, 'n_words': 206, 'path': 'old_and_other_codes/wordCloud_Covid19_keyFindings.ipynb', 'repo_name': 'R4HUL-ROY/Multimodal_feature_extraction', 'size': '2830'}