<jupyter_start><jupyter_text>Activity 3: Text Visualization Read the content of the text file named text_corpus.txt present in the data_ch2 folder. Do the pre-processing steps (like text cleaning, tokenization, stop word removal, lemmatization/stemming) which are essential. After that,  i) create a set of unique words along with their frequencies for top 50 highly occurring words ii) create a word cloud for these top 50 wordsJustify the word cloud by comparing it with the word frequency calculated.<jupyter_code>from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
%matplotlib inline
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer
import nltk
from collections import Counter
import re
text = open('data_ch2/text_corpus.txt', 'r').read()
text
#Text cleaning, tokenization, lemmatization
nltk.download('wordnet')
lemmatizer = WordNetLemmatizer()
cleaned_lemmatized_tokens = [lemmatizer.lemmatize(word.lower()) \
                             for word in word_tokenize(re.sub(r'([^\s\w]|_)+', ' ', text))]
# i) Set of unique words along with their frequencies for top 50 highly occurring words
Counter(cleaned_lemmatized_tokens).most_common(50)
#ii)  Stopwords removal and wordcloud for top 50 words
stopwords = set(STOPWORDS)
cleaned_text = ' '.join(cleaned_lemmatized_tokens)
wordcloud = WordCloud(width = 800, height = 800, 
                background_color ='white', 
                max_words=50,
                stopwords = stopwords, 
                min_font_size = 10).generate(cleaned_text)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08171912832929783, 'fraction_numerical': 0.013922518159806295, 'mean_word_length': 4.528428093645485, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '630061', 'license': 'MIT', 'n_tokens_mistral': 516, 'n_tokens_neox': 479, 'n_words': 162, 'path': 'Lesson2/Activity4.ipynb', 'repo_name': 'TrainingByPackt/Introduction-to-Natural-Language-Processing', 'size': '1652'}