<jupyter_start><jupyter_text>Visualize Word Count (CountVectorizer)<jupyter_code>vect = CountVectorizer()
vect.fit(df['Tokens'])

# Create DTM
dtm = vect.transform(df['Tokens'])

dtm = pd.DataFrame(dtm.todense(), columns=vect.get_feature_names())

most_common_words = pd.DataFrame(dtm.sum().sort_values(ascending=False))

most_common_words.reset_index(inplace=True)  # Resetting the index to replace column names

most_common_words.columns = ['word', 'count']

most_common_words.head()
import matplotlib.pyplot as plt

# Visualize for better understanding
rcParams['figure.figsize'] = 25, 10

ten_most_common_words = most_common_words.head(25)  # The 10 most common words

plt.bar(x=ten_most_common_words['word'], height=ten_most_common_words['count'])
plt.ylabel('Number of Times Word Used')
plt.title('Most commonly used words in Tech Support Subreddits')
plt.show();

# Few more stop words to add here: ['im', 'like', 'know', 'dont', 'ive']<jupyter_output><empty_output><jupyter_text>Visualize Word Frequency (TF-IDF Vectorizer)<jupyter_code>vect = TfidfVectorizer()
vect.fit(df['Tokens'])

# Create DTM
dtm = vect.transform(df['Tokens'])

dtm = pd.DataFrame(dtm.todense(), columns=vect.get_feature_names())

most_common_words = pd.DataFrame(dtm.sum().sort_values(ascending=False))

most_common_words.reset_index(inplace=True)  # Resetting the index to replace column names

most_common_words.columns = ['word', 'count']

most_common_words.head()

ten_most_common_words = most_common_words.head(25)  # The 10 most common words

plt.bar(x=ten_most_common_words['word'], height=ten_most_common_words['count'])
plt.ylabel('Frequency of Word')
plt.title('Most commonly used words in Tech Support Subreddits')
plt.show();

# Few more stop words to add here: ['im', 'like', 'know', 'dont', 'ive']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13823368074602305, 'fraction_numerical': 0.006582556226001097, 'mean_word_length': 7.563380281690141, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '52791', 'license': 'MIT', 'n_tokens_mistral': 662, 'n_tokens_neox': 642, 'n_words': 133, 'path': 'notebooks/data_exploration.ipynb', 'repo_name': 'LambdaSchool-Post-Here-1/DS', 'size': '1823'}