<jupyter_start><jupyter_text>Term Frequency Demo<jupyter_code>import nltk 
from nltk.probability import FreqDist
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
stop_words
# Read the Corpus
nltk.download('gutenberg')
words = nltk.Text(nltk.corpus.gutenberg.words('bryant-stories.txt'))

print(words[:500])
# Convert to small letters
words = [word.lower() for word in words if word.isalpha()]
words = [word.lower() for word in words if word not in stop_words]
fDist = FreqDist(words)
print(len(words))
print(len(set(words)))
for x, v in fDist.most_common(10):
    print('{:<10}{:<10}'.format(x, v))
for x, v in fDist.most_common(10):
    print('{:<10}{:<10}'.format(x, v/len(fDist)))
fDist
fDist['little']
fDist.items()
fDist.keys()
fDist.get('yes')
from nltk.tokenize import word_tokenize
from nltk.probability import FreqDist
sent = 'This is an example sentence'
fdist = FreqDist()
for word in word_tokenize(sent):
    fdist[word.lower()] += 1
fdist
# An equivalent way to do this is with the initializer:

fdist = FreqDist(word.lower() for word in word_tokenize(sent)) 
fdist<jupyter_output><empty_output><jupyter_text>Add<jupyter_code>#Add
FreqDist('abbb') + FreqDist('bcc')<jupyter_output><empty_output><jupyter_text>Finding frequency of Common Alphabets/Words in two strings<jupyter_code># Intersection
FreqDist('abbb') & FreqDist('bcc')
FreqDist('hello boy how are you') & FreqDist('hello girl who are you')
FreqDist('hello boy how are you'.split()) & FreqDist('hello hello girl who are you'.split())<jupyter_output><empty_output><jupyter_text>Union<jupyter_code># Union
FreqDist('abbb') | FreqDist('bcc')<jupyter_output><empty_output><jupyter_text>Substract<jupyter_code># Substract
FreqDist('abbbc') - FreqDist('bccd')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14437604924454392, 'fraction_numerical': 0.008953553441522105, 'mean_word_length': 7.514285714285714, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '175459', 'license': 'MIT', 'n_tokens_mistral': 690, 'n_tokens_neox': 676, 'n_words': 147, 'path': 'nltk/General/2_Term_Frequency_Demo.ipynb', 'repo_name': 'sudevansujit/7_NLP', 'size': '1787'}