<jupyter_start><jupyter_text>Zipf's Empirical Formulae<jupyter_code>import re
from operator import itemgetter
import nltk
import pandas as pd

frequency = {}
words_emma = nltk.Text(nltk.corpus.gutenberg.words('austen-emma.txt'))
for word in words_emma:
    if word not in frequency:
        frequency[word] =  1
    else:
        frequency[word] += 1
from collections import Counter
freq = Counter(frequency)
freq.most_common(10)
col_name = ['Rank', 'Freq', 'Rank*Freq']
data = pd.DataFrame(columns = col_name)
data.head()
for  rank, (word, frq) in enumerate(freq.most_common(len(freq)), start = 1):
    data.loc[word] = [rank, frq, rank*frq]
print(data)<jupyter_output>Rank   Freq Rank*Freq
,                   1  11454     11454
.                   2   6928     13856
to                  3   5183     15549
the                 4   4844     19376
and                 5   4672     23360
of                  6   4279     25674
I                   7   3178     22246
a                   8   3004     24032
was                 9   2385     21465
her                10   2381     23810
;                  11   2199     24189
it                 12   2128     25536
in                 13   2118     27534
not                14   2101     29414
"                  15   2004     30060
be                 16   1970     31520
she                17   1778     30226
that               18   1730     31140
you                19   1677     31863
had                20   1606     32120
as                 21   1387     29127
--                 22   1382     30404
he                 23   1365     31395
for                24   1321     31704
have               25   1[...]<jupyter_text>2nd Method<jupyter_code>import re
from operator import itemgetter
import nltk
import pandas as pd
frequency = {}
words_emma = nltk.Text(nltk.corpus.gutenberg.words('austen-emma.txt'))
for word in words_emma:
    count = frequency.get(word, 0)
    frequency[word] = count + 1
rank = 1
column_header = ['Rank', 'Frequency', 'Frequency*Rank']
df = pd.DataFrame(columns = column_header)

for word, freq in reversed(sorted(frequency.items(), key = itemgetter(1))):
    df.loc[word] = [rank, freq, rank * freq]
    rank += 1
print(df)
frequency['Austen']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08, 'fraction_numerical': 0.12, 'mean_word_length': 1.620488940628638, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '340696', 'license': 'MIT', 'n_tokens_mistral': 891, 'n_tokens_neox': 714, 'n_words': 205, 'path': "nltk/General/4_Zip's_Empirical_Formulae.ipynb", 'repo_name': 'sudevansujit/7_NLP', 'size': '2250'}