<jupyter_start><jupyter_text>N-Gram Frequency Calculator from ScratchSource: https://stackoverflow.com/questions/17531684/n-grams-in-python-four-five-six-grams<jupyter_code># import packages 
import nltk 
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
import string
import pandas as pd 
import numpy as np 
from bs4 import BeautifulSoup
import urllib.request 

# download example text as an example 
url = 'http://www.textfiles.com/etext/FICTION/mobydick'
response = urllib.request.urlopen(url) 
html = response.read()
soup = BeautifulSoup(html,"html5lib")
data = soup.get_text(strip=True)
print (data[:1000])
# since the file is too big, reduce the file 
data = data[:int(len(data)/3)]
# remove punctuations 
translator = str.maketrans('', '', string.punctuation)
data = data.translate(translator)

# split data into tokens 
tokens = [t for t in data.split()] 

tokens[:10]
# remove stop words 
clean_tokens = tokens.copy() # or tokens[:]
for token in tokens:
    if token.lower() in stopwords.words('english'): 
        clean_tokens.remove(token)    

clean_tokens[:10]<jupyter_output><empty_output><jupyter_text>Function<jupyter_code>def ngram(tokens,ngrams): 
    '''
    Looks through each word in the document and grab next ngrams and returns tuple 
    '''
    return [tuple(tokens[i:i+ngrams]) for i in range(len(tokens)-ngrams+1)]

def FrequencyCalculator(tokens):
    ''' 
    Given list of tokens, calculate each token's frequency. 
    By putting the final result into the dictionary, it removes duplicate entries. 
    '''
    word_count = [tokens.count(word) for word in tokens]
    return dict(zip(tokens,word_count))<jupyter_output><empty_output><jupyter_text>Unigram<jupyter_code>ngram_words = ngram(clean_tokens,ngrams=1)
word_count_dict = FrequencyCalculator(ngram_words)

# Show top 20 most frequency words 
result = sorted(word_count_dict.items(), key=lambda x: x[1],reverse=True)[:20]
result<jupyter_output><empty_output><jupyter_text>Bigram<jupyter_code>ngram_words2 = ngram(clean_tokens,ngrams=2)
word_count_dict2 = FrequencyCalculator(ngram_words2)

# Show top 20 most frequency words 
result2 = sorted(word_count_dict2.items(), key=lambda x: x[1],reverse=True)[:20]
result2<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11140466754733598, 'fraction_numerical': 0.01673271686481726, 'mean_word_length': 5.761904761904762, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '514548', 'license': 'MIT', 'n_tokens_mistral': 804, 'n_tokens_neox': 758, 'n_words': 180, 'path': '*NLP/N-Gram From Scratch.ipynb', 'repo_name': 'jasjung/Machine_Learnin', 'size': '2271'}