<jupyter_start><jupyter_text><jupyter_code>import nltk,re,pprint,collections
from nltk import word_tokenize
from urllib import request
from nltk.util import ngrams
from collections import Counter,defaultdict
nltk.download('punkt')
import spacy
import numpy as np
import pandas as pd
from collections import Counter, defaultdict
from itertools import tee
#This is a text corpora which can we use for testing purpose
#url = "http://www.gutenberg.org/files/2554/2554-0.txt"
#response = request.urlopen(url)
#raw = response.read().decode('utf8')
#wordlist = re.sub("[^\w]", " ",  raw).split()


inputstring = "I am Sam. Sam I am. I do not like green eggs."
normalizedstring = inputstring.lower()
#a = re.sub(r'[?|$|.|!|\n|\r|\]',r'',normalizedstring).split()
#print(a)
#dictionary = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',' ',"'"]
inputwordlist = re.sub("[^\w]", " ",  normalizedstring).split()
nlp = spacy.load('en_core_web_sm', disable=['parser', 'tagger', 'ner'])
inputwordlist
def get_bigrams(inputwordlist):
  bigrams = []
  for i in range(len(inputwordlist)-(2-1)):
    bigrams.append(inputwordlist[i:i+2])
  return bigrams
get_bigrams(inputwordlist)
import csv
with open('bigram_probability.csv','w',newline = '') as f:
    thewriter = csv.writer(f)
    for bigrams in get_bigrams(inputwordlist):
        probability_bigram = get_bigrams(inputwordlist).count(bigrams) / inputwordlist.count(bigrams[0])
        #print(bigrams,probability_bigram)
        thewriter.writerow([bigrams,probability_bigram])
tokenized = nlp.tokenizer.pipe(a)
tokenized = [t.text for doc in tokenized for t in doc]
tokenized
from collections import Counter, defaultdict
from itertools import tee
Bigrams = ngrams(a,2)
BigramFreq = collections.Counter(Bigrams)
bigram_prob = BigramFreq / BigramFreq.Count(Bigrams[0])
bigram_prob
def make_bigrams(iterator):
    a, b = tee(iterator,2)
    next(b)
    return zip(a,b)
bigrams = Counter(make_bigrams(a))
bigrams.most_common(10)
def make_trigrams(iterator):
    a, b, c = tee(iterator, 3)
    next(b)
    next(c)
    next(c)
    return zip(a, b, c)
trigrams = Counter(make_trigrams(a))
trigrams.most_common(10)
d = defaultdict(Counter)
for a, b, c in trigrams:
    d[a, b][c] += trigrams[a, b, c]
d["don't",'like']
list(d["i", "am"].elements())
import random

def pick(counter):
  return random.choice(list(counter.elements()))

prefix = "i","am"

print(" ".join(prefix), end=' ')
for i in range(8):
    suffix = pick(d[prefix])
    print(suffix, end=' ')
    prefix = prefix[1], suffix<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17498081350729086, 'fraction_numerical': 0.00920951650038373, 'mean_word_length': 5.485074626865671, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '636276', 'license': 'MIT', 'n_tokens_mistral': 1003, 'n_tokens_neox': 949, 'n_words': 212, 'path': 'Wordprediction_model.ipynb', 'repo_name': 'Aditya3107/CTCDecoder', 'size': '2606'}