<jupyter_start><jupyter_text>Text Generation using a random walk over a Markov Chain Using North of Boston by <NAME>, Guttenburg Project<jupyter_code>import re
import random
OPEN_DATA_URL = '../data/robertfrost/pg3026.txt'

f=open(OPEN_DATA_URL,'r',encoding='utf-8')
text=f.read()
f.close()
HANDLE = '@\w+\n'
LINK = 'https?://t\.co/\w+'
SPECIAL_CHARS = '&lt;|&lt;|&amp;|#'
PARA='\n+'
def clean(text):
    #text = re.sub(HANDLE, ' ', text)
    text = re.sub(LINK, ' ', text)
    text = re.sub(SPECIAL_CHARS, ' ', text)
    text = re.sub(PARA, '\n', text)
    return text

text = clean(text)
corpus=text.split()
corpus_length=len(corpus)
corpus_length
succ_func={}
corpus_counter=0
for token in corpus:
    corpus_counter=corpus_counter+1
    if corpus_counter<corpus_length:
        if token not in succ_func.keys():
            succ_func[token]=[corpus[corpus_counter]]
        else:
            succ_func[token].append(corpus[corpus_counter])
succ_func
initial_states=['The','A','I','He','She','If','This','Why','Where']
def generate_words(k=5):
    initial_state=random.choice(initial_states)
    current_state=initial_state
    text=current_state+' '
    for i in range(k):
        succ_state=random.choice(succ_func[current_state])
        text=text+succ_state+' '
        current_state=succ_state
    print(text.split('.')[0])

for k in range(3,5):
    for j in range(10):
        generate_words(k)<jupyter_output>A horse's hoof pawed 
She let me go!" 
This and hear some 
If you must include 
A man's affairs
The rumbling voice said
The lawyer said
The Purple Lady's Slipper 
Why didn't see at 
If he's kinder than 
She does she couldn't lead 
He has gone
Where you couldn't keep in 
This is your efforts and 
The Seven Caves that was 
She reached the neighbours, Being 
If you 'AS-IS' WITH NO 
Why not so With white-faced 
I said there and any 
Why didn't think we know
{'fraction_non_alphanumeric': 0.12846481876332622, 'fraction_numerical': 0.006929637526652452, 'mean_word_length': 4.045698924731183, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '450252', 'license': 'MIT', 'n_tokens_mistral': 685, 'n_tokens_neox': 660, 'n_words': 181, 'path': 'Chapter07/Exercise 7.01/Exercise 7.01.ipynb', 'repo_name': 'abhishekr128/The-Natural-Language-Processing-Workshop', 'size': '1882'}