<jupyter_start><jupyter_text><jupyter_code>#ref :https://sookocheff.com/post/nlp/ngram-modeling-with-markov-chains/
s = "I am Sam. Sam I am. I do not like green eggs and ham."
tokens = s.split(" ")
  
bigrams = [(tokens[i],tokens[i+1]) for i in range(0,len(tokens)-1)]
bigrams
import random
random.sample(['am', 'am.', 'do'], 1)
class MarkovChain:

    def __init__(self):
        self.memory = {}

    def _learn_key(self, key, value):
        if key not in self.memory:
            self.memory[key] = []

        self.memory[key].append(value)

    def learn(self, text):
        tokens = text.split(" ")
        bigrams = [(tokens[i], tokens[i + 1]) for i in range(0, len(tokens) - 1)]
        for bigram in bigrams:
            self._learn_key(bigram[0], bigram[1])

    def _next(self, current_state):
        next_possible = self.memory.get(current_state)

        if not next_possible:
            next_possible = self.memory.keys()

        return random.sample(next_possible, 1)[0]

    def babble(self, amount, state=''):
        if not amount:
            return state

        next_word = self._next(state)
        return state + ' ' + self.babble(amount - 1, next_word)

m = MarkovChain()
m.learn('I am Sam.')
m.memory
m.learn('I am Kevin.')
m.memory
m.babble(5,'the')
m.learn('Putting it all together we have a simple Markov Chain that can learn bigrams and babble text given the probability of bigrams that it has learned. Markov Chain’s are a simple way to store and query n-gram probabilities. Full source code for this example follows.')
m.memory<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13048933500627352, 'fraction_numerical': 0.008155583437892095, 'mean_word_length': 2.9775561097256857, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '132743', 'license': 'MIT', 'n_tokens_mistral': 546, 'n_tokens_neox': 515, 'n_words': 155, 'path': 'markov_chain.ipynb', 'repo_name': 'maneeshdisodia/DearML', 'size': '1594'}