<commit_before>#!python3

import string
import random
import time
import sys


'''
This is an implementation of a markov chain used for text generation.
Just pass a file name as an argument and it should load it up, build a markov
chain with a state for each word(s), and start walking through the chain, writing
incoherent text to the terminal.
'''


asciiset = set(string.ascii_letters)
asciiset.add(' ')
asciiset.add('.')
def strip2ascii(txt):
    return ''.join([ch for ch in txt if ch in asciiset])


def tokenize(fname):
    '''
    Generate tokens defined by
    - Sequences of characters that aren't spaces
    - Periods

    For example, 'This is a test. Ok.' => ('This', 'is', 'a', 'test', '.', 'Ok, '.')
    '''
    with open(fname, 'r') as f:
        for line in f:
            stripped = strip2ascii(line)
            for word in stripped.split():
                
                if word[-1] == '.':
                    yield word[:-1]
                    yield '.'
                else:
                    yield word


def buildtransitionmap(tokens, order):
    dct = {}
    prev = ('',)*order

    for token in tokens:
        if prev in dct:
            dct[prev].append(token)
        else:
            dct[prev] = [token]

        prev = prev[1:]+(token,)
        
    return dct


def transition(word, transmap):
    return random.choice(transmap[word])


def eternalramble(fname, order):
    '''
    Walk through the markov chain printing out words to the terminal one at a time
    '''
    transmap = buildtransitionmap(tokenize(fname), order)
    prev = random.choice(list(transmap.keys()))
    while True:
        word = transition(prev, transmap)
        print(word, end=' ')
        prev  = prev[1:]+(word,)
        sys.stdout.flush()
        time.sleep(0.25)


def printusage():
    print('Usage: markov filename order')
    print('  filename: the filename of the text to base the markov chain on.')
    print('  order: how many consecutive words make up each state (2 works well)')


def launch():
    if len(sys.argv) != 3:
        printusage()
        return

    try:<commit_msg>Fix error when reaching a dead end in the markov chain<commit_after>#!python3

import string
import random
import time
import sys


'''
This is an implementation of a markov chain used for text generation.
Just pass a file name as an argument and it should load it up, build a markov
chain with a state for each word(s), and start walking through the chain, writing
incoherent text to the terminal.
'''


asciiset = set(string.ascii_letters)
asciiset.add(' ')
asciiset.add('.')
def strip2ascii(txt):
    return ''.join([ch for ch in txt if ch in asciiset])


def tokenize(fname):
    '''
    Generate tokens defined by
    - Sequences of characters that aren't spaces
    - Periods

    For example, 'This is a test. Ok.' => ('This', 'is', 'a', 'test', '.', 'Ok, '.')
    '''
    with open(fname, 'r') as f:
        for line in f:
            stripped = strip2ascii(line)
            for word in stripped.split():
                
                if word[-1] == '.':
                    yield word[:-1]
                    yield '.'
                else:
                    yield word


def buildtransitionmap(tokens, order):
    dct = {}
    prev = ('',)*order

    for token in tokens:
        if prev in dct:
            dct[prev].append(token)
        else:
            dct[prev] = [token]

        prev = prev[1:]+(token,)
        
    return dct


def walk(transmap, prev=None):
    if prev == None:
        prev = random.choice(list(transmap.keys()))

    while True:
        if not prev in transmap:
            prev = random.choice(list(transmap.keys()))

        word = random.choice(transmap[prev])
        yield word
        prev = prev[1:]+(word,)


def eternalramble(fname, order):
    '''
    Walk through the markov chain printing out words to the terminal one at a time
    '''
    transmap = buildtransitionmap(tokenize(fname), order)
    for word in walk(transmap):
        print(word, end=' ')
        sys.stdout.flush()
        time.sleep(0.25)


def printusage():
    print('Usage: markov filename order')
    print('  filename: the filename of the text to base the markov chain on.')
    print('  order: how many consecutive words make up each state (2 works well)')


def launch():
    if len(sys.argv) != 3:
        printusage()
        return

    try:
{'fraction_non_alphanumeric': 0.11480637813211846, 'fraction_numerical': 0.005466970387243736, 'mean_word_length': 2.212143379663497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5157366', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1271, 'n_words': 466}