<commit_before>import string
from collections import defaultdict
import random
import itertools
import bisect

def pairwise(iterable):
    """
    Yield pairs of consecutive elements in iterable.

    >>> list(pairwise('abcd'))
    [('a', 'b'), ('b', 'c'), ('c', 'd')]
    """
    iterator = iter(iterable)
    try:
        a = iterator.__next__()
    except StopIteration:
        return
    for b in iterator:
        yield a, b
        a = b

with open('corpus.txt') as f:
    corpus = f.read()

corpus = ''.join(c.lower() if c in string.ascii_letters else ' ' for c in corpus).split()
corpus = [word for word in corpus if len(word) > 1]
counts = defaultdict(lambda: defaultdict(int))
for word in corpus:
    for a, b in pairwise(word):
        counts[a][b] += 1

sys_rand = random.SystemRandom()

def gen_password(l=16):
    password = []
    state = sys_rand.choice(string.ascii_lowercase)
    password.append(state)
    while len(password) < l:
        choices, weights = zip(*counts[state].items())
        cumdist = list(itertools.accumulate(weights))
        x = sys_rand.random() * cumdist[-1]
        state = choices[bisect.bisect(cumdist, x)]
        password.append(state)
    cap, num = sys_rand.sample(range(len(password)), 2)
    password[num] = str(sys_rand.randrange(10))
    password[cap] = password[cap].upper()
    return ''.join(password)

print(gen_password())
<commit_msg>Made markov.py an importable class.<commit_after>import string
from collections import defaultdict
import random
import itertools
import bisect

def pairwise(iterable):
    """
    Yield pairs of consecutive elements in iterable.

    >>> list(pairwise('abcd'))
    [('a', 'b'), ('b', 'c'), ('c', 'd')]
    """
    iterator = iter(iterable)
    try:
        a = iterator.__next__()
    except StopIteration:
        return
    for b in iterator:
        yield a, b
        a = b

class Markov(object):
    def __init__(self, corpus_file='corpus.txt'):
        with open(corpus_file) as f:
            corpus = f.read()
        corpus = ''.join(c.lower() if c in string.ascii_letters else ' ' for c in corpus).split()
        corpus = [word for word in corpus if len(word) > 1]
        self.counts = defaultdict(lambda: defaultdict(int))
        for word in corpus:
            for a, b in pairwise(word):
                self.counts[a][b] += 1
        self.sys_rand = random.SystemRandom()

    def gen_password(self, l=16):
        password = []
        state = self.sys_rand.choice(string.ascii_lowercase)
        password.append(state)
        while len(password) < l:
            choices, weights = zip(*self.counts[state].items())
            cumdist = list(itertools.accumulate(weights))
            x = self.sys_rand.random() * cumdist[-1]
            state = choices[bisect.bisect(cumdist, x)]
            password.append(state)
        cap, num = self.sys_rand.sample(range(len(password)), 2)
        password[num] = str(self.sys_rand.randrange(10))
        password[cap] = password[cap].upper()
        return ''.join(password)

{'fraction_non_alphanumeric': 0.13612392880685564, 'fraction_numerical': 0.005273566249176005, 'mean_word_length': 2.7422934648581996, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3035697', 'n_tokens_mistral': 984, 'n_tokens_neox': 953, 'n_words': 247}