<commit_before>import re

import markovify

# takes irc logs (as ZNC writes them, and in utf-8) and generates
# markov models for each nick in them
# put your concatenated logs in one file and set the markovLogFile
# variable in the config json

# for concatenations you can do something like
#   find ~/.znc -iname "*#*.log" -exec cat {} \; > mark.txt

nick_models = {}  # mapping nick -> markov model


# prepare the markov chains for users
def prepare():
    msg_regex = re.compile(r'^\[..:..:..\] <(.*?)> (.*)$')

    text = ''
    try:
        dict_file_name = yui.config_val('markovLogFile')
        with open(dict_file_name, errors='replace') as f:
            text = f.read()
    except Exception as ex:
        return

    # go through all lines and extract nick + messages
    nick_lines = {}
    for line in text.splitlines():
        match = msg_regex.match(line)
        if not match:
            continue
        u = match.group(1).lower()
        m = match.group(2)
        # omit any sentences shorter than 3 words
        if len(m.split()) < 3:
            continue
        if u not in nick_lines:
            nick_lines[u] = list()
        nick_lines[u].append(m)

    # generate markov models for each nick
    for u, m in nick_lines.items():
        try:
            mark = markovify.NewlineText('\n'.join(m), state_size=3)
            nick_models[u] = mark
        except Exception as ex:
            pass


# generate a sentence for a given nick
def generate_sentence(nick):
    for i in range(1, 100):
        ret = nick_models[nick.lower()].make_sentence()
        if ret:
            return ret
    return None


@yui.threaded
@yui.command('markov', 'mark')
def markov(argv, user):
    """Generate a random sentence for a given nick. Usage: markov [nick]"""
    name = user.nick
    if len(argv) > 1:
        name = argv[1]
    if name not in nick_models.keys():<commit_msg>ya miss'd one<commit_after>import re

import markovify

# takes irc logs (as ZNC writes them, and in utf-8) and generates
# markov models for each nick in them
# put your concatenated logs in one file and set the markovLogFile
# variable in the config json

# for concatenations you can do something like
#   find ~/.znc -iname "*#*.log" -exec cat {} \; > mark.txt

nick_models = {}  # mapping nick -> markov model


# prepare the markov chains for users
def prepare():
    msg_regex = re.compile(r'^\[..:..:..\] <(.*?)> (.*)$')

    text = ''
    try:
        dict_file_name = yui.config_val('markovLogFile')
        with open(dict_file_name, errors='replace') as f:
            text = f.read()
    except Exception as ex:
        return

    # go through all lines and extract nick + messages
    nick_lines = {}
    for line in text.splitlines():
        match = msg_regex.match(line)
        if not match:
            continue
        u = match.group(1).lower()
        m = match.group(2)
        # omit any sentences shorter than 3 words
        if len(m.split()) < 3:
            continue
        if u not in nick_lines:
            nick_lines[u] = list()
        nick_lines[u].append(m)

    # generate markov models for each nick
    for u, m in nick_lines.items():
        try:
            mark = markovify.NewlineText('\n'.join(m), state_size=3)
            nick_models[u] = mark
        except Exception as ex:
            pass


# generate a sentence for a given nick
def generate_sentence(nick):
    for i in range(1, 100):
        ret = nick_models[nick.lower()].make_sentence()
        if ret:
            return ret
    return None


@yui.threaded
@yui.command('markov', 'mark')
def markov(argv, user):
    """Generate a random sentence for a given nick. Usage: markov [nick]"""
    name = user.nick
    if len(argv) > 1:
        name = argv[1]
    if name.lower() not in nick_models.keys():
{'fraction_non_alphanumeric': 0.11455596426694692, 'fraction_numerical': 0.006305832895428271, 'mean_word_length': 2.2875647668393784, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '58373', 'n_tokens_mistral': 1200, 'n_tokens_neox': 1162, 'n_words': 399}