<commit_before># -*- coding: utf-8 -*-
"""
#Use like this:

#Initialize:
import wordai
wordai.dbfile = "<your db file>"
wordai.load()

#Main AI function
textin = input()
print(wordai.process(textin))

#Get Dictionary Output
print(wordai.getdictstring())

#Replace a word
wordai.replace("old","new")

#It will auto-save to the database.
"""

from random import choice
import copy
import random
import string
import pickle
import pprint

dbfile = ''

data_dict = {}


def load():
    global data_dict
    try:
        dict_file = open(dbfile, 'rb')
        data_dict = pickle.load(dict_file)
        dict_file.close()
    except:
        pass


def save():
    global data_dict
    output = open(dbfile, 'wb')
    pickle.dump(data_dict, output)
    output.close()


choices = list()


def addchoice(a):
    choices.append(a)


def ms(r):
    global choices
    global data_dict
    exclude = set(string.punctuation)
    r = ''.join(ch for ch in r if ch not in exclude)
    inp = r.lower().split()
    if len(inp):
        if not ';start' in data_dict:
            data_dict[';start'] = list()
        if not ';end' in data_dict:
            data_dict[';end'] = list()
        if not inp[0] in data_dict[';start'] or True:
            data_dict[';start'].append(inp[0])
        if not inp[-1] in data_dict[';end'] or True:
            data_dict[';end'].append(inp[-1])
    for i in range(len(inp)):
        if not inp[i] in data_dict:
            data_dict[inp[i]] = list()
        try:
            if not inp[i + 1] in data_dict[inp[i]] or True:
                data_dict[inp[i]].append(inp[i + 1])
        except IndexError:
            pass
    ret = ''
    try:
        choices = list()
        for ch in range(4):
            try:
                addchoice(choice(inp))
            except:
                pass
        try:
            addchoice(inp[0])
        except:
            pass
        for ch in range(random.randrange(8, 16)):
            try:
                addchoice(choice(data_dict[';start']))
            except:
                pass
        try:
            addchoice(choice(data_dict[inp[0]]))
        except:
            pass
        first = choice(choices)
        ret += first + ' '
        nextword = first
        for numwords in range(100):
            if nextword in data_dict:
                if nextword in data_dict[';end'] \
                    and (int(random.randrange(0, 100)) < 5 + numwords
                         + data_dict[';end'].count(nextword)
                         / len(data_dict[';end']) * 1000
                         or len(data_dict[nextword]) == 0):
                    break
                cnext = choice(data_dict[nextword])
                ret += cnext + ' '
                nextword = cnext
            else:
                break
    except IndexError:
        pass
    except KeyError:
        pass
    try:
        return str(str(ret[0]).upper() + ret[1:]).strip() + '.'
    except IndexError:
        return '?'


def process(mp):
    global data_dict
    load()
    out = ms(mp)
    save()
    return out


def replace(w, n):
    global data_dict
    if n != w:
        data_dict[n] = data_dict[w]
        del data_dict[w]
    for k in data_dict:
        for (index, item) in enumerate(data_dict[k]):
            if item == w:
                data_dict[k][index] = n
    output = open(dbfile, 'wb')
    pickle.dump(data_dict, output)
    output.close()


def getdictstring():
    global data_dict
    load()
    data_dict_tmp = copy.deepcopy(data_dict)
    if ';record' in data_dict_tmp:
        del data_dict_tmp[';record']
    return pprint.pformat(data_dict_tmp)


def getwords():
    global data_dict<commit_msg>human-readable wordai<commit_after># -*- coding: utf-8 -*-
"""
#Use like this:

#Initialize:
import wordai
wordai.dbfile = "<your db file>"
wordai.load()

#Main AI function
textin = input()
print(wordai.process(textin))

#Get Dictionary Output
print(wordai.getdictstring())

#Replace a word
wordai.replace("old","new")

#It will auto-save to the database.
"""

from random import choice
import copy
import random
import string
import pickle
import pprint
import ast

dbfile = ''

data_dict = {}


def load():
    global data_dict
    try:
        dict_file = open(dbfile, 'rb')
        try:
            data_dict = pickle.load(dict_file)
        except:
            data_dict = ast.literal_eval(dict_file.read().decode())
        dict_file.close()
    except:
        data_dict = {}
        pass


def save():
    output = open(dbfile, 'w')
    output.write(pprint.pformat(data_dict))
    output.close()


choices = list()


def addchoice(a):
    choices.append(a)


def ms(r):
    global choices
    global data_dict
    exclude = set(string.punctuation)
    r = ''.join(ch for ch in r if ch not in exclude)
    inp = r.lower().split()
    if len(inp):
        if not ';start' in data_dict:
            data_dict[';start'] = list()
        if not ';end' in data_dict:
            data_dict[';end'] = list()
        if not inp[0] in data_dict[';start'] or True:
            data_dict[';start'].append(inp[0])
        if not inp[-1] in data_dict[';end'] or True:
            data_dict[';end'].append(inp[-1])
    for i in range(len(inp)):
        if not inp[i] in data_dict:
            data_dict[inp[i]] = list()
        try:
            if not inp[i + 1] in data_dict[inp[i]] or True:
                data_dict[inp[i]].append(inp[i + 1])
        except IndexError:
            pass
    ret = ''
    try:
        choices = list()
        for ch in range(4):
            try:
                addchoice(choice(inp))
            except:
                pass
        try:
            addchoice(inp[0])
        except:
            pass
        for ch in range(random.randrange(8, 16)):
            try:
                addchoice(choice(data_dict[';start']))
            except:
                pass
        try:
            addchoice(choice(data_dict[inp[0]]))
        except:
            pass
        first = choice(choices)
        ret += first + ' '
        nextword = first
        for numwords in range(100):
            if nextword in data_dict:
                if nextword in data_dict[';end'] \
                    and (int(random.randrange(0, 100)) < 5 + numwords
                         + data_dict[';end'].count(nextword)
                         / len(data_dict[';end']) * 1000
                         or len(data_dict[nextword]) == 0):
                    break
                cnext = choice(data_dict[nextword])
                ret += cnext + ' '
                nextword = cnext
            else:
                break
    except IndexError:
        pass
    except KeyError:
        pass
    try:
        return str(str(ret[0]).upper() + ret[1:]).strip() + '.'
    except IndexError:
        return '?'


def process(mp):
    load()
    out = ms(mp)
    save()
    return out


def replace(w, n):
    if n != w:
        data_dict[n] = data_dict[w]
        del data_dict[w]
    for k in data_dict:
        for (index, item) in enumerate(data_dict[k]):
            if item == w:
                data_dict[k][index] = n
    save()


def getdictstring():
    load()
    data_dict_tmp = copy.deepcopy(data_dict)
    if ';record' in data_dict_tmp:
        del data_dict_tmp[';record']
    return pprint.pformat(data_dict_tmp)


def getwords():
{'fraction_non_alphanumeric': 0.12317555585868231, 'fraction_numerical': 0.0076387941617787475, 'mean_word_length': 1.7793783169067476, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7556347', 'n_tokens_mistral': 2399, 'n_tokens_neox': 2322, 'n_words': 585}