<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import pickle, os, time, asyncio, concurrent, functools, bisect, random, shutil
from glob import glob
from itertools import accumulate
from collections import defaultdict
import unicodedata2 as unicodedata
import config
import cachetools
# import markovify
import aiopg

model_cache = cachetools.LFUCache(10)

BEGIN = "\u0002" # Start of Text
END = "\u0003" # End of Text

async def init():
    global pool, executor
    pool = await aiopg.create_pool("dbname={} user={}".format(config.DB_NAME, config.DB_USER), minsize=config.DB_POOL_MIN, maxsize=config.DB_POOL_MAX)
    executor = concurrent.futures.ProcessPoolExecutor(max_workers=config.MAX_WORKERS)

async def run_in_process(*args, **kwargs):
    loop = asyncio.get_event_loop()
    return await loop.run_in_executor(executor, functools.partial(*args, **kwargs))

def _pickle_load(path):
    return pickle.load(open(path, 'rb'))

async def pickle_load(*args):
    return await run_in_process(_pickle_load, *args)

def _pickle_dump(obj, path):
    pickle.dump(obj, open(path, 'wb'), pickle.HIGHEST_PROTOCOL)

async def pickle_dump(*args):
    return await run_in_process(_pickle_dump, *args)

class PLText:
    
    def __init__(self, uid, state_size, expr_score_list, chain=None):
        self.uid = uid
        self.state_size = state_size
        self.expr_set = {ex[0] for ex in expr_score_list}
        self.chain = chain or PLChain(expr_score_list, state_size)

    def test_expr_output(self, expr):
        return expr not in self.expr_set

    def make_sentence(self, init_state=None, tries=10, test_output=True, max_chars=None, probability=False):
        for _ in range(tries):
            if init_state:
                if init_state[0] == BEGIN:
                    prefix = init_state[1:]
                else:
                    prefix = init_state
            else:
                prefix = ''
            if probability:
                expr, prob = self.chain.walk(init_state, probability)
                expr = prefix + expr
            else:
                expr = prefix + self.chain.walk(init_state, probability)
            if max_chars and len(expr) > max_chars:
                continue
            if test_output:
                if self.test_expr_output(expr):
                    if probability:
                        return expr, prob
                    else:
                        return expr
            else:
                if probability:
                    return expr, prob
                else:<commit_msg>fixed preexisting expr testing<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import pickle, os, time, asyncio, concurrent, functools, bisect, random, shutil
from glob import glob
from itertools import accumulate
from collections import defaultdict
import unicodedata2 as unicodedata
import config
import cachetools
# import markovify
import aiopg

model_cache = cachetools.LFUCache(10)

BEGIN = "\u0002" # Start of Text
END = "\u0003" # End of Text

async def init():
    global pool, executor
    pool = await aiopg.create_pool("dbname={} user={}".format(config.DB_NAME, config.DB_USER), minsize=config.DB_POOL_MIN, maxsize=config.DB_POOL_MAX)
    executor = concurrent.futures.ProcessPoolExecutor(max_workers=config.MAX_WORKERS)

async def run_in_process(*args, **kwargs):
    loop = asyncio.get_event_loop()
    return await loop.run_in_executor(executor, functools.partial(*args, **kwargs))

def _pickle_load(path):
    return pickle.load(open(path, 'rb'))

async def pickle_load(*args):
    return await run_in_process(_pickle_load, *args)

def _pickle_dump(obj, path):
    pickle.dump(obj, open(path, 'wb'), pickle.HIGHEST_PROTOCOL)

async def pickle_dump(*args):
    return await run_in_process(_pickle_dump, *args)

class PLText:
    
    def __init__(self, uid, state_size, expr_score_list, chain=None):
        self.uid = uid
        self.state_size = state_size
        self.expr_set = {ex[0] for ex in expr_score_list}
        self.chain = chain or PLChain(expr_score_list, state_size)

    def test_expr_output(self, expr):
        return unicodedata.normalize("NFC", expr) not in self.expr_set

    def make_sentence(self, init_state=None, tries=10, test_output=True, max_chars=None, probability=False):
        for _ in range(tries):
            if init_state:
                if init_state[0] == BEGIN:
                    prefix = init_state[1:]
                else:
                    prefix = init_state
            else:
                prefix = ''
            if probability:
                expr, prob = self.chain.walk(init_state, probability)
                expr = prefix + expr
            else:
                expr = prefix + self.chain.walk(init_state, probability)
            if max_chars and len(expr) > max_chars:
                continue
            if test_output:
                if self.test_expr_output(expr):
                    if probability:
                        return expr, prob
                    else:
                        return expr
            else:
                if probability:
                    return expr, prob
                else:
{'fraction_non_alphanumeric': 0.09148812970469021, 'fraction_numerical': 0.006948465547191662, 'mean_word_length': 2.3869281045751634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '167058', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1530, 'n_words': 391}