<jupyter_start><jupyter_text>**Load vocaburay from tencent's word embedding file**<jupyter_code># with open('/home/shizai/qfeng/Tencent_AILab_ChineseEmbedding.txt', 'r', errors='ignore') as f:
#     tencent_words = [line.strip('\n').split(' ')[0] for line in f.readlines()]
    
# tencent_chars = list(set(c for w in tencent_words for c in w))
# len(tencent_words), len(tencent_chars)
# tencent_words[:10]
# tencent_chars[:10]<jupyter_output><empty_output><jupyter_text>**Load vocabury from jieba's dictionary file**<jupyter_code>jieba_words = []

with open('/datadisk2/qfeng/jieba/extra_dict/dict.txt.small', 'r') as f:
    for line in f.readlines():
        word_unit = line.strip('\n').split(' ')        
        jieba_words.append(word_unit)

jieba_chars = list(set(c for w in jieba_words for c in w[0]))
len(jieba_words), len(jieba_chars)
jieba_words[:10]
jieba_chars[:10]
jieba_word_freq_sum = sum([int(w[1]) for w in jieba_words])
jieba_pure_words = [w[0] for w in jieba_words]
jieba_word_freq_pcts = [int(w[1])/word_freq_sum for w in jieba_words]<jupyter_output><empty_output><jupyter_text>**Generate from jieba vocabulary**<jupyter_code>outp_path = '/datadisk2/qfeng/ocr_synthesis/jieba_texts'
import random
import bisect
import collections

def cdf(weights):
    total = sum(weights)
    result = []
    cumsum = 0
    for w in weights:
        cumsum += w
        result.append(cumsum / total)
    return result

def choice(population, weights):
    assert len(population) == len(weights)
    cdf_vals = cdf(weights)
    x = random.random()
    idx = bisect.bisect(cdf_vals, x)
    return population[idx]
choice(jieba_pure_words, jieba_word_freq_pcts)
# def gen_texts(words, freq_pcts, min_len, max_len, n_outp_words):
#     outp = []
#     while len(outp) < n_outp_words :
#         text = ''
#         while len(text) < min_len:
#             choiced = choice(words, freq_pcts).strip()
#             text = text + choiced
#         if len(text) <= max_len:
#             outp.append(text)
#     return outp

# gen_texts(jieba_pure_words, jieba_word_freq_pcts, min_len=5, max_len=5, n_outp_words=10)
def next_text(words, freq_pcts, min_len, max_len):
    while True:
        text = ''
        while len(text) < min_len:
            choiced = choice(words, freq_pcts).strip()
            text = text + choiced
        if len(text) <= max_len:
            yield text
def gen_texts(fid, gen_counts, gen_len, pure_words, word_freq_pcts):
    random.seed(random.randint(1,100))
    generator = next_text(pure_words, word_freq_pcts, min_len=gen_len, max_len=gen_len)
    with open(outp_path  + '/{}_lens/{}.txt'.format(gen_len, fid), 'w') as f:
        for _ in range(gen_counts):
            f.write(next(generator) + '\n')
# gen_texts(fid = 0, gen_counts=10, gen_len=5, pure_words=jieba_pure_words, word_freq_pcts=jieba_word_freq_pcts)
gen_counts = 10000
gen_batches = 200
for gen_len in [5, 10, 15]:
    with ProcessPoolExecutor(max_workers=40) as ex:
        func = partial(gen_texts, gen_counts=gen_counts, gen_len=gen_len,
                       pure_words=jieba_pure_words,
                       word_freq_pcts=jieba_word_freq_pcts)
        for i in range(gen_batches):
            ex.submit(func, i)
from pathlib import Path

word_counter = set()

for gen_len in [5, 10, 15]:
    for p in (Path(outp_path)/('{}_lens'.format(gen_len))).glob('*'):
        with open(str(p), 'r') as f:
            for line in f.readlines():
                word_counter.add(line.strip('\n'))
                
char_counter = {c for w in word_counter for c in w}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12827663134411602, 'fraction_numerical': 0.013385387618516454, 'mean_word_length': 3.5347661188369153, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '98173', 'license': 'MIT', 'n_tokens_mistral': 1426, 'n_tokens_neox': 1372, 'n_words': 272, 'path': 'notebook/text_gen_with_jieba_corpus.ipynb', 'repo_name': 'qf6101/TextRecognitionDataGenerator', 'size': '3586'}