<jupyter_start><jupyter_text>__author__ = "<NAME>"__time__   = "2021/11/22"__version__= "1.0"<jupyter_code>!pip install pygame
import pygame
from pygame.surfarray import array3d
import numpy as np
# import os
# chinese_dir = 'Chinese'
# if not os.path.exists(chinese_dir):
#     os.mkdir(chinese_dir)
pygame.init()
font = pygame.font.Font('simfang.ttf',24) #仿宋
font.size('双')<jupyter_output>pygame 2.1.0 (SDL 2.0.16, Python 3.8.5)
Hello from the pygame community. https://www.pygame.org/contribute.html<jupyter_text>Example of a Char to Glyph<jupyter_code>rtext = font.render('双',True,(0, 0, 0), (255, 255, 255))
# pygame.image.save(rtext, os.path.join(chinese_dir,'双'+".png"))
rtext = array3d(rtext) # shape: 24,24,3

tmp = np.zeros((576,1),dtype='float32')
for i, rgb in enumerate(rtext.reshape(576,3)):
    # Convert rgb [0,0,0] to 1.      
    tmp[i] = 1-rgb[0]/255.
tmp.shape
import json
import numpy as np
import pygame
from pygame.surfarray import array3d
def generate_Glyph_initial_weight(chardict_filepath,font_filepath,font_size=24):
    """
    filepath: path to char_map
    return a Glyph Embedding layer weights
        line 0 is reserved for special_tokens like [CLS] [PAD] [SEP]
        line 1 is reserved for [UNK]
    """

    with open(chardict_filepath,'r',encoding='utf-8') as f:
        
        pygame.init()
        font = pygame.font.Font(font_filepath,font_size) #仿宋
        
        char2idx = json.load(f)
        matrix = np.zeros((len(char2idx)+2,font_size**2),dtype='float32')
        matrix[1] = np.random.uniform(-0.1,0.1,font_size**2) # unknown words
        
        for i, char in enumerate(char2idx.keys(),2):
            # shape: font_size,font_size,3
            try:

                char_array = array3d(font.render(char,True,(0,0,0),(255,255,255)))
                # Convert rgb [0,0,0] to 1.
                char_array = 1-char_array[:font_size,:font_size,0].reshape(-1)/255
                matrix[i][:char_array.shape[0]] = char_array
            except:
                print(char)
    return matrix
matrix = generate_Glyph_initial_weight('data/char_map.json','utils/simfang.ttf')
# font = pygame.font.Font('simfang.ttf',25) #仿宋
# a = array3d(font.render('[UNK]',True,(0,0,0),(255,255,255)))
font_weights = np.load('data/glyph_weight.npy').astype(np.float32)
matrix.shape
font_weights.shape
np.save('glyph_weight',matrix)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1506677796327212, 'fraction_numerical': 0.05091819699499165, 'mean_word_length': 3.5744274809160306, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '581264', 'license': 'MIT', 'n_tokens_mistral': 972, 'n_tokens_neox': 904, 'n_words': 163, 'path': 'notebook/Glyph.ipynb', 'repo_name': 'chiang9/NLP-Chinese_couplet_generation', 'size': '2400'}