<jupyter_start><jupyter_text>Language Model<jupyter_code>import torch
from transformers import BertTokenizer, BertForMaskedLM
model_name = 'nghuyong/ernie-1.0'
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertForMaskedLM.from_pretrained(model_name)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)
pass
def language_model(input_text, debug=False):
    input_tokens = tokenizer.tokenize(input_text)
    input_ids = tokenizer.convert_tokens_to_ids(input_tokens)
    input_ids = torch.tensor(input_ids, device=device).unsqueeze(0)
    token_type_ids = torch.zeros_like(input_ids)
    
    model.eval()
    with torch.no_grad():
        outputs = model(input_ids=input_ids, token_type_ids=token_type_ids)

        preds = outputs[0].squeeze()
        pred_ids = torch.argmax(preds, dim=1)
        pred_text = ''.join(tokenizer.convert_ids_to_tokens(pred_ids.squeeze()[1:-1]))
        
    if debug:
        print('input_ids shape: {}, token_type_ids shape:{}'.format(input_ids.shape, token_type_ids.shape))
        print('outputs shape: {}'.format(outputs[0].shape))
        print('pred_ids shape: {}'.format(pred_ids.shape))
        
    return pred_text
language_model('[CLS] [MASK] [MASK] 是中国的首都。[SEP]', debug=True)
language_model('[CLS] [MASK] [MASK] [MASK]是美国的首都。[SEP]')
language_model('[CLS] [MASK] [MASK] [MASK] 是河北省的省会。[SEP]')
language_model("[CLS] [MASK] [MASK] 是中国最伟大的企业之一。[SEP]")
language_model("[CLS] [MASK] [MASK]是美国最伟大的企业之一。[SEP]")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14416775884665792, 'fraction_numerical': 0.005242463958060288, 'mean_word_length': 5.443037974683544, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '263916', 'license': 'MIT', 'n_tokens_mistral': 589, 'n_tokens_neox': 599, 'n_words': 83, 'path': 'docs/Language-Model.ipynb', 'repo_name': 'hotbaby/mmn', 'size': '1526'}