<jupyter_start><jupyter_text>REFERENCE1. Source code for AddedTokens: https://github.com/huggingface/tokenizers/blob/master/tokenizers/src/tokenizer/added_vocabulary.rs<jupyter_code>import os
import json
import re
from collections import defaultdict, Counter
from multiprocessing import Pool

from tqdm import tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import emoji


%matplotlib inline
%load_ext autoreload
%autoreload 2
JSON_PATH = '../../data/instagram/json/insta-caption-train.json'
from transformers import XLNetTokenizer
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
test = 'This is an order! Understand@?'

out = tokenizer.tokenize(test)
out
test = 'This is so awesome!!@username!'
out = tokenizer.tokenize(test)
enc = tokenizer.encode(test, add_special_tokens = False)
dec = tokenizer.decode(enc)

print(out)
print(enc)
print(dec)
from tokenizers import AddedToken
new_tokens = AddedToken(
    '@username', single_word = True, 
    lstrip = False, rstrip = True
)
tokenizer.add_tokens(new_tokens)

out = tokenizer.tokenize(test)
enc = tokenizer.encode(test, add_special_tokens = False)
dec = tokenizer.decode(enc)

print(out)
print(enc)
print(dec)
tokenizer.tokenize('we arew')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10276679841897234, 'fraction_numerical': 0.0015810276679841897, 'mean_word_length': 6.193181818181818, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '597958', 'license': 'MIT', 'n_tokens_mistral': 434, 'n_tokens_neox': 427, 'n_words': 104, 'path': 'data/notebooks/07_pretrained_subword_tokenization.ipynb', 'repo_name': 'riven314/Stylised-Controllable-Image-Captioning', 'size': '1265'}