def build_tokenizer(dataset_list, max_seq_len, dat_fname, opt):
    if os.path.exists(os.path.join(opt.dataset_name, dat_fname)):
        print('Loading tokenizer on {}'.format(os.path.join(opt.dataset_name, dat_fname)))
        tokenizer = pickle.load(open(os.path.join(opt.dataset_name, dat_fname), 'rb'))
    else:
        text = ''
        for dataset_type in dataset_list:
            for file in dataset_list[dataset_type]:
                fin = open(file, 'r', encoding='utf-8', newline='\n', errors='ignore')
                lines = fin.readlines()
                fin.close()
                for i in range(0, len(lines), 3):
                    (text_left, _, text_right) = [s.lower().strip() for s in lines[i].partition('$T$')]
                    aspect = lines[(i + 1)].lower().strip()
                    text_raw = ((((text_left + ' ') + aspect) + ' ') + text_right)
                    text += (text_raw + ' ')
        tokenizer = Tokenizer(max_seq_len)
        tokenizer.fit_on_text(text)
        pickle.dump(tokenizer, open(os.path.join(opt.dataset_name, dat_fname), 'wb'))
    return tokenizer
def build_tokenizer(dataset_list, max_seq_len, dat_fname, opt):
    if os.path.exists(os.path.join(opt.dataset_name, dat_fname)):
        print('Loading tokenizer on {}'.format(os.path.join(opt.dataset_name, dat_fname)))
        tokenizer = pickle.load(open(os.path.join(opt.dataset_name, dat_fname), 'rb'))
    else:
        text = 
        for dataset_type in dataset_list:
            for file in dataset_list[dataset_type]:
                fin = open(file, 'r', encoding='utf-8', newline='\n', errors='ignore')
                lines = fin.readlines()
                fin.close()
                for i in range(0, len(lines), 3):
                    (text_left, _, text_right) = [s.lower().strip() for s in lines[i].partition('$T$')]
                    aspect = lines[(i + 1)].lower().strip()
                    text_raw = ((((text_left + ' ') + aspect) + ' ') + text_right)
                    text += (text_raw + ' ')
        tokenizer = Tokenizer(max_seq_len)
        tokenizer.fit_on_text(text)
        pickle.dump(tokenizer, open(os.path.join(opt.dataset_name, dat_fname), 'wb'))
    return tokenizer