def setUp(self):
    super().setUp()
    vocab = ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'w</w>', 'r</w>', 't</w>', 'lo', 'low', 'er</w>', 'low</w>', 'lowest</w>', 'newer</w>', 'wider</w>', '<unk>']
    vocab_tokens = dict(zip(vocab, range(len(vocab))))
    merges = ['l o 123', 'lo w 1456', 'e r</w> 1789', '']
    self.langs = ['en', 'ru']
    config = {'langs': self.langs, 'src_vocab_size': 10, 'tgt_vocab_size': 20}
    self.src_vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['src_vocab_file'])
    self.tgt_vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['tgt_vocab_file'])
    config_file = os.path.join(self.tmpdirname, 'tokenizer_config.json')
    self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file'])
    with open(self.src_vocab_file, 'w') as fp:
        fp.write(json.dumps(vocab_tokens))
    with open(self.tgt_vocab_file, 'w') as fp:
        fp.write(json.dumps(vocab_tokens))
    with open(self.merges_file, 'w') as fp:
        fp.write('\n'.join(merges))
    with open(config_file, 'w') as fp:
        fp.write(json.dumps(config))
def setUp(self):
    super().setUp()
    vocab = ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'w</w>', 'r</w>', 't</w>', 'lo', 'low', 'er</w>', 'low</w>', 'lowest</w>', 'newer</w>', 'wider</w>', '<unk>']
    vocab_tokens = dict(zip(vocab, range(len(vocab))))
    merges = ['l o 123', 'lo w 1456', 'e r</w> 1789', ]
    self.langs = ['en', 'ru']
    config = {'langs': self.langs, 'src_vocab_size': 10, 'tgt_vocab_size': 20}
    self.src_vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['src_vocab_file'])
    self.tgt_vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['tgt_vocab_file'])
    config_file = os.path.join(self.tmpdirname, 'tokenizer_config.json')
    self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file'])
    with open(self.src_vocab_file, 'w') as fp:
        fp.write(json.dumps(vocab_tokens))
    with open(self.tgt_vocab_file, 'w') as fp:
        fp.write(json.dumps(vocab_tokens))
    with open(self.merges_file, 'w') as fp:
        fp.write('\n'.join(merges))
    with open(config_file, 'w') as fp:
        fp.write(json.dumps(config))