issue with opus100/en-fr dataset 
{'login': 'dorost1234', 'id': 79165106, 'node_id': 'MDQ6VXNlcjc5MTY1MTA2', 'avatar_url': 'https://avatars.githubusercontent.com/u/79165106?v=4', 'gravatar_id': '', 'url': 'https://api.github.com/users/dorost1234', 'html_url': 'https://github.com/dorost1234', 'followers_url': 'https://api.github.com/users/dorost1234/followers', 'following_url': 'https://api.github.com/users/dorost1234/following{/other_user}', 'gists_url': 'https://api.github.com/users/dorost1234/gists{/gist_id}', 'starred_url': 'https://api.github.com/users/dorost1234/starred{/owner}{/repo}', 'subscriptions_url': 'https://api.github.com/users/dorost1234/subscriptions', 'organizations_url': 'https://api.github.com/users/dorost1234/orgs', 'repos_url': 'https://api.github.com/users/dorost1234/repos', 'events_url': 'https://api.github.com/users/dorost1234/events{/privacy}', 'received_events_url': 'https://api.github.com/users/dorost1234/received_events', 'type': 'User', 'site_admin': False}
[]
[]
['@lhoestq  I also deleted the cache and redownload the file and still the same issue, I appreciate any help on this. thanks '
 'Here please find the minimal code to reproduce the issue @lhoestq  note this only happens with MT5TokenizerFast\r\n\r\n```\r\nfrom datasets import load_dataset\r\nfrom transformers import MT5TokenizerFast\r\n\r\ndef get_tokenized_dataset(dataset_name, dataset_config_name, tokenizer):\r\n    datasets = load_dataset(dataset_name, dataset_config_name, script_version="master")\r\n    column_names = datasets["train"].column_names\r\n    text_column_name = "translation"\r\n    def process_dataset(datasets):\r\n        def process_function(examples):\r\n            lang = "fr"\r\n            return {"src_texts": [example[lang] for example in examples[text_column_name]]}\r\n        datasets = datasets.map(\r\n            process_function,\r\n            batched=True,\r\n            num_proc=None,\r\n            remove_columns=column_names,\r\n            load_from_cache_file=True,\r\n        )\r\n        return datasets\r\n    datasets = process_dataset(datasets)\r\n    text_column_name = "src_texts"\r\n    column_names = ["src_texts"]\r\n    def tokenize_function(examples):\r\n            return tokenizer(examples[text_column_name], return_special_tokens_mask=True)\r\n    tokenized_datasets = datasets.map(\r\n            tokenize_function,\r\n            batched=True,\r\n            num_proc=None,\r\n            remove_columns=column_names,\r\n            load_from_cache_file=True\r\n    )\r\n\r\nif __name__ == "__main__":\r\n     tokenizer_kwargs = {\r\n        "cache_dir": None,\r\n        "use_fast": True,\r\n        "revision": "main",\r\n        "use_auth_token": None\r\n     }\r\n     tokenizer = MT5TokenizerFast.from_pretrained("google/mt5-small", **tokenizer_kwargs)\r\n     get_tokenized_dataset(dataset_name="opus100", dataset_config_name="en-fr", tokenizer=tokenizer)\r\n~     \r\n```'
 'as per https://github.com/huggingface/tokenizers/issues/626 this looks like to be the tokenizer bug, I therefore, reported it there https://github.com/huggingface/tokenizers/issues/626 and  I am closing this one.']
Hi
I am running run_mlm.py code of huggingface repo with opus100/fr-en pair, I am getting this error, note that this error occurs for only this pairs and not the other pairs. Any idea why this is occurring? and how I can solve this? 

Thanks a lot  @lhoestq for your help in advance.

`
thread '<unnamed>' panicked at 'index out of bounds: the len is 617 but the index is 617', /__w/tokenizers/tokenizers/tokenizers/src/tokenizer/normalizer.rs:382:21
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
 63%|██████████████████████████████████████████████████████████▊                                   | 626/1000 [00:27<00:16, 22.69ba/s]

Traceback (most recent call last):
  File "run_mlm.py", line 550, in <module>
    main()
  File "run_mlm.py", line 412, in main
    in zip(data_args.dataset_name, data_args.dataset_config_name)]
  File "run_mlm.py", line 411, in <listcomp>
    logger) for dataset_name, dataset_config_name\
  File "/user/dara/dev/codes/seq2seq/data/tokenize_datasets.py", line 96, in get_tokenized_dataset
    load_from_cache_file=not data_args.overwrite_cache,
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/datasets/dataset_dict.py", line 448, in map
    for k, dataset in self.items()
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/datasets/dataset_dict.py", line 448, in <dictcomp>
    for k, dataset in self.items()
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/datasets/arrow_dataset.py", line 1309, in map
    update_data=update_data,
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/datasets/arrow_dataset.py", line 204, in wrapper
    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/datasets/fingerprint.py", line 337, in wrapper
    out = func(self, *args, **kwargs)
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/datasets/arrow_dataset.py", line 1574, in _map_single
    batch, indices, check_same_num_examples=len(self.list_indexes()) > 0, offset=offset
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/datasets/arrow_dataset.py", line 1490, in apply_function_on_filtered_inputs
    function(*fn_args, effective_indices, **fn_kwargs) if with_indices else function(*fn_args, **fn_kwargs)
  File "/user/dara/dev/codes/seq2seq/data/tokenize_datasets.py", line 89, in tokenize_function
    return tokenizer(examples[text_column_name], return_special_tokens_mask=True)
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/transformers/tokenization_utils_base.py", line 2347, in __call__
    **kwargs,
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/transformers/tokenization_utils_base.py", line 2532, in batch_encode_plus
    **kwargs,
  File "/user/dara/libs/anaconda3/envs/fast/lib/python3.7/site-packages/transformers/tokenization_utils_fast.py", line 384, in _batch_encode_plus
    is_pretokenized=is_split_into_words,
pyo3_runtime.PanicException: index out of bounds: the len is 617 but the index is 617

`
{'url': 'https://api.github.com/repos/huggingface/datasets/issues/2056/reactions', 'total_count': 0, '+1': 0, '-1': 0, 'laugh': 0, 'hooray': 0, 'confused': 0, 'heart': 0, 'rocket': 0, 'eyes': 0}