runing dataset.map, it raises TypeError: can't pickle Tokenizer objects
{'login': 'xixiaoyao', 'id': 24541791, 'node_id': 'MDQ6VXNlcjI0NTQxNzkx', 'avatar_url': 'https://avatars.githubusercontent.com/u/24541791?v=4', 'gravatar_id': '', 'url': 'https://api.github.com/users/xixiaoyao', 'html_url': 'https://github.com/xixiaoyao', 'followers_url': 'https://api.github.com/users/xixiaoyao/followers', 'following_url': 'https://api.github.com/users/xixiaoyao/following{/other_user}', 'gists_url': 'https://api.github.com/users/xixiaoyao/gists{/gist_id}', 'starred_url': 'https://api.github.com/users/xixiaoyao/starred{/owner}{/repo}', 'subscriptions_url': 'https://api.github.com/users/xixiaoyao/subscriptions', 'organizations_url': 'https://api.github.com/users/xixiaoyao/orgs', 'repos_url': 'https://api.github.com/users/xixiaoyao/repos', 'events_url': 'https://api.github.com/users/xixiaoyao/events{/privacy}', 'received_events_url': 'https://api.github.com/users/xixiaoyao/received_events', 'type': 'User', 'site_admin': False}
[]
[]
['Hi !\r\nIt works on my side with both the LongFormerTokenizer and the LongFormerTokenizerFast.\r\n\r\nWhich version of transformers/datasets are you using ?'
 'transformers and datasets are both the latest'
 'Then I guess you need to give us more informations on your setup (OS, python, GPU, etc) or a Google Colab reproducing the error for us to be able to debug this error.'
 'And your version of `dill` if possible :)'
 'I have the same issue with `transformers/BertJapaneseTokenizer`.\r\n\r\n\r\n\r\n```python\r\n# train_ds = Dataset(features: {\r\n#     \'title\': Value(dtype=\'string\', id=None), \r\n#     \'score\': Value(dtype=\'float64\', id=None)\r\n# }, num_rows: 99999)\r\n\r\nt = BertJapaneseTokenizer.from_pretrained(\'bert-base-japanese-whole-word-masking\')\r\nencoded = train_ds.map(lambda examples: {\'tokens\': t.encode(examples[\'title\'])}, batched=True)\r\n```\r\n\r\n<details><summary>Error Message</summary>\r\n\r\n```\r\n---------------------------------------------------------------------------\r\nTypeError                                 Traceback (most recent call last)\r\n<ipython-input-35-2b7d66b291c1> in <module>\r\n      2 \r\n      3 encoded = train_ds.map(lambda examples:\r\n----> 4   {\'tokens\': t.encode(examples[\'title\'])}, batched=True)\r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/arrow_dataset.py in map(self, function, with_indices, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint)\r\n   1242                 fn_kwargs=fn_kwargs,\r\n   1243                 new_fingerprint=new_fingerprint,\r\n-> 1244                 update_data=update_data,\r\n   1245             )\r\n   1246         else:\r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/arrow_dataset.py in wrapper(*args, **kwargs)\r\n    151             "output_all_columns": self._output_all_columns,\r\n    152         }\r\n--> 153         out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)\r\n    154         if new_format["columns"] is not None:\r\n    155             new_format["columns"] = list(set(new_format["columns"]) & set(out.column_names))\r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/fingerprint.py in wrapper(*args, **kwargs)\r\n    156                         kwargs_for_fingerprint["fingerprint_name"] = fingerprint_name\r\n    157                         kwargs[fingerprint_name] = update_fingerprint(\r\n--> 158                             self._fingerprint, transform, kwargs_for_fingerprint\r\n    159                         )\r\n    160 \r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/fingerprint.py in update_fingerprint(fingerprint, transform, transform_args)\r\n    103     for key in sorted(transform_args):\r\n    104         hasher.update(key)\r\n--> 105         hasher.update(transform_args[key])\r\n    106     return hasher.hexdigest()\r\n    107 \r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/fingerprint.py in update(self, value)\r\n     55     def update(self, value):\r\n     56         self.m.update(f"=={type(value)}==".encode("utf8"))\r\n---> 57         self.m.update(self.hash(value).encode("utf-8"))\r\n     58 \r\n     59     def hexdigest(self):\r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/fingerprint.py in hash(cls, value)\r\n     51             return cls.dispatch[type(value)](cls, value)\r\n     52         else:\r\n---> 53             return cls.hash_default(value)\r\n     54 \r\n     55     def update(self, value):\r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/fingerprint.py in hash_default(cls, value)\r\n     44     @classmethod\r\n     45     def hash_default(cls, value):\r\n---> 46         return cls.hash_bytes(dumps(value))\r\n     47 \r\n     48     @classmethod\r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/utils/py_utils.py in dumps(obj)\r\n    365     file = StringIO()\r\n    366     with _no_cache_fields(obj):\r\n--> 367         dump(obj, file)\r\n    368     return file.getvalue()\r\n    369 \r\n\r\n/usr/local/lib/python3.6/site-packages/datasets/utils/py_utils.py in dump(obj, file)\r\n    337 def dump(obj, file):\r\n    338     """pickle an object to a file"""\r\n--> 339     Pickler(file, recurse=True).dump(obj)\r\n    340     return\r\n    341 \r\n\r\n/usr/local/lib/python3.6/site-packages/dill/_dill.py in dump(self, obj)\r\n    444             raise PicklingError(msg)\r\n    445         else:\r\n--> 446             StockPickler.dump(self, obj)\r\n    447         stack.clear()  # clear record of \'recursion-sensitive\' pickled objects\r\n    448         return\r\n\r\n/usr/local/lib/python3.6/pickle.py in dump(self, obj)\r\n    407         if self.proto >= 4:\r\n    408             self.framer.start_framing()\r\n--> 409         self.save(obj)\r\n    410         self.write(STOP)\r\n    411         self.framer.end_framing()\r\n\r\n/usr/local/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)\r\n    474         f = self.dispatch.get(t)\r\n    475         if f is not None:\r\n--> 476             f(self, obj) # Call unbound method with explicit self\r\n    477             return\r\n    478 \r\n\r\n/usr/local/lib/python3.6/site-packages/dill/_dill.py in save_function(pickler, obj)\r\n   1436                                 globs, obj.__name__,\r\n   1437                                 obj.__defaults__, obj.__closure__,\r\n-> 1438                                 obj.__dict__, fkwdefaults), obj=obj)\r\n   1439         else:\r\n   1440             _super = (\'super\' in getattr(obj.func_code,\'co_names\',())) and (_byref is not None) and getattr(pickler, \'_recurse\', False)\r\n\r\n/usr/local/lib/python3.6/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)\r\n    608         else:\r\n    609             save(func)\r\n--> 610             save(args)\r\n    611             write(REDUCE)\r\n    612 \r\n\r\n/usr/local/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)\r\n    474         f = self.dispatch.get(t)\r\n    475         if f is not None:\r\n--> 476             f(self, obj) # Call unbound method with explicit self\r\n    477             return\r\n    478 \r\n\r\n/usr/local/lib/python3.6/pickle.py in save_tuple(self, obj)\r\n    749         write(MARK)\r\n    750         for element in obj:\r\n--> 751             save(element)\r\n    752 \r\n    753         if id(obj) in memo:\r\n\r\n/usr/local/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)\r\n    474         f = self.dispatch.get(t)\r\n    475         if f is not None:\r\n--> 476             f(self, obj) # Call unbound method with explicit self\r\n    477             return\r\n    478 \r\n\r\n/usr/local/lib/python3.6/site-packages/dill/_dill.py in save_module_dict(pickler, obj)\r\n    931             # we only care about session the first pass thru\r\n    932             pickler._session = False\r\n--> 933         StockPickler.save_dict(pickler, obj)\r\n    934         log.info("# D2")\r\n    935     return\r\n\r\n/usr/local/lib/python3.6/pickle.py in save_dict(self, obj)\r\n    819 \r\n    820         self.memoize(obj)\r\n--> 821         self._batch_setitems(obj.items())\r\n    822 \r\n    823     dispatch[dict] = save_dict\r\n\r\n/usr/local/lib/python3.6/pickle.py in _batch_setitems(self, items)\r\n    850                 k, v = tmp[0]\r\n    851                 save(k)\r\n--> 852                 save(v)\r\n    853                 write(SETITEM)\r\n    854             # else tmp is empty, and we\'re done\r\n\r\n/usr/local/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)\r\n    519 \r\n    520         # Save the reduce() output and finally memoize the object\r\n--> 521         self.save_reduce(obj=obj, *rv)\r\n    522 \r\n    523     def persistent_id(self, obj):\r\n\r\n/usr/local/lib/python3.6/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)\r\n    632 \r\n    633         if state is not None:\r\n--> 634             save(state)\r\n    635             write(BUILD)\r\n    636 \r\n\r\n/usr/local/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)\r\n    474         f = self.dispatch.get(t)\r\n    475         if f is not None:\r\n--> 476             f(self, obj) # Call unbound method with explicit self\r\n    477             return\r\n    478 \r\n\r\n/usr/local/lib/python3.6/site-packages/dill/_dill.py in save_module_dict(pickler, obj)\r\n    931             # we only care about session the first pass thru\r\n    932             pickler._session = False\r\n--> 933         StockPickler.save_dict(pickler, obj)\r\n    934         log.info("# D2")\r\n    935     return\r\n\r\n/usr/local/lib/python3.6/pickle.py in save_dict(self, obj)\r\n    819 \r\n    820         self.memoize(obj)\r\n--> 821         self._batch_setitems(obj.items())\r\n    822 \r\n    823     dispatch[dict] = save_dict\r\n\r\n/usr/local/lib/python3.6/pickle.py in _batch_setitems(self, items)\r\n    845                 for k, v in tmp:\r\n    846                     save(k)\r\n--> 847                     save(v)\r\n    848                 write(SETITEMS)\r\n    849             elif n:\r\n\r\n/usr/local/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)\r\n    519 \r\n    520         # Save the reduce() output and finally memoize the object\r\n--> 521         self.save_reduce(obj=obj, *rv)\r\n    522 \r\n    523     def persistent_id(self, obj):\r\n\r\n/usr/local/lib/python3.6/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)\r\n    632 \r\n    633         if state is not None:\r\n--> 634             save(state)\r\n    635             write(BUILD)\r\n    636 \r\n\r\n/usr/local/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)\r\n    474         f = self.dispatch.get(t)\r\n    475         if f is not None:\r\n--> 476             f(self, obj) # Call unbound method with explicit self\r\n    477             return\r\n    478 \r\n\r\n/usr/local/lib/python3.6/site-packages/dill/_dill.py in save_module_dict(pickler, obj)\r\n    931             # we only care about session the first pass thru\r\n    932             pickler._session = False\r\n--> 933         StockPickler.save_dict(pickler, obj)\r\n    934         log.info("# D2")\r\n    935     return\r\n\r\n/usr/local/lib/python3.6/pickle.py in save_dict(self, obj)\r\n    819 \r\n    820         self.memoize(obj)\r\n--> 821         self._batch_setitems(obj.items())\r\n    822 \r\n    823     dispatch[dict] = save_dict\r\n\r\n/usr/local/lib/python3.6/pickle.py in _batch_setitems(self, items)\r\n    845                 for k, v in tmp:\r\n    846                     save(k)\r\n--> 847                     save(v)\r\n    848                 write(SETITEMS)\r\n    849             elif n:\r\n\r\n/usr/local/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)\r\n    494             reduce = getattr(obj, "__reduce_ex__", None)\r\n    495             if reduce is not None:\r\n--> 496                 rv = reduce(self.proto)\r\n    497             else:\r\n    498                 reduce = getattr(obj, "__reduce__", None)\r\n\r\nTypeError: can\'t pickle Tagger objects\r\n```\r\n\r\n</details>\r\n\r\ntrainsformers: 2.10.0\r\ndatasets: 1.0.2\r\ndill: 0.3.2\r\npython: 3.6.8\r\n\r\nOS: ubuntu 16.04 (Docker Image) on [Deep Learning VM](https://console.cloud.google.com/marketplace/details/click-to-deploy-images/deeplearning) (GCP)\r\nGPU: Tesla P100 (CUDA 10)\r\n'
 '> I have the same issue with `transformers/BertJapaneseTokenizer`.\r\n\r\nIt looks like it this tokenizer is not supported unfortunately.\r\nThis is because `t.word_tokenizer.mecab` is a `fugashi.fugashi.GenericTagger` which is not compatible with pickle nor dill.\r\n\r\nWe need objects passes to `map` to be picklable for our caching system to work properly.\r\nHere it crashes because the caching system is not able to pickle the GenericTagger.\r\n\r\n\\> Maybe you can create an issue on [fugashi](https://github.com/polm/fugashi/issues) \'s repo and ask to make `fugashi.fugashi.GenericTagger` compatible with pickle ?\r\n\r\nWhat you can do in the meantime is use a picklable wrapper of the tokenizer:\r\n\r\n\r\n```python\r\nfrom transformers import BertJapaneseTokenizer, MecabTokenizer\r\n\r\nclass PicklableTokenizer(BertJapaneseTokenizer):\r\n\r\n    def __getstate__(self):\r\n        state = dict(self.__dict__)\r\n        state["do_lower_case"] = self.word_tokenizer.do_lower_case\r\n        state["never_split"] = self.word_tokenizer.never_split \r\n        del state["word_tokenizer"]\r\n        return state\r\n\r\n    def __setstate__(self, state):\r\n        do_lower_case = state.pop("do_lower_case")\r\n        never_split = state.pop("never_split")\r\n        self.__dict__ = state\r\n        self.word_tokenizer = MecabTokenizer(\r\n            do_lower_case=do_lower_case, never_split=never_split)\r\n        )\r\n\r\nt = PicklableTokenizer.from_pretrained("cl-tohoku/bert-base-japanese-whole-word-masking")\r\nencoded = train_ds.map(lambda examples: {\'tokens\': t.encode(examples[\'title\'])}, batched=True)  # it works\r\n```'
 "We can also update the `BertJapaneseTokenizer` in `transformers` as you just shown @lhoestq to make it compatible with pickle. It will be faster than asking on fugashi 's repo and good for the other users of `transformers` as well.\r\n\r\nI'm currently working on `transformers` I'll include it in the https://github.com/huggingface/transformers/pull/7141 PR and the next release of `transformers`."
 "Thank you for the rapid and polite response!\r\n\r\n@lhoestq Thanks for the suggestion! I've passed the pickle phase, but another `ArrowInvalid` problem occored. I created another issue #687 .\r\n\r\n@thomwolf Wow, really fast work. I'm looking forward to the next release 🤗"]
I load squad dataset. Then want to process data use following function with `Huggingface Transformers LongformerTokenizer`.

```
def convert_to_features(example):
    # Tokenize contexts and questions (as pairs of inputs)
    input_pairs = [example['question'], example['context']]
    encodings = tokenizer.encode_plus(input_pairs, pad_to_max_length=True, max_length=512)
    context_encodings = tokenizer.encode_plus(example['context'])
    

    # Compute start and end tokens for labels using Transformers's fast tokenizers alignement methodes.
    # this will give us the position of answer span in the context text
    start_idx, end_idx = get_correct_alignement(example['context'], example['answers'])
    start_positions_context = context_encodings.char_to_token(start_idx)
    end_positions_context = context_encodings.char_to_token(end_idx-1)

    # here we will compute the start and end position of the answer in the whole example
    # as the example is encoded like this <s> question</s></s> context</s>
    # and we know the postion of the answer in the context
    # we can just find out the index of the sep token and then add that to position + 1 (+1 because there are two sep tokens)
    # this will give us the position of the answer span in whole example 
    sep_idx = encodings['input_ids'].index(tokenizer.sep_token_id)
    start_positions = start_positions_context + sep_idx + 1
    end_positions = end_positions_context + sep_idx + 1

    if end_positions > 512:
      start_positions, end_positions = 0, 0

    encodings.update({'start_positions': start_positions,
                      'end_positions': end_positions,
                      'attention_mask': encodings['attention_mask']})
    return encodings
```

Then I run `dataset.map(convert_to_features)`, it raise
```
In [59]: a.map(convert_to_features)                                                                                                                        
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-59-c453b508761d> in <module>
----> 1 a.map(convert_to_features)

/opt/conda/lib/python3.7/site-packages/datasets/arrow_dataset.py in map(self, function, with_indices, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint)
   1242                 fn_kwargs=fn_kwargs,
   1243                 new_fingerprint=new_fingerprint,
-> 1244                 update_data=update_data,
   1245             )
   1246         else:

/opt/conda/lib/python3.7/site-packages/datasets/arrow_dataset.py in wrapper(*args, **kwargs)
    151             "output_all_columns": self._output_all_columns,
    152         }
--> 153         out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
    154         if new_format["columns"] is not None:
    155             new_format["columns"] = list(set(new_format["columns"]) & set(out.column_names))

/opt/conda/lib/python3.7/site-packages/datasets/fingerprint.py in wrapper(*args, **kwargs)
    156                         kwargs_for_fingerprint["fingerprint_name"] = fingerprint_name
    157                         kwargs[fingerprint_name] = update_fingerprint(
--> 158                             self._fingerprint, transform, kwargs_for_fingerprint
    159                         )
    160 

/opt/conda/lib/python3.7/site-packages/datasets/fingerprint.py in update_fingerprint(fingerprint, transform, transform_args)
    103     for key in sorted(transform_args):
    104         hasher.update(key)
--> 105         hasher.update(transform_args[key])
    106     return hasher.hexdigest()
    107 

/opt/conda/lib/python3.7/site-packages/datasets/fingerprint.py in update(self, value)
     55     def update(self, value):
     56         self.m.update(f"=={type(value)}==".encode("utf8"))
---> 57         self.m.update(self.hash(value).encode("utf-8"))
     58 
     59     def hexdigest(self):

/opt/conda/lib/python3.7/site-packages/datasets/fingerprint.py in hash(cls, value)
     51             return cls.dispatch[type(value)](cls, value)
     52         else:
---> 53             return cls.hash_default(value)
     54 
     55     def update(self, value):

/opt/conda/lib/python3.7/site-packages/datasets/fingerprint.py in hash_default(cls, value)
     44     @classmethod
     45     def hash_default(cls, value):
---> 46         return cls.hash_bytes(dumps(value))
     47 
     48     @classmethod

/opt/conda/lib/python3.7/site-packages/datasets/utils/py_utils.py in dumps(obj)
    365     file = StringIO()
    366     with _no_cache_fields(obj):
--> 367         dump(obj, file)
    368     return file.getvalue()
    369 

/opt/conda/lib/python3.7/site-packages/datasets/utils/py_utils.py in dump(obj, file)
    337 def dump(obj, file):
    338     """pickle an object to a file"""
--> 339     Pickler(file, recurse=True).dump(obj)
    340     return
    341 

/opt/conda/lib/python3.7/site-packages/dill/_dill.py in dump(self, obj)
    444             raise PicklingError(msg)
    445         else:
--> 446             StockPickler.dump(self, obj)
    447         stack.clear()  # clear record of 'recursion-sensitive' pickled objects
    448         return

/opt/conda/lib/python3.7/pickle.py in dump(self, obj)
    435         if self.proto >= 4:
    436             self.framer.start_framing()
--> 437         self.save(obj)
    438         self.write(STOP)
    439         self.framer.end_framing()

/opt/conda/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/conda/lib/python3.7/site-packages/dill/_dill.py in save_function(pickler, obj)
   1436                                 globs, obj.__name__,
   1437                                 obj.__defaults__, obj.__closure__,
-> 1438                                 obj.__dict__, fkwdefaults), obj=obj)
   1439         else:
   1440             _super = ('super' in getattr(obj.func_code,'co_names',())) and (_byref is not None) and getattr(pickler, '_recurse', False)

/opt/conda/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
    636         else:
    637             save(func)
--> 638             save(args)
    639             write(REDUCE)
    640 

/opt/conda/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/conda/lib/python3.7/pickle.py in save_tuple(self, obj)
    787         write(MARK)
    788         for element in obj:
--> 789             save(element)
    790 
    791         if id(obj) in memo:

/opt/conda/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/conda/lib/python3.7/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    931             # we only care about session the first pass thru
    932             pickler._session = False
--> 933         StockPickler.save_dict(pickler, obj)
    934         log.info("# D2")
    935     return

/opt/conda/lib/python3.7/pickle.py in save_dict(self, obj)
    857 
    858         self.memoize(obj)
--> 859         self._batch_setitems(obj.items())
    860 
    861     dispatch[dict] = save_dict

/opt/conda/lib/python3.7/pickle.py in _batch_setitems(self, items)
    883                 for k, v in tmp:
    884                     save(k)
--> 885                     save(v)
    886                 write(SETITEMS)
    887             elif n:

/opt/conda/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    547 
    548         # Save the reduce() output and finally memoize the object
--> 549         self.save_reduce(obj=obj, *rv)
    550 
    551     def persistent_id(self, obj):

/opt/conda/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
    660 
    661         if state is not None:
--> 662             save(state)
    663             write(BUILD)
    664 

/opt/conda/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/conda/lib/python3.7/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    931             # we only care about session the first pass thru
    932             pickler._session = False
--> 933         StockPickler.save_dict(pickler, obj)
    934         log.info("# D2")
    935     return

/opt/conda/lib/python3.7/pickle.py in save_dict(self, obj)
    857 
    858         self.memoize(obj)
--> 859         self._batch_setitems(obj.items())
    860 
    861     dispatch[dict] = save_dict

/opt/conda/lib/python3.7/pickle.py in _batch_setitems(self, items)
    883                 for k, v in tmp:
    884                     save(k)
--> 885                     save(v)
    886                 write(SETITEMS)
    887             elif n:

/opt/conda/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    547 
    548         # Save the reduce() output and finally memoize the object
--> 549         self.save_reduce(obj=obj, *rv)
    550 
    551     def persistent_id(self, obj):

/opt/conda/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
    660 
    661         if state is not None:
--> 662             save(state)
    663             write(BUILD)
    664 

/opt/conda/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/conda/lib/python3.7/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    931             # we only care about session the first pass thru
    932             pickler._session = False
--> 933         StockPickler.save_dict(pickler, obj)
    934         log.info("# D2")
    935     return

/opt/conda/lib/python3.7/pickle.py in save_dict(self, obj)
    857 
    858         self.memoize(obj)
--> 859         self._batch_setitems(obj.items())
    860 
    861     dispatch[dict] = save_dict

/opt/conda/lib/python3.7/pickle.py in _batch_setitems(self, items)
    883                 for k, v in tmp:
    884                     save(k)
--> 885                     save(v)
    886                 write(SETITEMS)
    887             elif n:

/opt/conda/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    522             reduce = getattr(obj, "__reduce_ex__", None)
    523             if reduce is not None:
--> 524                 rv = reduce(self.proto)
    525             else:
    526                 reduce = getattr(obj, "__reduce__", None)

TypeError: can't pickle Tokenizer objects
```


{'url': 'https://api.github.com/repos/huggingface/datasets/issues/665/reactions', 'total_count': 1, '+1': 1, '-1': 0, 'laugh': 0, 'hooray': 0, 'confused': 0, 'heart': 0, 'rocket': 0, 'eyes': 0}