Upload 2 files

sentencepiece_bpe.py

@@ -28,6 +28,7 @@ class SentencepieceConfig:
 
 class SentencepieceBPE(object):
     def __init__(self, cfg):
+        cfg = SentencepieceConfig(**cfg)
         self.enable_sampling = cfg.sentencepiece_enable_sampling
         self.alpha = cfg.sentencepiece_alpha
         sentencepiece_model = cfg.sentencepiece_model

tokenization_fairseq_t5.py

@@ -47,10 +47,8 @@ class FairseqT5Tokenizer(PreTrainedTokenizer):
             )
         )
         self.fs_bpe = SentencepieceBPE(
-            DictConfig(
-                dict(
-                    sentencepiece_model=sp_path,
-                )
+            dict(
+                sentencepiece_model=sp_path,
             )
         )
         self.fs_dict = Dictionary.load(dict_path)