Pedro Cuenca commited on
Commit
a77c0d4
1 Parent(s): 55a631d

Load tokenizer associated to the model checkpoint, if possible.

Browse files
Files changed (1) hide show
  1. tools/train/train.py +2 -1
tools/train/train.py CHANGED
@@ -484,8 +484,9 @@ def main():
484
  model_args.tokenizer_name, use_fast=True
485
  )
486
  else:
 
487
  tokenizer = AutoTokenizer.from_pretrained(
488
- model_args.model_name_or_path,
489
  use_fast=True,
490
  )
491
 
 
484
  model_args.tokenizer_name, use_fast=True
485
  )
486
  else:
487
+ # Use non-standard configuration property set by `DalleBart.from_pretrained`
488
  tokenizer = AutoTokenizer.from_pretrained(
489
+ model.config.resolved_name_or_path,
490
  use_fast=True,
491
  )
492