d-matrix commited on
Commit
b09fb1d
1 Parent(s): 504216e

Update dmx_perplexity.py

Browse files

Accepting kwargs for _compute

Files changed (1) hide show
  1. dmx_perplexity.py +2 -1
dmx_perplexity.py CHANGED
@@ -63,6 +63,7 @@ class DmxPerplexity(evaluate.Metric):
63
  model: Union[str, AutoModelForCausalLM],
64
  device=None,
65
  max_length=None,
 
66
  ):
67
  if device is not None:
68
  assert device in [
@@ -80,7 +81,7 @@ class DmxPerplexity(evaluate.Metric):
80
  model = AutoModelForCausalLM.from_pretrained(model)
81
 
82
  else:
83
- tokenizer = AutoTokenizer.from_pretrained(model.config._name_or_path)
84
 
85
  if max_length:
86
  max_seq_len = max_length
 
63
  model: Union[str, AutoModelForCausalLM],
64
  device=None,
65
  max_length=None,
66
+ **kwargs,
67
  ):
68
  if device is not None:
69
  assert device in [
 
81
  model = AutoModelForCausalLM.from_pretrained(model)
82
 
83
  else:
84
+ tokenizer = AutoTokenizer.from_pretrained(model.config._name_or_path,**kwargs)
85
 
86
  if max_length:
87
  max_seq_len = max_length