model improved for transformers 4.42

config.json

@@ -3,22 +3,11 @@
     "MistralForTokenClassification"
   ],
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoModelForTokenClassification": "upos.MistralForTokenClassification"
-  },
   "bos_token_id": 1,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
-    },
-    "token-classification": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
-    },
-    "ner": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 2,
@@ -159,9 +148,9 @@
   "rope_theta": 10000.0,
   "sliding_window": 4096,
   "tie_word_embeddings": false,
-  "torch_dtype": "float32",
   "tokenizer_class": "LlamaTokenizerFast",
-  "transformers_version": "4.41.2",
+  "torch_dtype": "float32",
+  "transformers_version": "4.42.4",
   "use_cache": true,
   "use_transformers_inputs": true,
   "vocab_size": 48000

maker.sh

@@ -7,43 +7,7 @@ cat << 'EOF' > $TMP
 #! /usr/bin/env deepspeed
 src="KoichiYasuoka/RakutenAI-7B-upos"
 tgt="KoichiYasuoka/RakutenAI-7B-char-upos"
-from transformers import LlamaTokenizerFast,MistralModel,MistralPreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
-from transformers.modeling_outputs import TokenClassifierOutput
-
-class MistralForTokenClassification(MistralPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=MistralModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[1:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)
+from transformers import LlamaTokenizerFast,MistralForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):

pytorch_model-00001-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9390f326afba489dfb25938fe3e8f2f8d1d7227d1eb8b33d92a24069dced0b6f
+oid sha256:99e57c2c6739ec6e5cf7442d46b02f67ab16997b35987abc10e4dfc0476ffecc
 size 4913773120

pytorch_model-00002-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:138623c84c3e5d85b51ae8017f08b39958a38f3dfb3978e16dc8d50d87209476
+oid sha256:ea8b58c17ad22f5fbf33b7b887d6572a55b1e2f8e1fbc4d202cd4979f6504e83
 size 4999825256

pytorch_model-00003-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:708976207133843ca3ae6c243cca9d5f54e8e089c695dcd910234e78149fb472
+oid sha256:ff9c56ec56c78cab1dc779aed4488eec481789f1c02cb6f38b7ee0f19f110a20
 size 4999825316

pytorch_model-00004-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50c0d1273e3cc6e45ccdc865e3d78bac97353b6181bb640a6664f5e831f80d8c
+oid sha256:d735a8bc7052b2432f8d3b3b2facc38f3a479c26795921ba2b82756a3af77080
 size 4832018324

pytorch_model-00005-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b0c0e4fdb9b4893ca0ab5ff41d3cff0162f54a20823b13369720e68308fca53
+oid sha256:16cbd68930b892b79b995dc7749719655ce3d19ceea5d3fa4233200c8ce7c4c2
 size 4999825320

pytorch_model-00006-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:076923a6b328a83f9c8030cfa6fbd91d8f30d9f6a9fedda4ba069152e0f16a81
+oid sha256:8a86e2932a529df7ecbc9fa0870bf79967830a33e97f4b023b891519663b591f
 size 3960601264

pytorch_model.bin.index.json

@@ -3,8 +3,6 @@
     "total_size": 28705767664
   },
   "weight_map": {
-    "classifier.bias": "pytorch_model-00006-of-00006.bin",
-    "classifier.weight": "pytorch_model-00006-of-00006.bin",
     "model.embed_tokens.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
@@ -294,6 +292,8 @@
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
-    "model.norm.weight": "pytorch_model-00006-of-00006.bin"
+    "model.norm.weight": "pytorch_model-00006-of-00006.bin",
+    "score.bias": "pytorch_model-00006-of-00006.bin",
+    "score.weight": "pytorch_model-00006-of-00006.bin"
   }
 }

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",

upos.py

@@ -1,5 +1,4 @@
-from transformers import TokenClassificationPipeline,MistralModel,MistralPreTrainedModel
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import TokenClassificationPipeline
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -40,41 +39,3 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
 
-class RawTokenClassificationPipeline(TokenClassificationPipeline):
-  def check_model_type(self,supported_models):
-    pass
-
-class MistralForTokenClassification(MistralPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=MistralModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)