initial release

README.md

@@ -0,0 +1,86 @@
+---
+language:
+- "vi"
+tags:
+- "vietnamese"
+- "token-classification"
+- "pos"
+- "dependency-parsing"
+datasets:
+- "universal_dependencies"
+license: "cc-by-sa-4.0"
+pipeline_tag: "token-classification"
+widget:
+- text: "Hai cái đầu thì tốt hơn một"
+---
+
+# phobert-base-vietnamese-ud-goeswith
+
+## Model Description
+
+This is a PhoBERT model pre-trained on Vietnamese texts for POS-tagging and dependency-parsing (using `goeswith` for subwords), derived from [phobert-base](https://huggingface.co/vinai/phobert-base).
+
+## How to Use
+
+```py
+class UDgoeswithViNLP(object):
+  def __init__(self,bert):
+    from transformers import AutoTokenizer,AutoModelForTokenClassification
+    from ViNLP import word_tokenize
+    self.tokenizer=AutoTokenizer.from_pretrained(bert)
+    self.model=AutoModelForTokenClassification.from_pretrained(bert)
+    self.vinlp=word_tokenize
+  def __call__(self,text):
+    import numpy,torch,ufal.chu_liu_edmonds
+    t=self.vinlp(text)
+    w=self.tokenizer(t,add_special_tokens=False)["input_ids"]
+    z=[]
+    for i,j in enumerate(t):
+      if j.find("_")>0 and [k for k in w[i] if k==self.tokenizer.unk_token_id]!=[]:
+          w[i]=self.tokenizer(j.replace("_"," "))["input_ids"][1:-1]
+      if [k for k in w[i] if k==self.tokenizer.unk_token_id]!=[]:
+        w[i]=[self.tokenizer.unk_token_id]
+        z.append(j)
+    v=[self.tokenizer.cls_token_id]+sum(w,[])+[self.tokenizer.sep_token_id]
+    x=[v[0:i]+[self.tokenizer.mask_token_id]+v[i+1:]+[j] for i,j in enumerate(v[1:-1],1)]
+    with torch.no_grad():
+      e=self.model(input_ids=torch.tensor(x)).logits.numpy()[:,1:-2,:]
+    r=[1 if i==0 else -1 if j.endswith("|root") else 0 for i,j in sorted(self.model.config.id2label.items())]
+    e+=numpy.where(numpy.add.outer(numpy.identity(e.shape[0]),r)==0,0,numpy.nan)
+    g=self.model.config.label2id["X|_|goeswith"]
+    r=numpy.tri(e.shape[0])
+    for i in range(e.shape[0]):
+      for j in range(i+2,e.shape[1]):
+        r[i,j]=r[i,j-1] if numpy.nanargmax(e[i,j-1])==g else 1
+    e[:,:,g]+=numpy.where(r==0,0,numpy.nan)
+    m=numpy.full((e.shape[0]+1,e.shape[1]+1),numpy.nan)
+    m[1:,1:]=numpy.nanmax(e,axis=2).transpose()
+    p=numpy.zeros(m.shape)
+    p[1:,1:]=numpy.nanargmax(e,axis=2).transpose()
+    for i in range(1,m.shape[0]):
+      m[i,0],m[i,i],p[i,0]=m[i,i],numpy.nan,p[i,i]
+    h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
+    if [0 for i in h if i==0]!=[0]:
+      m[:,0]+=numpy.where(m[:,0]==numpy.nanmax(m[[i for i,j in enumerate(h) if j==0],0]),0,numpy.nan)
+      m[[i for i,j in enumerate(h) if j==0]]+=[0 if i==0 or j==0 else numpy.nan for i,j in enumerate(h)]
+      h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
+    u="# text = "+text+"\n"
+    q=[self.model.config.id2label[p[i,j]].split("|") for i,j in enumerate(h)]
+    t=[i.replace("_"," ") for i in t]
+    if len(t)!=len(v)-2:
+      t=[z.pop(0) if i==self.tokenizer.unk_token else i.replace("_"," ") for i in self.tokenizer.convert_ids_to_tokens(v[1:-1])]
+      for i,j in reversed(list(enumerate(q[2:],2))):
+        if j[-1]=="goeswith" and set([k[-1] for k in q[h[i]+1:i+1]])=={"goeswith"}:
+          h=[b if i>b else b-1 for a,b in enumerate(h) if i!=a]
+          t[i-2]=(t[i-2][0:-2] if t[i-2].endswith("@@") else t[i-2]+" ")+t.pop(i-1)
+          q.pop(i)
+      t=[i[0:-2].strip() if i.endswith("@@") else i.strip() for i in t]
+    for i,j in enumerate(t,1):
+      u+="\t".join([str(i),j,j,q[i][0],"_","|".join(q[i][1:-1]),str(h[i]),q[i][-1],"_","_"])+"\n"
+    return u+"\n"
+
+nlp=UDgoeswithViNLP("KoichiYasuoka/phobert-base-vietnamese-ud-goeswith")
+print(nlp("Hai cái đầu thì tốt hơn một."))
+```
+
+with [ufal.chu-liu-edmonds](https://pypi.org/project/ufal.chu-liu-edmonds/) and [ViNLP](https://pypi.org/project/ViNLP/).

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 64000
+}

config.json

@@ -0,0 +1,766 @@
+{
+  "architectures": [
+    "RobertaForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "-|_|dep",
+    "1": "ADJ|_|acl",
+    "2": "ADJ|_|acl:subj",
+    "3": "ADJ|_|acl:tmod",
+    "4": "ADJ|_|acl:tonp",
+    "5": "ADJ|_|advcl",
+    "6": "ADJ|_|advcl:objective",
+    "7": "ADJ|_|advmod",
+    "8": "ADJ|_|advmod:adj",
+    "9": "ADJ|_|advmod:neg",
+    "10": "ADJ|_|amod",
+    "11": "ADJ|_|appos",
+    "12": "ADJ|_|appos:nmod",
+    "13": "ADJ|_|ccomp",
+    "14": "ADJ|_|compound",
+    "15": "ADJ|_|compound:adj",
+    "16": "ADJ|_|compound:amod",
+    "17": "ADJ|_|compound:apr",
+    "18": "ADJ|_|compound:atov",
+    "19": "ADJ|_|compound:dir",
+    "20": "ADJ|_|compound:prt",
+    "21": "ADJ|_|compound:svc",
+    "22": "ADJ|_|compound:verbnoun",
+    "23": "ADJ|_|compound:vmod",
+    "24": "ADJ|_|conj",
+    "25": "ADJ|_|csubj",
+    "26": "ADJ|_|csubj:asubj",
+    "27": "ADJ|_|dep",
+    "28": "ADJ|_|discourse",
+    "29": "ADJ|_|dislocated",
+    "30": "ADJ|_|fixed",
+    "31": "ADJ|_|flat",
+    "32": "ADJ|_|flat:name",
+    "33": "ADJ|_|nmod",
+    "34": "ADJ|_|nsubj",
+    "35": "ADJ|_|obj",
+    "36": "ADJ|_|obl",
+    "37": "ADJ|_|obl:about",
+    "38": "ADJ|_|obl:adj",
+    "39": "ADJ|_|obl:comp",
+    "40": "ADJ|_|obl:tmod",
+    "41": "ADJ|_|obl:with",
+    "42": "ADJ|_|parataxis",
+    "43": "ADJ|_|root",
+    "44": "ADJ|_|xcomp",
+    "45": "ADJ|_|xcomp:adj",
+    "46": "ADP|_|acl:tmod",
+    "47": "ADP|_|advcl",
+    "48": "ADP|_|case",
+    "49": "ADP|_|cc",
+    "50": "ADP|_|ccomp",
+    "51": "ADP|_|compound",
+    "52": "ADP|_|compound:atov",
+    "53": "ADP|_|compound:dir",
+    "54": "ADP|_|compound:prt",
+    "55": "ADP|_|compound:svc",
+    "56": "ADP|_|conj",
+    "57": "ADP|_|csubj",
+    "58": "ADP|_|dep",
+    "59": "ADP|_|discourse",
+    "60": "ADP|_|fixed",
+    "61": "ADP|_|mark",
+    "62": "ADP|_|mark:pcomp",
+    "63": "ADP|_|nmod",
+    "64": "ADP|_|obl",
+    "65": "ADP|_|obl:tmod",
+    "66": "ADP|_|parataxis",
+    "67": "ADP|_|root",
+    "68": "ADP|_|xcomp",
+    "69": "ADV|_|acl:subj",
+    "70": "ADV|_|advcl",
+    "71": "ADV|_|advcl:objective",
+    "72": "ADV|_|advmod",
+    "73": "ADV|_|advmod:adj",
+    "74": "ADV|_|advmod:dir",
+    "75": "ADV|_|advmod:neg",
+    "76": "ADV|_|appos:nmod",
+    "77": "ADV|_|case",
+    "78": "ADV|_|compound",
+    "79": "ADV|_|compound:apr",
+    "80": "ADV|_|compound:atov",
+    "81": "ADV|_|compound:dir",
+    "82": "ADV|_|compound:prt",
+    "83": "ADV|_|compound:redup",
+    "84": "ADV|_|compound:svc",
+    "85": "ADV|_|conj",
+    "86": "ADV|_|discourse",
+    "87": "ADV|_|fixed",
+    "88": "ADV|_|flat:redup",
+    "89": "ADV|_|mark",
+    "90": "ADV|_|nmod",
+    "91": "ADV|_|obj",
+    "92": "ADV|_|obl",
+    "93": "ADV|_|obl:adv",
+    "94": "ADV|_|obl:tmod",
+    "95": "ADV|_|root",
+    "96": "ADV|_|xcomp",
+    "97": "AUX|_|aux",
+    "98": "AUX|_|aux:pass",
+    "99": "AUX|_|compound",
+    "100": "AUX|_|cop",
+    "101": "AUX|_|discourse",
+    "102": "AUX|_|parataxis",
+    "103": "AUX|_|root",
+    "104": "AUX|_|xcomp",
+    "105": "CCONJ|_|case",
+    "106": "CCONJ|_|cc",
+    "107": "CCONJ|_|flat",
+    "108": "CCONJ|_|mark",
+    "109": "DET|_|advmod:adj",
+    "110": "DET|_|clf:det",
+    "111": "DET|_|det",
+    "112": "DET|_|discourse",
+    "113": "DET|_|nmod:poss",
+    "114": "DET|_|nsubj",
+    "115": "DET|_|obj",
+    "116": "DET|_|obl:tmod",
+    "117": "INTJ|_|discourse",
+    "118": "INTJ|_|root",
+    "119": "NOUN|_|acl",
+    "120": "NOUN|_|acl:subj",
+    "121": "NOUN|_|acl:tmod",
+    "122": "NOUN|_|advcl",
+    "123": "NOUN|_|advcl:objective",
+    "124": "NOUN|_|amod",
+    "125": "NOUN|_|appos",
+    "126": "NOUN|_|appos:nmod",
+    "127": "NOUN|_|case",
+    "128": "NOUN|_|ccomp",
+    "129": "NOUN|_|clf",
+    "130": "NOUN|_|clf:det",
+    "131": "NOUN|_|compound",
+    "132": "NOUN|_|compound:amod",
+    "133": "NOUN|_|compound:dir",
+    "134": "NOUN|_|compound:verbnoun",
+    "135": "NOUN|_|compound:vmod",
+    "136": "NOUN|_|conj",
+    "137": "NOUN|_|csubj",
+    "138": "NOUN|_|csubj:pass",
+    "139": "NOUN|_|csubj:vsubj",
+    "140": "NOUN|_|dep",
+    "141": "NOUN|_|discourse",
+    "142": "NOUN|_|dislocated",
+    "143": "NOUN|_|fixed",
+    "144": "NOUN|_|flat",
+    "145": "NOUN|_|flat:name",
+    "146": "NOUN|_|flat:number",
+    "147": "NOUN|_|flat:time",
+    "148": "NOUN|_|iobj",
+    "149": "NOUN|_|list",
+    "150": "NOUN|_|nmod",
+    "151": "NOUN|_|nmod:poss",
+    "152": "NOUN|_|nsubj",
+    "153": "NOUN|_|nsubj:nn",
+    "154": "NOUN|_|nsubj:pass",
+    "155": "NOUN|_|nsubj:xsubj",
+    "156": "NOUN|_|nummod",
+    "157": "NOUN|_|obj",
+    "158": "NOUN|_|obl",
+    "159": "NOUN|_|obl:about",
+    "160": "NOUN|_|obl:adj",
+    "161": "NOUN|_|obl:adv",
+    "162": "NOUN|_|obl:agent",
+    "163": "NOUN|_|obl:comp",
+    "164": "NOUN|_|obl:iobj",
+    "165": "NOUN|_|obl:tmod",
+    "166": "NOUN|_|obl:with",
+    "167": "NOUN|_|parataxis",
+    "168": "NOUN|_|root",
+    "169": "NOUN|_|vocative",
+    "170": "NOUN|_|xcomp",
+    "171": "NUM|_|amod",
+    "172": "NUM|_|appos",
+    "173": "NUM|_|appos:nmod",
+    "174": "NUM|_|clf",
+    "175": "NUM|_|clf:det",
+    "176": "NUM|_|compound",
+    "177": "NUM|_|compound:verbnoun",
+    "178": "NUM|_|conj",
+    "179": "NUM|_|flat:date",
+    "180": "NUM|_|flat:name",
+    "181": "NUM|_|flat:number",
+    "182": "NUM|_|flat:time",
+    "183": "NUM|_|nmod",
+    "184": "NUM|_|nsubj",
+    "185": "NUM|_|nummod",
+    "186": "NUM|_|obj",
+    "187": "NUM|_|obl",
+    "188": "NUM|_|obl:comp",
+    "189": "NUM|_|obl:tmod",
+    "190": "NUM|_|parataxis",
+    "191": "NUM|_|root",
+    "192": "PART|_|advcl",
+    "193": "PART|_|advmod",
+    "194": "PART|_|amod",
+    "195": "PART|_|case",
+    "196": "PART|_|clf:det",
+    "197": "PART|_|compound",
+    "198": "PART|_|compound:prt",
+    "199": "PART|_|discourse",
+    "200": "PART|_|fixed",
+    "201": "PART|_|mark",
+    "202": "PART|_|obl",
+    "203": "PART|_|parataxis",
+    "204": "PRON|_|acl:tmod",
+    "205": "PRON|_|advcl",
+    "206": "PRON|_|appos:nmod",
+    "207": "PRON|_|ccomp",
+    "208": "PRON|_|compound",
+    "209": "PRON|_|compound:pron",
+    "210": "PRON|_|compound:prt",
+    "211": "PRON|_|conj",
+    "212": "PRON|_|det",
+    "213": "PRON|_|det:pmod",
+    "214": "PRON|_|discourse",
+    "215": "PRON|_|expl",
+    "216": "PRON|_|fixed",
+    "217": "PRON|_|iobj",
+    "218": "PRON|_|nmod",
+    "219": "PRON|_|nmod:poss",
+    "220": "PRON|_|nsubj",
+    "221": "PRON|_|nsubj:nn",
+    "222": "PRON|_|nsubj:pass",
+    "223": "PRON|_|nsubj:xsubj",
+    "224": "PRON|_|obj",
+    "225": "PRON|_|obl",
+    "226": "PRON|_|obl:about",
+    "227": "PRON|_|obl:adj",
+    "228": "PRON|_|obl:comp",
+    "229": "PRON|_|obl:iobj",
+    "230": "PRON|_|obl:tmod",
+    "231": "PRON|_|obl:with",
+    "232": "PRON|_|parataxis",
+    "233": "PRON|_|root",
+    "234": "PROPN|_|acl:subj",
+    "235": "PROPN|_|advcl",
+    "236": "PROPN|_|appos",
+    "237": "PROPN|_|appos:nmod",
+    "238": "PROPN|_|ccomp",
+    "239": "PROPN|_|compound",
+    "240": "PROPN|_|compound:verbnoun",
+    "241": "PROPN|_|conj",
+    "242": "PROPN|_|csubj:pass",
+    "243": "PROPN|_|dep",
+    "244": "PROPN|_|flat",
+    "245": "PROPN|_|flat:name",
+    "246": "PROPN|_|iobj",
+    "247": "PROPN|_|list",
+    "248": "PROPN|_|nmod",
+    "249": "PROPN|_|nmod:poss",
+    "250": "PROPN|_|nsubj",
+    "251": "PROPN|_|nsubj:nn",
+    "252": "PROPN|_|nsubj:pass",
+    "253": "PROPN|_|nsubj:xsubj",
+    "254": "PROPN|_|obj",
+    "255": "PROPN|_|obl",
+    "256": "PROPN|_|obl:agent",
+    "257": "PROPN|_|obl:comp",
+    "258": "PROPN|_|obl:iobj",
+    "259": "PROPN|_|obl:with",
+    "260": "PROPN|_|parataxis",
+    "261": "PROPN|_|root",
+    "262": "PROPN|_|vocative",
+    "263": "PUNCT|_|punct",
+    "264": "SCONJ|_|advcl",
+    "265": "SCONJ|_|case",
+    "266": "SCONJ|_|cc",
+    "267": "SCONJ|_|compound",
+    "268": "SCONJ|_|compound:svc",
+    "269": "SCONJ|_|discourse",
+    "270": "SCONJ|_|fixed",
+    "271": "SCONJ|_|mark",
+    "272": "SCONJ|_|obl",
+    "273": "SCONJ|_|parataxis",
+    "274": "SCONJ|_|root",
+    "275": "SCONJ|_|vocative",
+    "276": "SYM|_|advcl",
+    "277": "SYM|_|appos:nmod",
+    "278": "SYM|_|compound",
+    "279": "SYM|_|compound:z",
+    "280": "SYM|_|discourse",
+    "281": "SYM|_|flat",
+    "282": "SYM|_|flat:date",
+    "283": "SYM|_|flat:name",
+    "284": "SYM|_|flat:number",
+    "285": "SYM|_|flat:time",
+    "286": "SYM|_|nmod",
+    "287": "SYM|_|nsubj",
+    "288": "SYM|_|obj",
+    "289": "VERB|_|acl",
+    "290": "VERB|_|acl:relcl",
+    "291": "VERB|_|acl:subj",
+    "292": "VERB|_|acl:tmod",
+    "293": "VERB|_|acl:tonp",
+    "294": "VERB|_|advcl",
+    "295": "VERB|_|advcl:objective",
+    "296": "VERB|_|advmod",
+    "297": "VERB|_|amod",
+    "298": "VERB|_|appos",
+    "299": "VERB|_|appos:nmod",
+    "300": "VERB|_|case",
+    "301": "VERB|_|ccomp",
+    "302": "VERB|_|compound",
+    "303": "VERB|_|compound:amod",
+    "304": "VERB|_|compound:atov",
+    "305": "VERB|_|compound:dir",
+    "306": "VERB|_|compound:prt",
+    "307": "VERB|_|compound:redup",
+    "308": "VERB|_|compound:svc",
+    "309": "VERB|_|compound:verbnoun",
+    "310": "VERB|_|compound:vmod",
+    "311": "VERB|_|conj",
+    "312": "VERB|_|csubj",
+    "313": "VERB|_|csubj:pass",
+    "314": "VERB|_|csubj:vsubj",
+    "315": "VERB|_|discourse",
+    "316": "VERB|_|fixed",
+    "317": "VERB|_|flat:redup",
+    "318": "VERB|_|iobj",
+    "319": "VERB|_|mark",
+    "320": "VERB|_|mark:pcomp",
+    "321": "VERB|_|nmod",
+    "322": "VERB|_|nmod:poss",
+    "323": "VERB|_|nsubj",
+    "324": "VERB|_|nsubj:pass",
+    "325": "VERB|_|nsubj:xsubj",
+    "326": "VERB|_|obj",
+    "327": "VERB|_|obl",
+    "328": "VERB|_|obl:about",
+    "329": "VERB|_|obl:comp",
+    "330": "VERB|_|obl:iobj",
+    "331": "VERB|_|obl:tmod",
+    "332": "VERB|_|parataxis",
+    "333": "VERB|_|root",
+    "334": "VERB|_|vocative",
+    "335": "VERB|_|xcomp",
+    "336": "VERB|_|xcomp:adj",
+    "337": "VERB|_|xcomp:vcomp",
+    "338": "X|_|acl",
+    "339": "X|_|acl:subj",
+    "340": "X|_|acl:tonp",
+    "341": "X|_|advcl",
+    "342": "X|_|amod",
+    "343": "X|_|case",
+    "344": "X|_|cc",
+    "345": "X|_|ccomp",
+    "346": "X|_|compound",
+    "347": "X|_|compound:adj",
+    "348": "X|_|compound:prt",
+    "349": "X|_|compound:vmod",
+    "350": "X|_|compound:z",
+    "351": "X|_|conj",
+    "352": "X|_|discourse",
+    "353": "X|_|dislocated",
+    "354": "X|_|goeswith",
+    "355": "X|_|mark",
+    "356": "X|_|nmod",
+    "357": "X|_|nmod:poss",
+    "358": "X|_|nsubj",
+    "359": "X|_|obj",
+    "360": "X|_|obl",
+    "361": "X|_|obl:about",
+    "362": "X|_|obl:comp",
+    "363": "X|_|obl:tmod",
+    "364": "X|_|parataxis",
+    "365": "X|_|root",
+    "366": "X|_|xcomp"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "-|_|dep": 0,
+    "ADJ|_|acl": 1,
+    "ADJ|_|acl:subj": 2,
+    "ADJ|_|acl:tmod": 3,
+    "ADJ|_|acl:tonp": 4,
+    "ADJ|_|advcl": 5,
+    "ADJ|_|advcl:objective": 6,
+    "ADJ|_|advmod": 7,
+    "ADJ|_|advmod:adj": 8,
+    "ADJ|_|advmod:neg": 9,
+    "ADJ|_|amod": 10,
+    "ADJ|_|appos": 11,
+    "ADJ|_|appos:nmod": 12,
+    "ADJ|_|ccomp": 13,
+    "ADJ|_|compound": 14,
+    "ADJ|_|compound:adj": 15,
+    "ADJ|_|compound:amod": 16,
+    "ADJ|_|compound:apr": 17,
+    "ADJ|_|compound:atov": 18,
+    "ADJ|_|compound:dir": 19,
+    "ADJ|_|compound:prt": 20,
+    "ADJ|_|compound:svc": 21,
+    "ADJ|_|compound:verbnoun": 22,
+    "ADJ|_|compound:vmod": 23,
+    "ADJ|_|conj": 24,
+    "ADJ|_|csubj": 25,
+    "ADJ|_|csubj:asubj": 26,
+    "ADJ|_|dep": 27,
+    "ADJ|_|discourse": 28,
+    "ADJ|_|dislocated": 29,
+    "ADJ|_|fixed": 30,
+    "ADJ|_|flat": 31,
+    "ADJ|_|flat:name": 32,
+    "ADJ|_|nmod": 33,
+    "ADJ|_|nsubj": 34,
+    "ADJ|_|obj": 35,
+    "ADJ|_|obl": 36,
+    "ADJ|_|obl:about": 37,
+    "ADJ|_|obl:adj": 38,
+    "ADJ|_|obl:comp": 39,
+    "ADJ|_|obl:tmod": 40,
+    "ADJ|_|obl:with": 41,
+    "ADJ|_|parataxis": 42,
+    "ADJ|_|root": 43,
+    "ADJ|_|xcomp": 44,
+    "ADJ|_|xcomp:adj": 45,
+    "ADP|_|acl:tmod": 46,
+    "ADP|_|advcl": 47,
+    "ADP|_|case": 48,
+    "ADP|_|cc": 49,
+    "ADP|_|ccomp": 50,
+    "ADP|_|compound": 51,
+    "ADP|_|compound:atov": 52,
+    "ADP|_|compound:dir": 53,
+    "ADP|_|compound:prt": 54,
+    "ADP|_|compound:svc": 55,
+    "ADP|_|conj": 56,
+    "ADP|_|csubj": 57,
+    "ADP|_|dep": 58,
+    "ADP|_|discourse": 59,
+    "ADP|_|fixed": 60,
+    "ADP|_|mark": 61,
+    "ADP|_|mark:pcomp": 62,
+    "ADP|_|nmod": 63,
+    "ADP|_|obl": 64,
+    "ADP|_|obl:tmod": 65,
+    "ADP|_|parataxis": 66,
+    "ADP|_|root": 67,
+    "ADP|_|xcomp": 68,
+    "ADV|_|acl:subj": 69,
+    "ADV|_|advcl": 70,
+    "ADV|_|advcl:objective": 71,
+    "ADV|_|advmod": 72,
+    "ADV|_|advmod:adj": 73,
+    "ADV|_|advmod:dir": 74,
+    "ADV|_|advmod:neg": 75,
+    "ADV|_|appos:nmod": 76,
+    "ADV|_|case": 77,
+    "ADV|_|compound": 78,
+    "ADV|_|compound:apr": 79,
+    "ADV|_|compound:atov": 80,
+    "ADV|_|compound:dir": 81,
+    "ADV|_|compound:prt": 82,
+    "ADV|_|compound:redup": 83,
+    "ADV|_|compound:svc": 84,
+    "ADV|_|conj": 85,
+    "ADV|_|discourse": 86,
+    "ADV|_|fixed": 87,
+    "ADV|_|flat:redup": 88,
+    "ADV|_|mark": 89,
+    "ADV|_|nmod": 90,
+    "ADV|_|obj": 91,
+    "ADV|_|obl": 92,
+    "ADV|_|obl:adv": 93,
+    "ADV|_|obl:tmod": 94,
+    "ADV|_|root": 95,
+    "ADV|_|xcomp": 96,
+    "AUX|_|aux": 97,
+    "AUX|_|aux:pass": 98,
+    "AUX|_|compound": 99,
+    "AUX|_|cop": 100,
+    "AUX|_|discourse": 101,
+    "AUX|_|parataxis": 102,
+    "AUX|_|root": 103,
+    "AUX|_|xcomp": 104,
+    "CCONJ|_|case": 105,
+    "CCONJ|_|cc": 106,
+    "CCONJ|_|flat": 107,
+    "CCONJ|_|mark": 108,
+    "DET|_|advmod:adj": 109,
+    "DET|_|clf:det": 110,
+    "DET|_|det": 111,
+    "DET|_|discourse": 112,
+    "DET|_|nmod:poss": 113,
+    "DET|_|nsubj": 114,
+    "DET|_|obj": 115,
+    "DET|_|obl:tmod": 116,
+    "INTJ|_|discourse": 117,
+    "INTJ|_|root": 118,
+    "NOUN|_|acl": 119,
+    "NOUN|_|acl:subj": 120,
+    "NOUN|_|acl:tmod": 121,
+    "NOUN|_|advcl": 122,
+    "NOUN|_|advcl:objective": 123,
+    "NOUN|_|amod": 124,
+    "NOUN|_|appos": 125,
+    "NOUN|_|appos:nmod": 126,
+    "NOUN|_|case": 127,
+    "NOUN|_|ccomp": 128,
+    "NOUN|_|clf": 129,
+    "NOUN|_|clf:det": 130,
+    "NOUN|_|compound": 131,
+    "NOUN|_|compound:amod": 132,
+    "NOUN|_|compound:dir": 133,
+    "NOUN|_|compound:verbnoun": 134,
+    "NOUN|_|compound:vmod": 135,
+    "NOUN|_|conj": 136,
+    "NOUN|_|csubj": 137,
+    "NOUN|_|csubj:pass": 138,
+    "NOUN|_|csubj:vsubj": 139,
+    "NOUN|_|dep": 140,
+    "NOUN|_|discourse": 141,
+    "NOUN|_|dislocated": 142,
+    "NOUN|_|fixed": 143,
+    "NOUN|_|flat": 144,
+    "NOUN|_|flat:name": 145,
+    "NOUN|_|flat:number": 146,
+    "NOUN|_|flat:time": 147,
+    "NOUN|_|iobj": 148,
+    "NOUN|_|list": 149,
+    "NOUN|_|nmod": 150,
+    "NOUN|_|nmod:poss": 151,
+    "NOUN|_|nsubj": 152,
+    "NOUN|_|nsubj:nn": 153,
+    "NOUN|_|nsubj:pass": 154,
+    "NOUN|_|nsubj:xsubj": 155,
+    "NOUN|_|nummod": 156,
+    "NOUN|_|obj": 157,
+    "NOUN|_|obl": 158,
+    "NOUN|_|obl:about": 159,
+    "NOUN|_|obl:adj": 160,
+    "NOUN|_|obl:adv": 161,
+    "NOUN|_|obl:agent": 162,
+    "NOUN|_|obl:comp": 163,
+    "NOUN|_|obl:iobj": 164,
+    "NOUN|_|obl:tmod": 165,
+    "NOUN|_|obl:with": 166,
+    "NOUN|_|parataxis": 167,
+    "NOUN|_|root": 168,
+    "NOUN|_|vocative": 169,
+    "NOUN|_|xcomp": 170,
+    "NUM|_|amod": 171,
+    "NUM|_|appos": 172,
+    "NUM|_|appos:nmod": 173,
+    "NUM|_|clf": 174,
+    "NUM|_|clf:det": 175,
+    "NUM|_|compound": 176,
+    "NUM|_|compound:verbnoun": 177,
+    "NUM|_|conj": 178,
+    "NUM|_|flat:date": 179,
+    "NUM|_|flat:name": 180,
+    "NUM|_|flat:number": 181,
+    "NUM|_|flat:time": 182,
+    "NUM|_|nmod": 183,
+    "NUM|_|nsubj": 184,
+    "NUM|_|nummod": 185,
+    "NUM|_|obj": 186,
+    "NUM|_|obl": 187,
+    "NUM|_|obl:comp": 188,
+    "NUM|_|obl:tmod": 189,
+    "NUM|_|parataxis": 190,
+    "NUM|_|root": 191,
+    "PART|_|advcl": 192,
+    "PART|_|advmod": 193,
+    "PART|_|amod": 194,
+    "PART|_|case": 195,
+    "PART|_|clf:det": 196,
+    "PART|_|compound": 197,
+    "PART|_|compound:prt": 198,
+    "PART|_|discourse": 199,
+    "PART|_|fixed": 200,
+    "PART|_|mark": 201,
+    "PART|_|obl": 202,
+    "PART|_|parataxis": 203,
+    "PRON|_|acl:tmod": 204,
+    "PRON|_|advcl": 205,
+    "PRON|_|appos:nmod": 206,
+    "PRON|_|ccomp": 207,
+    "PRON|_|compound": 208,
+    "PRON|_|compound:pron": 209,
+    "PRON|_|compound:prt": 210,
+    "PRON|_|conj": 211,
+    "PRON|_|det": 212,
+    "PRON|_|det:pmod": 213,
+    "PRON|_|discourse": 214,
+    "PRON|_|expl": 215,
+    "PRON|_|fixed": 216,
+    "PRON|_|iobj": 217,
+    "PRON|_|nmod": 218,
+    "PRON|_|nmod:poss": 219,
+    "PRON|_|nsubj": 220,
+    "PRON|_|nsubj:nn": 221,
+    "PRON|_|nsubj:pass": 222,
+    "PRON|_|nsubj:xsubj": 223,
+    "PRON|_|obj": 224,
+    "PRON|_|obl": 225,
+    "PRON|_|obl:about": 226,
+    "PRON|_|obl:adj": 227,
+    "PRON|_|obl:comp": 228,
+    "PRON|_|obl:iobj": 229,
+    "PRON|_|obl:tmod": 230,
+    "PRON|_|obl:with": 231,
+    "PRON|_|parataxis": 232,
+    "PRON|_|root": 233,
+    "PROPN|_|acl:subj": 234,
+    "PROPN|_|advcl": 235,
+    "PROPN|_|appos": 236,
+    "PROPN|_|appos:nmod": 237,
+    "PROPN|_|ccomp": 238,
+    "PROPN|_|compound": 239,
+    "PROPN|_|compound:verbnoun": 240,
+    "PROPN|_|conj": 241,
+    "PROPN|_|csubj:pass": 242,
+    "PROPN|_|dep": 243,
+    "PROPN|_|flat": 244,
+    "PROPN|_|flat:name": 245,
+    "PROPN|_|iobj": 246,
+    "PROPN|_|list": 247,
+    "PROPN|_|nmod": 248,
+    "PROPN|_|nmod:poss": 249,
+    "PROPN|_|nsubj": 250,
+    "PROPN|_|nsubj:nn": 251,
+    "PROPN|_|nsubj:pass": 252,
+    "PROPN|_|nsubj:xsubj": 253,
+    "PROPN|_|obj": 254,
+    "PROPN|_|obl": 255,
+    "PROPN|_|obl:agent": 256,
+    "PROPN|_|obl:comp": 257,
+    "PROPN|_|obl:iobj": 258,
+    "PROPN|_|obl:with": 259,
+    "PROPN|_|parataxis": 260,
+    "PROPN|_|root": 261,
+    "PROPN|_|vocative": 262,
+    "PUNCT|_|punct": 263,
+    "SCONJ|_|advcl": 264,
+    "SCONJ|_|case": 265,
+    "SCONJ|_|cc": 266,
+    "SCONJ|_|compound": 267,
+    "SCONJ|_|compound:svc": 268,
+    "SCONJ|_|discourse": 269,
+    "SCONJ|_|fixed": 270,
+    "SCONJ|_|mark": 271,
+    "SCONJ|_|obl": 272,
+    "SCONJ|_|parataxis": 273,
+    "SCONJ|_|root": 274,
+    "SCONJ|_|vocative": 275,
+    "SYM|_|advcl": 276,
+    "SYM|_|appos:nmod": 277,
+    "SYM|_|compound": 278,
+    "SYM|_|compound:z": 279,
+    "SYM|_|discourse": 280,
+    "SYM|_|flat": 281,
+    "SYM|_|flat:date": 282,
+    "SYM|_|flat:name": 283,
+    "SYM|_|flat:number": 284,
+    "SYM|_|flat:time": 285,
+    "SYM|_|nmod": 286,
+    "SYM|_|nsubj": 287,
+    "SYM|_|obj": 288,
+    "VERB|_|acl": 289,
+    "VERB|_|acl:relcl": 290,
+    "VERB|_|acl:subj": 291,
+    "VERB|_|acl:tmod": 292,
+    "VERB|_|acl:tonp": 293,
+    "VERB|_|advcl": 294,
+    "VERB|_|advcl:objective": 295,
+    "VERB|_|advmod": 296,
+    "VERB|_|amod": 297,
+    "VERB|_|appos": 298,
+    "VERB|_|appos:nmod": 299,
+    "VERB|_|case": 300,
+    "VERB|_|ccomp": 301,
+    "VERB|_|compound": 302,
+    "VERB|_|compound:amod": 303,
+    "VERB|_|compound:atov": 304,
+    "VERB|_|compound:dir": 305,
+    "VERB|_|compound:prt": 306,
+    "VERB|_|compound:redup": 307,
+    "VERB|_|compound:svc": 308,
+    "VERB|_|compound:verbnoun": 309,
+    "VERB|_|compound:vmod": 310,
+    "VERB|_|conj": 311,
+    "VERB|_|csubj": 312,
+    "VERB|_|csubj:pass": 313,
+    "VERB|_|csubj:vsubj": 314,
+    "VERB|_|discourse": 315,
+    "VERB|_|fixed": 316,
+    "VERB|_|flat:redup": 317,
+    "VERB|_|iobj": 318,
+    "VERB|_|mark": 319,
+    "VERB|_|mark:pcomp": 320,
+    "VERB|_|nmod": 321,
+    "VERB|_|nmod:poss": 322,
+    "VERB|_|nsubj": 323,
+    "VERB|_|nsubj:pass": 324,
+    "VERB|_|nsubj:xsubj": 325,
+    "VERB|_|obj": 326,
+    "VERB|_|obl": 327,
+    "VERB|_|obl:about": 328,
+    "VERB|_|obl:comp": 329,
+    "VERB|_|obl:iobj": 330,
+    "VERB|_|obl:tmod": 331,
+    "VERB|_|parataxis": 332,
+    "VERB|_|root": 333,
+    "VERB|_|vocative": 334,
+    "VERB|_|xcomp": 335,
+    "VERB|_|xcomp:adj": 336,
+    "VERB|_|xcomp:vcomp": 337,
+    "X|_|acl": 338,
+    "X|_|acl:subj": 339,
+    "X|_|acl:tonp": 340,
+    "X|_|advcl": 341,
+    "X|_|amod": 342,
+    "X|_|case": 343,
+    "X|_|cc": 344,
+    "X|_|ccomp": 345,
+    "X|_|compound": 346,
+    "X|_|compound:adj": 347,
+    "X|_|compound:prt": 348,
+    "X|_|compound:vmod": 349,
+    "X|_|compound:z": 350,
+    "X|_|conj": 351,
+    "X|_|discourse": 352,
+    "X|_|dislocated": 353,
+    "X|_|goeswith": 354,
+    "X|_|mark": 355,
+    "X|_|nmod": 356,
+    "X|_|nmod:poss": 357,
+    "X|_|nsubj": 358,
+    "X|_|obj": 359,
+    "X|_|obl": 360,
+    "X|_|obl:about": 361,
+    "X|_|obl:comp": 362,
+    "X|_|obl:tmod": 363,
+    "X|_|parataxis": 364,
+    "X|_|root": 365,
+    "X|_|xcomp": 366
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 258,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "PhobertTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.22.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 64001
+}

maker.py

@@ -0,0 +1,69 @@
+#! /usr/bin/python3
+src="vinai/phobert-base"
+tgt="KoichiYasuoka/phobert-base-vietnamese-ud-goeswith"
+import os
+url="https://github.com/UniversalDependencies/UD_Vietnamese-VTB"
+d=os.path.basename(url)
+os.system("test -d "+d+" || git clone --depth=1 "+url)
+os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done")
+url="https://github.com/datquocnguyen/VnDT"
+d=os.path.basename(url)
+os.system("test -d "+d+" || git clone --depth=1 "+url)
+os.system("for F in train dev test ; do cp "+d+"/*-gold-*-$F.conll pre-$F.conll ; done")
+class UDgoeswithDataset(object):
+  def __init__(self,conllu,tokenizer):
+    self.ids,self.tags,label=[],[],set()
+    with open(conllu,"r",encoding="utf-8") as r:
+      cls,sep,msk=tokenizer.cls_token_id,tokenizer.sep_token_id,tokenizer.mask_token_id
+      dep,c="-|_|dep",[]
+      for s in r:
+        t=s.split("\t")
+        if len(t)==10 and t[0].isdecimal():
+          c.append(t)
+        elif c!=[]:
+          v=tokenizer([t[1].replace(" ","_") for t in c],add_special_tokens=False)["input_ids"]
+          for i in range(len(v)-1,-1,-1):
+            for j in range(1,len(v[i])):
+              c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"])
+          y=["0"]+[t[0] for t in c]
+          h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
+          p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in c],sum(v,[])
+          if len(v)<tokenizer.model_max_length-3:
+            self.ids.append([cls]+v+[sep])
+            self.tags.append([dep]+p+[dep])
+            label=set(sum([self.tags[-1],list(label)],[]))
+            for i,k in enumerate(v):
+              self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k])
+              self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
+          c=[]
+    self.label2id={l:i for i,l in enumerate(sorted(label))}
+  def __call__(*args):
+    label=set(sum([list(t.label2id) for t in args],[]))
+    lid={l:i for i,l in enumerate(sorted(label))}
+    for t in args:
+      t.label2id=lid
+    return lid
+  __len__=lambda self:len(self.ids)
+  __getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
+from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
+tkz=AutoTokenizer.from_pretrained(src)
+trainDS=UDgoeswithDataset("pre-train.conll",tkz)
+devDS=UDgoeswithDataset("pre-dev.conll",tkz)
+testDS=UDgoeswithDataset("pre-test.conll",tkz)
+lid=trainDS(devDS,testDS)
+cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS,eval_dataset=devDS)
+trn.train()
+trn.save_model("tmpdir")
+tkz.save_pretrained("tmpdir")
+trainDS=UDgoeswithDataset("train.conllu",tkz)
+devDS=UDgoeswithDataset("dev.conllu",tkz)
+testDS=UDgoeswithDataset("test.conllu",tkz)
+lid=trainDS(devDS,testDS)
+cfg=AutoConfig.from_pretrained("tmpdir",num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained("tmpdir",config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS,eval_dataset=devDS)
+trn.train()
+trn.save_model(tgt)
+tkz.save_pretrained(tgt)

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4b75a510688c84e07ecf781ed194dd71d70867edd9f6ae3fab0d93cead7b9fe
+size 538828593

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 256,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "PhobertTokenizer",
+  "unk_token": "<unk>"
+}