Upload 12 files

args.json

@@ -0,0 +1 @@
+{"model_name": "multicoptic/entire_clean_corpus", "src_language": "eng", "tgt_language": "cop", "max_input_length": 128, "max_target_length": 128}

config.json

@@ -0,0 +1,76 @@
+{
+  "_name_or_path": "megalaa/en-cop-mul-norm-group-greekified",
+  "activation_dropout": 0.0,
+  "activation_function": "swish",
+  "add_bias_logits": false,
+  "add_final_layer_norm": false,
+  "architectures": [
+    "MarianMTModel"
+  ],
+  "attention_dropout": 0.0,
+  "bad_words_ids": [
+    [
+      64109
+    ]
+  ],
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "custom_pipelines": {
+    "english-coptic-translation": {
+      "default": {
+        "model": {
+          "pt": "megalaa/en-cop-mul-norm-group-greekified"
+        }
+      },
+      "impl": "english_coptic_pipeline.EnglishCopticPipeline",
+      "pt": [
+        "AutoModelForSeq2SeqLM"
+      ],
+      "tf": [],
+      "type": "text"
+    }
+  },
+  "d_model": 512,
+  "decoder_attention_heads": 8,
+  "decoder_ffn_dim": 2048,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_start_token_id": 64109,
+  "decoder_vocab_size": 64110,
+  "dropout": 0.1,
+  "encoder_attention_heads": 8,
+  "encoder_ffn_dim": 2048,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "eos_token_id": 0,
+  "extra_pos_embeddings": 64110,
+  "forced_eos_token_id": 0,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "marian",
+  "normalize_before": false,
+  "normalize_embedding": false,
+  "num_beams": 4,
+  "num_hidden_layers": 6,
+  "pad_token_id": 64109,
+  "scale_embedding": true,
+  "share_encoder_decoder_embeddings": true,
+  "static_position_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.1",
+  "use_cache": true,
+  "vocab_size": 64110
+}

english_coptic_pipeline.py

@@ -0,0 +1,130 @@
+from typing import Dict
+import numpy as np
+import torch
+from transformers import Pipeline
+from transformers.utils import ModelOutput
+from transformers import pipeline
+from transformers.pipelines import PIPELINE_REGISTRY
+from transformers import AutoModelForSeq2SeqLM
+from huggingface_hub import Repository
+
+SAHIDIC_TAG = "з"
+BOHAIRIC_TAG = "б"
+
+from transformers import GenerationConfig
+
+GENERATION_CONFIG = GenerationConfig(
+    max_length=20,
+    max_new_tokens=128,
+    min_new_tokens=1,
+    min_length=0,
+    early_stopping=True,
+    do_sample=True,
+    num_beams=5,
+    num_beam_groups=1,
+    top_k=50,
+    top_p=0.95,
+    temperature=1.0,
+    diversity_penalty=0.0,
+    output_scores=True,
+    return_dict_in_generate=True,
+)
+
+
+class EnglishCopticPipeline(Pipeline):
+    def _sanitize_parameters(self, **kwargs):
+        preprocess_kwargs = {}
+        if "to_bohairic" in kwargs and kwargs["to_bohairic"]:
+            preprocess_kwargs["to_bohairic"] = True
+        forward_kwargs = {}
+        if "output_confidence" in kwargs and kwargs["output_confidence"]:
+            forward_kwargs["output_confidence"] = True
+
+        return preprocess_kwargs, forward_kwargs, {}
+
+    def preprocess(self, text, to_bohairic=False):
+        if to_bohairic:
+            text = f"{BOHAIRIC_TAG} {text}"
+        else:
+            text = f"{SAHIDIC_TAG} {text}"
+
+        return self.tokenizer.encode(text, return_tensors="pt")
+
+    def _forward(self, input_tensors, output_confidence=False) -> ModelOutput:
+        outputs = self.model.generate(
+            input_tensors[:, : self.tokenizer.model_max_length],
+            generation_config=GENERATION_CONFIG,
+        )
+
+        translated_text = self.tokenizer.decode(
+            outputs.sequences[0], skip_special_tokens=True
+        )
+
+        if output_confidence:
+            scores = outputs.scores
+            confidences = [
+                torch.softmax(score, dim=-1).max().item() for score in scores
+            ]
+            num_words = len(translated_text.split())
+            # scale the predicition probability by the number of words in the sentence
+            scaled_probability = np.exp(sum(np.log(confidences)) / num_words)
+            return translated_text, scaled_probability
+
+        return translated_text, None
+
+    def postprocess(self, outputs):
+        text, confidence = outputs
+        text = degreekify(text)
+
+        if confidence is None:
+            return {
+                "translation": text,
+            }
+        return {
+            "translation": text,
+            "confidence": confidence,
+        }
+
+
+GREEK_TO_COPTIC = {
+    "α": "ⲁ",
+    "β": "ⲃ",
+    "γ": "ⲅ",
+    "δ": "ⲇ",
+    "ε": "ⲉ",
+    "ϛ": "ⲋ",
+    "ζ": "ⲍ",
+    "η": "ⲏ",
+    "θ": "ⲑ",
+    "ι": "ⲓ",
+    "κ": "ⲕ",
+    "λ": "ⲗ",
+    "μ": "ⲙ",
+    "ν": "ⲛ",
+    "ξ": "ⲝ",
+    "ο": "ⲟ",
+    "π": "ⲡ",
+    "ρ": "ⲣ",
+    "σ": "ⲥ",
+    "τ": "ⲧ",
+    "υ": "ⲩ",
+    "φ": "ⲫ",
+    "χ": "ⲭ",
+    "ψ": "ⲯ",
+    "ω": "ⲱ",
+    "s": "ϣ",
+    "f": "ϥ",
+    "k": "ϧ",
+    "h": "ϩ",
+    "j": "ϫ",
+    "c": "ϭ",
+    "t": "ϯ",
+}
+
+
+def degreekify(greek_text):
+    chars = []
+    for c in greek_text:
+        l_c = c.lower()
+        chars.append(GREEK_TO_COPTIC.get(l_c, l_c))
+    return "".join(chars)

generation_config.json

@@ -0,0 +1,16 @@
+{
+  "bad_words_ids": [
+    [
+      64109
+    ]
+  ],
+  "bos_token_id": 0,
+  "decoder_start_token_id": 64109,
+  "eos_token_id": 0,
+  "forced_eos_token_id": 0,
+  "max_length": 512,
+  "num_beams": 4,
+  "pad_token_id": 64109,
+  "renormalize_logits": true,
+  "transformers_version": "4.38.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55169bef13952d3624e764b9cfc12184dfe36b791dcd62ac6c00969732fcd649
+size 308136760

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64109": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "separate_vocabs": false,
+  "source_lang": "eng",
+  "sp_model_kwargs": {},
+  "target_lang": "mul",
+  "tokenizer_class": "MarianTokenizer",
+  "unk_token": "<unk>"
+}

train_config.json

@@ -0,0 +1 @@
+{"job_hash": "3d348028d8010224e22c79db67880e89", "evaluation_strategy": "steps", "learning_rate": 5e-05, "per_device_train_batch_size": 96, "per_device_eval_batch_size": 96, "weight_decay": 0, "save_total_limit": 4, "num_train_epochs": 12, "num_train_steps": null, "predict_with_generate": true, "eval_steps": 238, "logging_steps": 50, "label_smoothing_factor": 0, "commit_hash": "e045ea0"}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22aa8d502427aefee756f474d55cd2a0d9395db1d71f196d9e7a571d0b785272
+size 4603