Add bert-base-uncased JPQD text-classification model

README.md

@@ -0,0 +1,85 @@
+---
+language:
+- en
+license: apache-2.0
+tags:
+- generated_from_trainer
+datasets:
+- glue
+metrics:
+- accuracy
+model-index:
+- name: jpqd-bert-base-ft-sst2
+  results:
+  - task:
+      name: Text Classification
+      type: text-classification
+    dataset:
+      name: GLUE SST2
+      type: glue
+      args: sst2
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.9254587155963303
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# jpqd-bert-base-ft-sst2
+
+> **Note**
+> This model was trained for only 1 epoch and is shared for testing purposes
+
+This model is a fine-tuned version of [bert-base-uncased](https://huggingface.co/bert-base-uncased) on the GLUE SST2 dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2181
+- Accuracy: 0.9255
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 32
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 1.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|
+| 0.4129        | 0.12  | 250  | 0.4416          | 0.8761   |
+| 0.412         | 0.24  | 500  | 0.4969          | 0.8899   |
+| 0.3191        | 0.36  | 750  | 0.2717          | 0.9163   |
+| 0.2688        | 0.48  | 1000 | 0.2432          | 0.9117   |
+| 0.3306        | 0.59  | 1250 | 0.2033          | 0.9243   |
+| 0.224         | 0.71  | 1500 | 0.2383          | 0.9243   |
+| 0.2082        | 0.83  | 1750 | 0.2233          | 0.9255   |
+| 0.2161        | 0.95  | 2000 | 0.2207          | 0.9255   |
+
+
+### Framework versions
+
+- Transformers 4.25.1
+- Pytorch 1.13.1+cu117
+- Datasets 2.8.0
+- Tokenizers 0.13.2

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 1.0,
+    "eval_accuracy": 0.9254587155963303,
+    "eval_loss": 0.21810248494148254,
+    "eval_runtime": 22.1504,
+    "eval_samples": 872,
+    "eval_samples_per_second": 39.367,
+    "eval_steps_per_second": 4.921,
+    "train_loss": 0.40093172477146793,
+    "train_runtime": 1341.7612,
+    "train_samples": 67349,
+    "train_samples_per_second": 50.194,
+    "train_steps_per_second": 1.569
+}

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "bert-base-uncased",
+  "architectures": [
+    "NNCFNetwork"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "finetuning_task": "sst2",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "negative",
+    "1": "positive"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "negative": 0,
+    "positive": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.25.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "eval_accuracy": 0.9254587155963303,
+    "eval_loss": 0.21810248494148254,
+    "eval_runtime": 22.1504,
+    "eval_samples": 872,
+    "eval_samples_per_second": 39.367,
+    "eval_steps_per_second": 4.921
+}

model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74220f4ab04e8b2cf0a8eb0ac647ab95ad8e9cf3abe9af47aee0f4d2b45dab3b
+size 439470129

openvino_config.json

@@ -0,0 +1,101 @@
+{
+  "compression": [
+    {
+      "algorithm": "movement_sparsity",
+      "ignored_scopes": [
+        "{re}.*NNCFEmbedding.*",
+        "{re}.*LayerNorm.*",
+        "{re}.*pooler.*",
+        "{re}.*classifier.*"
+      ],
+      "params": {
+        "enable_structured_masking": true,
+        "importance_regularization_factor": 0.05,
+        "warmup_end_epoch": 2,
+        "warmup_start_epoch": 1
+      },
+      "sparse_structure_by_scopes": [
+        {
+          "mode": "block",
+          "sparse_factors": [
+            32,
+            32
+          ],
+          "target_scopes": "{re}.*BertAttention.*"
+        },
+        {
+          "axis": 0,
+          "mode": "per_dim",
+          "target_scopes": "{re}.*BertIntermediate.*"
+        },
+        {
+          "axis": 1,
+          "mode": "per_dim",
+          "target_scopes": "{re}.*BertOutput.*"
+        }
+      ]
+    },
+    {
+      "algorithm": "quantization",
+      "export_to_onnx_standard_ops": false,
+      "ignored_scopes": [
+        "{re}.*__add___[0-1]",
+        "{re}.*layer_norm_0",
+        "{re}.*matmul_1",
+        "{re}.*__truediv__*"
+      ],
+      "initializer": {
+        "batchnorm_adaptation": {
+          "num_bn_adaptation_samples": 200
+        },
+        "range": {
+          "num_init_samples": 32,
+          "params": {
+            "max_percentile": 99.99,
+            "min_percentile": 0.01
+          },
+          "type": "percentile"
+        }
+      },
+      "overflow_fix": "disable",
+      "preset": "mixed",
+      "scope_overrides": {
+        "activations": {
+          "{re}.*matmul_0": {
+            "mode": "symmetric"
+          }
+        }
+      }
+    }
+  ],
+  "input_info": [
+    {
+      "keyword": "input_ids",
+      "sample_size": [
+        32,
+        128
+      ],
+      "type": "long"
+    },
+    {
+      "keyword": "token_type_ids",
+      "sample_size": [
+        32,
+        128
+      ],
+      "type": "long"
+    },
+    {
+      "keyword": "attention_mask",
+      "sample_size": [
+        32,
+        128
+      ],
+      "type": "long"
+    }
+  ],
+  "log_dir": "/tmp/jpqd-bert-base-ft-sst2",
+  "optimum_version": "1.6.1",
+  "save_onnx_model": false,
+  "transformers_version": "4.25.1"
+}

openvino_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9cfad0e7bc7bf2932066a8a392d1315f9bbbaf4db18504ce184266bbdf288c8
+size 110187016

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99590d19bdea8d7f6dbf5a37e3947a09af3ef5805eb39e5238e5afa19452aeed
+size 779394143

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "name_or_path": "bert-base-uncased",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.40093172477146793,
+    "train_runtime": 1341.7612,
+    "train_samples": 67349,
+    "train_samples_per_second": 50.194,
+    "train_steps_per_second": 1.569
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fbb005c34586cdf1d9dfa3fa4f3e0e2b7209a1984b065821a0ac11dbbf8e40b
+size 3643