SecureBERT-V2

config.json

@@ -1 +1,112 @@
-{"_name_or_path": "ehsanaghaei/SecureBERT", "architectures": ["RobertaForTokenClassification"], "attention_probs_dropout_prob": 0.1, "bos_token_id": 0, "classifier_dropout": null, "eos_token_id": 2, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "id2label": {"0": "O", "1": "B-TIME", "2": "I-TIME", "3": "B-LOC", "4": "B-SECTEAM", "5": "I-SECTEAM", "6": "B-TOOL", "7": "B-IDTY", "8": "I-IDTY", "9": "B-MAL", "10": "B-APT", "11": "I-APT", "12": "I-TOOL", "13": "B-VULNAME", "14": "B-VULID", "15": "I-LOC", "16": "B-ENCR", "17": "B-FILE", "18": "B-SHA2", "19": "B-URL", "20": "B-IP", "21": "PROT", "22": "B-ACT", "23": "I-ACT", "24": "B-MD5", "25": "I-FILE", "26": "B-DOM", "27": "I-MAL", "28": "B-OS", "29": "B-SHA1", "30": "I-URL", "31": "I-IP", "32": "I-SHA2", "33": "I-OS", "34": "B-PROT", "35": "I-VULNAME", "36": "I-PROT", "37": "B-EMAIL", "38": "I-EMAIL", "39": "I-ENCR"}, "initializer_range": 0.02, "intermediate_size": 3072, "label2id": {"LABEL_0": 0, "LABEL_1": 1, "LABEL_10": 10, "LABEL_11": 11, "LABEL_12": 12, "LABEL_13": 13, "LABEL_14": 14, "LABEL_15": 15, "LABEL_16": 16, "LABEL_17": 17, "LABEL_18": 18, "LABEL_19": 19, "LABEL_2": 2, "LABEL_20": 20, "LABEL_21": 21, "LABEL_22": 22, "LABEL_23": 23, "LABEL_24": 24, "LABEL_25": 25, "LABEL_26": 26, "LABEL_27": 27, "LABEL_28": 28, "LABEL_29": 29, "LABEL_3": 3, "LABEL_30": 30, "LABEL_31": 31, "LABEL_32": 32, "LABEL_33": 33, "LABEL_34": 34, "LABEL_35": 35, "LABEL_36": 36, "LABEL_37": 37, "LABEL_38": 38, "LABEL_39": 39, "LABEL_4": 4, "LABEL_5": 5, "LABEL_6": 6, "LABEL_7": 7, "LABEL_8": 8, "LABEL_9": 9}, "layer_norm_eps": 1e-05, "max_position_embeddings": 514, "model_type": "roberta", "num_attention_heads": 12, "num_hidden_layers": 12, "pad_token_id": 1, "position_embedding_type": "absolute", "torch_dtype": "float32", "transformers_version": "4.30.2", "type_vocab_size": 1, "use_cache": true, "vocab_size": 50265}
+{
+  "_name_or_path": "ehsanaghaei/SecureBERT",
+  "architectures": [
+    "RobertaForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+  "0": "O", 
+  "1": "B-TIME", 
+  "2": "I-TIME", 
+  "3": "B-LOC", 
+  "4": "B-SECTEAM", 
+  "5": "I-SECTEAM", 
+  "6": "B-TOOL", 
+  "7": "B-IDTY", 
+  "8": "I-IDTY", 
+  "9": "B-MAL", 
+  "10": "B-APT", 
+  "11": "I-APT", 
+  "12": "I-TOOL", 
+  "13": "B-VULNAME", 
+  "14": "B-VULID", 
+  "15": "I-LOC", 
+  "16": "B-ENCR", 
+  "17": "B-FILE", 
+  "18": "B-SHA2", 
+  "19": "B-URL", 
+  "20": "B-IP", 
+  "21": "PROT", 
+  "22": "B-ACT", 
+  "23": "I-ACT", 
+  "24": "B-MD5", 
+  "25": "I-FILE", 
+  "26": "B-DOM", 
+  "27": "I-MAL", 
+  "28": "B-OS", 
+  "29": "B-SHA1", 
+  "30": "I-URL", 
+  "31": "I-IP", 
+  "32": "I-SHA2", 
+  "33": "I-OS", 
+  "34": "B-PROT", 
+  "35": "I-VULNAME", 
+  "36": "I-PROT", 
+  "37": "B-EMAIL", 
+  "38": "I-EMAIL", 
+  "39": "I-ENCR"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_10": 10,
+    "LABEL_11": 11,
+    "LABEL_12": 12,
+    "LABEL_13": 13,
+    "LABEL_14": 14,
+    "LABEL_15": 15,
+    "LABEL_16": 16,
+    "LABEL_17": 17,
+    "LABEL_18": 18,
+    "LABEL_19": 19,
+    "LABEL_2": 2,
+    "LABEL_20": 20,
+    "LABEL_21": 21,
+    "LABEL_22": 22,
+    "LABEL_23": 23,
+    "LABEL_24": 24,
+    "LABEL_25": 25,
+    "LABEL_26": 26,
+    "LABEL_27": 27,
+    "LABEL_28": 28,
+    "LABEL_29": 29,
+    "LABEL_3": 3,
+    "LABEL_30": 30,
+    "LABEL_31": 31,
+    "LABEL_32": 32,
+    "LABEL_33": 33,
+    "LABEL_34": 34,
+    "LABEL_35": 35,
+    "LABEL_36": 36,
+    "LABEL_37": 37,
+    "LABEL_38": 38,
+    "LABEL_39": 39,
+    "LABEL_4": 4,
+    "LABEL_5": 5,
+    "LABEL_6": 6,
+    "LABEL_7": 7,
+    "LABEL_8": 8,
+    "LABEL_9": 9
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

model.safetensors → optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77bf2a463b0e30ac6af59c0bcf001b0ca3d2a275e09b2d1817d66ec2b8f79a28
-size 496371344
+oid sha256:80eeff212abb98c229cf1a57c2c3ba32aabcaa4d0cc870227c25f7f199bf9ac8
+size 992803077

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4e81ebf0b524e792d70b6595bce9ae5ac0002f39ab99c530759a8750a77ce08
+oid sha256:2e7b4c4b81357d1b66817e0cd5fe8d98dec8c79e4cb9ba57bfcfaf802057f4e2
 size 496415917

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdfc53f7cc49e4005f5d8b36345a038ddf3baff2043f73abe2c889fc8a1bfa92
+size 14575

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74d996bcccf5d50efb552f4856a2c1fd07641d6aa7b4297df83f18bc3624e26a
+size 627

trainer_state.json

@@ -0,0 +1,274 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 9.433962264150944,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.38,
+      "learning_rate": 9.748427672955975e-05,
+      "loss": 1.3506,
+      "step": 200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.496855345911951e-05,
+      "loss": 1.0358,
+      "step": 400
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.9021613276727133,
+      "eval_f1": 0.6651721377101681,
+      "eval_loss": 0.9438411593437195,
+      "eval_precision": 0.6371165644171779,
+      "eval_recall": 0.6958123953098827,
+      "eval_runtime": 9.5281,
+      "eval_samples_per_second": 162.152,
+      "eval_steps_per_second": 10.18,
+      "step": 530
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 9.245283018867925e-05,
+      "loss": 0.9613,
+      "step": 600
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 8.9937106918239e-05,
+      "loss": 0.9255,
+      "step": 800
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 8.742138364779875e-05,
+      "loss": 0.9074,
+      "step": 1000
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.9115099275269094,
+      "eval_f1": 0.698074203024698,
+      "eval_loss": 0.9242610335350037,
+      "eval_precision": 0.6805982815317705,
+      "eval_recall": 0.7164712451144611,
+      "eval_runtime": 8.173,
+      "eval_samples_per_second": 189.038,
+      "eval_steps_per_second": 11.868,
+      "step": 1060
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 8.49056603773585e-05,
+      "loss": 0.8659,
+      "step": 1200
+    },
+    {
+      "epoch": 2.64,
+      "learning_rate": 8.238993710691824e-05,
+      "loss": 0.8605,
+      "step": 1400
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.9139971696899524,
+      "eval_f1": 0.7123908477948492,
+      "eval_loss": 0.9207843542098999,
+      "eval_precision": 0.7052193894299158,
+      "eval_recall": 0.7197096594081519,
+      "eval_runtime": 9.4717,
+      "eval_samples_per_second": 163.117,
+      "eval_steps_per_second": 10.241,
+      "step": 1590
+    },
+    {
+      "epoch": 3.02,
+      "learning_rate": 7.9874213836478e-05,
+      "loss": 0.8467,
+      "step": 1600
+    },
+    {
+      "epoch": 3.4,
+      "learning_rate": 7.735849056603774e-05,
+      "loss": 0.817,
+      "step": 1800
+    },
+    {
+      "epoch": 3.77,
+      "learning_rate": 7.484276729559749e-05,
+      "loss": 0.8003,
+      "step": 2000
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.9139757279471675,
+      "eval_f1": 0.7284265336009113,
+      "eval_loss": 0.9339075088500977,
+      "eval_precision": 0.7081839274414681,
+      "eval_recall": 0.7498604131769961,
+      "eval_runtime": 7.979,
+      "eval_samples_per_second": 193.634,
+      "eval_steps_per_second": 12.157,
+      "step": 2120
+    },
+    {
+      "epoch": 4.15,
+      "learning_rate": 7.232704402515723e-05,
+      "loss": 0.7937,
+      "step": 2200
+    },
+    {
+      "epoch": 4.53,
+      "learning_rate": 6.981132075471698e-05,
+      "loss": 0.777,
+      "step": 2400
+    },
+    {
+      "epoch": 4.91,
+      "learning_rate": 6.729559748427673e-05,
+      "loss": 0.7759,
+      "step": 2600
+    },
+    {
+      "epoch": 5.0,
+      "eval_accuracy": 0.9262404048201037,
+      "eval_f1": 0.7729500891265597,
+      "eval_loss": 0.9294289946556091,
+      "eval_precision": 0.7711459375347338,
+      "eval_recall": 0.7747627024008934,
+      "eval_runtime": 7.9554,
+      "eval_samples_per_second": 194.207,
+      "eval_steps_per_second": 12.193,
+      "step": 2650
+    },
+    {
+      "epoch": 5.28,
+      "learning_rate": 6.477987421383648e-05,
+      "loss": 0.7514,
+      "step": 2800
+    },
+    {
+      "epoch": 5.66,
+      "learning_rate": 6.226415094339622e-05,
+      "loss": 0.7585,
+      "step": 3000
+    },
+    {
+      "epoch": 6.0,
+      "eval_accuracy": 0.9251039924525065,
+      "eval_f1": 0.7699395015818395,
+      "eval_loss": 0.932030200958252,
+      "eval_precision": 0.765393952769808,
+      "eval_recall": 0.7745393634840871,
+      "eval_runtime": 7.9892,
+      "eval_samples_per_second": 193.385,
+      "eval_steps_per_second": 12.141,
+      "step": 3180
+    },
+    {
+      "epoch": 6.04,
+      "learning_rate": 5.974842767295597e-05,
+      "loss": 0.7529,
+      "step": 3200
+    },
+    {
+      "epoch": 6.42,
+      "learning_rate": 5.7232704402515724e-05,
+      "loss": 0.7338,
+      "step": 3400
+    },
+    {
+      "epoch": 6.79,
+      "learning_rate": 5.4716981132075475e-05,
+      "loss": 0.7357,
+      "step": 3600
+    },
+    {
+      "epoch": 7.0,
+      "eval_accuracy": 0.9156910673699559,
+      "eval_f1": 0.732724471830986,
+      "eval_loss": 0.9781692624092102,
+      "eval_precision": 0.7221559483787008,
+      "eval_recall": 0.743606923506421,
+      "eval_runtime": 9.4609,
+      "eval_samples_per_second": 163.304,
+      "eval_steps_per_second": 10.253,
+      "step": 3710
+    },
+    {
+      "epoch": 7.17,
+      "learning_rate": 5.220125786163522e-05,
+      "loss": 0.7282,
+      "step": 3800
+    },
+    {
+      "epoch": 7.55,
+      "learning_rate": 4.968553459119497e-05,
+      "loss": 0.7249,
+      "step": 4000
+    },
+    {
+      "epoch": 7.92,
+      "learning_rate": 4.716981132075472e-05,
+      "loss": 0.7207,
+      "step": 4200
+    },
+    {
+      "epoch": 8.0,
+      "eval_accuracy": 0.917277756336035,
+      "eval_f1": 0.757945684323743,
+      "eval_loss": 0.9978876113891602,
+      "eval_precision": 0.7310165975103734,
+      "eval_recall": 0.7869346733668342,
+      "eval_runtime": 7.9066,
+      "eval_samples_per_second": 195.405,
+      "eval_steps_per_second": 12.268,
+      "step": 4240
+    },
+    {
+      "epoch": 8.3,
+      "learning_rate": 4.4654088050314466e-05,
+      "loss": 0.7137,
+      "step": 4400
+    },
+    {
+      "epoch": 8.68,
+      "learning_rate": 4.213836477987422e-05,
+      "loss": 0.712,
+      "step": 4600
+    },
+    {
+      "epoch": 9.0,
+      "eval_accuracy": 0.9261117543633947,
+      "eval_f1": 0.7805572280913022,
+      "eval_loss": 0.9675424098968506,
+      "eval_precision": 0.7655142795791282,
+      "eval_recall": 0.7962032384142937,
+      "eval_runtime": 7.9396,
+      "eval_samples_per_second": 194.593,
+      "eval_steps_per_second": 12.217,
+      "step": 4770
+    },
+    {
+      "epoch": 9.06,
+      "learning_rate": 3.962264150943397e-05,
+      "loss": 0.7111,
+      "step": 4800
+    },
+    {
+      "epoch": 9.43,
+      "learning_rate": 3.710691823899371e-05,
+      "loss": 0.7048,
+      "step": 5000
+    }
+  ],
+  "max_steps": 7950,
+  "num_train_epochs": 15,
+  "total_flos": 4470902182940160.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b0aa52a3d66faa93cbd911a3f00510e7297e52451a4d13d72962992f719356e
+oid sha256:15afa73cc856b0ff58c9ad71cc382b4a12190fed06d9d360b7dbd0183bb05a5f
 size 3963