Upload DeTiME

config.json

@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "DeTiME"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_detime.DeTiMEAutoConfig",
+    "AutoModel": "modeling_detime.DeTiME"
+  },
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 512,
+  "dense_act_fn": "relu",
+  "dropout": 0.1,
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "relu",
+  "hidden_size1": 512,
+  "hidden_size2": 768,
+  "hidden_size3": 4,
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": false,
+  "layer_norm_epsilon": 1e-06,
+  "model": "google/flan-t5-large",
+  "model_name": null,
+  "model_type": "detime",
+  "num_decoder_layers": 6,
+  "num_heads": 8,
+  "num_layer": 1,
+  "num_layers": 6,
+  "output_size": 3072,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.1",
+  "use_cache": true,
+  "vocab_size": 32128
+}

configuration_detime.py

@@ -0,0 +1,26 @@
+
+from transformers import T5Config, PretrainedConfig
+from typing import List
+
+
+# define Flan-T5 nest CNN autoencoder here
+class DeTiMEAutoConfig(T5Config):
+    model_type = "detime"
+
+    def __init__(
+        self,
+        hidden_size1: int = 512,
+        hidden_size3: int = 512,
+        num_layer: int = 1,
+        dropout: float = 0.1,
+        max_length: int = 512,
+        model_name: str = None,
+        **kwargs,
+    ):
+        self.hidden_size1 = hidden_size1
+        self.hidden_size3 = hidden_size3
+        self.num_layer = num_layer
+        self.dropout = dropout
+        self.max_length = max_length
+        self.model_name = model_name
+        super().__init__(**kwargs)

modeling_detime.py

@@ -0,0 +1,120 @@
+"""
+This module includes all the classes and functions for the nested autoencoder.
+"""
+
+from transformers import PreTrainedModel
+from transformers import T5ForConditionalGeneration, AutoModelForSeq2SeqLM
+import datasets
+import torch
+import torch.nn.functional as F
+from torch import nn
+import random
+import os
+from configuration_detime import DeTiMEAutoConfig
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+
+# Define the CNN encoder and decoder model
+class CNNEncoder(nn.Module):
+    def __init__(self, hidden_size1, hidden_size3):
+        super().__init__()
+        # Define the encoder
+        self.encoder = nn.Sequential(
+            nn.Conv1d(in_channels=hidden_size1, out_channels=128, kernel_size=3, stride=1, padding=1),
+            nn.ReLU(),
+            nn.Conv1d(in_channels=128, out_channels=16, kernel_size=3, stride=1, padding=1),
+            nn.ReLU(),
+#             nn.Conv1d(in_channels=16, out_channels=4, kernel_size=3, stride=1, padding=1),
+#             nn.ReLU(),
+            nn.Conv1d(in_channels=16, out_channels=hidden_size3, kernel_size=3, stride=1, padding=1)
+        )
+
+    def forward(self, x):
+        # x = x.permute(0, 2, 1)
+        # Encode the input
+        encoded = self.encoder(x)
+        return encoded
+
+class CNNDecoder(nn.Module):
+        def __init__(self, hidden_size1, hidden_size3) -> None:
+            super().__init__()
+
+            # Define the decoder
+            self.decoder = nn.Sequential(
+                nn.Conv1d(in_channels=hidden_size3, out_channels=16, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(),
+                nn.Conv1d(in_channels=16, out_channels=128, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(),
+    #             nn.Conv1d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1),
+    #             nn.ReLU(),
+                nn.Conv1d(in_channels=128, out_channels=hidden_size1, kernel_size=3, stride=1, padding=1),
+                nn.Sigmoid()
+            )
+
+        def forward(self, x):
+            # Decode the encoding
+            decoded = self.decoder(x)
+            # decoded = decoded.permute(0, 2, 1)
+            return decoded
+        
+
+
+class DeTiME(PreTrainedModel):
+    config_class = DeTiMEAutoConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        #change t5-small to config 
+        model_name_or_path = config.model
+        # peft_config = PrefixTuningConfig(peft_type="PREFIX_TUNING", task_type=TaskType.SEQ_2_SEQ_LM, 
+        #                                  inference_mode=False, num_virtual_tokens=10)
+        # model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)
+        # model = get_peft_model(model, peft_config)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)
+
+        #model.print_trainable_parameters()
+        self.model = model
+        self.config_model = 'CNN'
+        if self.config_model == 'CNN':
+            # self.model = T5ForConditionalGeneration.from_pretrained("t5-small")
+            self.encoder = CNNEncoder(
+                                config.hidden_size1, config.hidden_size3)
+            self.decoder = CNNDecoder(
+                                config.hidden_size1, config.hidden_size3)
+            self.encoder.main_input_name = self.model.main_input_name
+
+
+            self.encoder.main_input_name = self.model.main_input_name
+        self.main_input_name = self.model.main_input_name
+
+    def forward(self, input_ids, attention_mask, labels, **kwargs):
+        output = self.model.encoder(
+            input_ids=input_ids, attention_mask=attention_mask).last_hidden_state   #batch size * seq length * embedding size, 
+        #print(output.shape)
+        if self.config_model == 'CNN':
+            encoder_output = self.encoder(output) #batch size * seq length * embedding size, 1 * batch size * hidden_size
+            #print(encoder_output.shape)
+            
+            output = self.decoder(encoder_output) #1 batch_size, hidden_size
+
+        return self.model.forward(input_ids=input_ids.contiguous(), encoder_outputs=(output.contiguous(), ), labels=labels.contiguous(),  **kwargs)
+
+    def generate(self, input_ids, attention_mask, **kwargs):
+        output = self.model.encoder(
+            input_ids=input_ids, attention_mask=attention_mask).last_hidden_state   #batch size * seq length * embedding size, 
+        #print(output.shape)
+        # encoder_output = self.encoder(output) #batch size * seq length * embedding size, 1 * batch size * hidden_size
+        # #print(encoder_output.shape)
+        if self.config_model == 'CNN':
+            encoder_output = self.encoder(output) #batch size * seq length * embedding size, 1 * batch size * hidden_size
+            #print(encoder_output.shape)
+            
+            output = self.decoder(encoder_output) #1 batch_size, hidden_size
+        elif self.config_model == 'RNN':
+            output = self.encoder(output) #batch size * seq length * embedding size, 1 * batch size * hidden_size
+
+        # output = self.decoder(encoder_output) #1 batch_size, hidden_size
+
+        return self.model.generate(input_ids=input_ids.contiguous(), encoder_outputs=(output.contiguous(), ),  **kwargs)
+

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a9e4b3b793cc18cbca71c6f4b988a6837765380dd02c75bbb8a213891b65d10
+size 3134420082