Added models and dataset for training

app.py

@@ -0,0 +1,87 @@
+import os
+import streamlit as st
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer
+from torch.utils.data import DataLoader
+import traceback
+
+dir_path = os.path.abspath('./')
+os.environ["HF_HOME"] = dir_path
+start_training = st.button("Train Model")
+
+
+def tokenize_function(examples):
+    # Concatenate Instruction and Response
+    combined_texts = [instr + " " + resp for instr, resp in zip(examples["Instruction"], examples["Response"])]
+    # return tokenizer(combined_texts, padding="max_length", truncation=True)
+    tokenized_inputs = tokenizer(combined_texts, padding="max_length", truncation=True, max_length=512)
+    tokenized_inputs["labels"] = tokenized_inputs["input_ids"].copy()
+    return tokenized_inputs
+
+
+if start_training:
+    st.write("Getting model and dataset ...")
+    # Load the dataset
+    dataset = load_dataset("viber1/indian-law-dataset", cache_dir=dir_path)
+
+    # Update this path based on where the tokenizer files are actually stored
+    tokenizer = AutoTokenizer.from_pretrained('gpt2')
+    tokenizer.pad_token = tokenizer.eos_token
+    # Load the model
+    model = AutoModelForCausalLM.from_pretrained('gpt2')
+    model.gradient_checkpointing_enable()
+
+    st.write("Training setup ...")
+    # Apply the tokenizer to the dataset
+    tokenized_dataset = dataset.map(tokenize_function, batched=True)
+
+    # Split the dataset manually into train and validation sets
+    split_dataset = tokenized_dataset["train"].train_test_split(test_size=0.1)
+
+    # Convert the dataset to PyTorch tensors
+    train_dataset = split_dataset["train"].with_format("torch")
+    eval_dataset = split_dataset["test"].with_format("torch")
+
+    # Create data loaders
+    # reduce batch size 8 to 1
+    train_dataloader = DataLoader(train_dataset, batch_size=1, shuffle=True, pin_memory=True)
+    eval_dataloader = DataLoader(eval_dataset, batch_size=1, pin_memory=True)
+
+    # Define training arguments
+    training_args = TrainingArguments(
+        output_dir="./results",
+        eval_strategy="epoch",
+        learning_rate=2e-5,
+        per_device_train_batch_size=1,
+        per_device_eval_batch_size=1,
+        num_train_epochs=3,
+        weight_decay=0.01,
+        fp16=True,  # Enable mixed precision
+        # save_total_limit=2,
+        logging_dir='./logs',  # Set logging directory
+        logging_steps=10,  # Log more frequently
+        gradient_checkpointing=True,  # Enable gradient checkpointing
+        gradient_accumulation_steps=8  # Accumulate gradients over 8
+    )
+
+    st.write("Training Started .....")
+
+    # Create the Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+    )
+
+    try:
+        trainer.train()
+    except Exception as e:
+        st.write(f"Error: {e}")
+        traceback.print_exc()
+        st.write("some error")
+
+    # Evaluate the model
+    st.write("Training Done ...")
+    results = trainer.evaluate()
+    st.write(results)

models--gpt2/refs/main

@@ -0,0 +1 @@
+607a30d783dfa663caf39e06633721c8d4cfcd7e

models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "resid_pdrop": 0.1,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "vocab_size": 50257
+}

models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.26.0.dev0",
+  "_from_model_config": true
+}

models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:248dfc3911869ec493c76e65bf2fcf7f615828b0254c12b473182f0f81d3a707
+size 548105171

models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e/tokenizer_config.json

@@ -0,0 +1 @@
+{"model_max_length": 1024}

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+transformers==4.41.2
+torch==2.3.1
+datasets==2.20.0
+huggingface_hub==0.23.2
+accelerate==0.32.1

viber1___indian-law-dataset/default/0.0.0/705c4e2c852380d1120f51121ac1ed020b4f743b/cache-22a773a22cb9ef7a.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b900e8f715eeef7213691a1042362250e295b57ef37e5c76ddb07b8ab79d8e08
+size 20424

viber1___indian-law-dataset/default/0.0.0/705c4e2c852380d1120f51121ac1ed020b4f743b/cache-bef1f90cc85606a0.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:135fff71f54f468e93884d1bf9e0d874eb71e7817c9fe5d7767072f9a27d74bd
+size 180784

viber1___indian-law-dataset/default/0.0.0/705c4e2c852380d1120f51121ac1ed020b4f743b/cache-ce2a18a54b30a39e.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bb3b6c99393e9313b34a63e943841617ef514ffd198e6e20eb5307a1ac2d90b
+size 177005624

viber1___indian-law-dataset/default/0.0.0/705c4e2c852380d1120f51121ac1ed020b4f743b/dataset_info.json

@@ -0,0 +1 @@
+{"description": "", "citation": "", "homepage": "", "license": "", "features": {"Instruction": {"dtype": "string", "_type": "Value"}, "Response": {"dtype": "string", "_type": "Value"}}, "builder_name": "json", "dataset_name": "indian-law-dataset", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 12911240, "num_examples": 24607, "dataset_name": "indian-law-dataset"}}, "download_checksums": {"hf://datasets/viber1/indian-law-dataset@705c4e2c852380d1120f51121ac1ed020b4f743b/train.jsonl": {"num_bytes": 14408595, "checksum": null}}, "download_size": 14408595, "dataset_size": 12911240, "size_in_bytes": 27319835}

viber1___indian-law-dataset/default/0.0.0/705c4e2c852380d1120f51121ac1ed020b4f743b/indian-law-dataset-train.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4760527523f1b93bfe0bd6c9521aa0873060b437702d095af44e0d3141cad759
+size 12919232