<jupyter_start><jupyter_text>Model Settings<jupyter_code>MODEL_NAME = 'gpt2-medium'<jupyter_output><empty_output><jupyter_text>Tokens<jupyter_code>tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_NAME)
print(len(tokenizer))

tokenizer.add_special_tokens(
  {'eos_token':'<|startoftext|>',
   'bos_token':'<|startoftext|>'
  }
)
tokenizer.add_tokens(['[prompt]','[response]','[category_1]',
                      '[category_2]','[origin]','[description]',
                      '<|endoftext|>'])

tokenizer.pad_token = tokenizer.eos_token

tokenizer.save_pretrained("data/modeling/trainer_42/")

print(len(tokenizer))
print("Created tokenizer")<jupyter_output>50257
50264
Created tokenizer<jupyter_text>Dataset<jupyter_code>class wineDataset(torch.utils.data.Dataset):
  def __init__(self, encodings):
    self.encodings = encodings
            
  def __len__(self):
    return len(self.encodings['input_ids'])
    
  def __getitem__(self, idx):
    item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
    item['labels'] = item['input_ids']
    return item

  
with open('data/scraped/name_desc_nlp_ready_train.txt', 'r', encoding='utf8') as file:
    wines_raw_train = file.read().splitlines()
with open('data/scraped/name_desc_nlp_ready_test.txt', 'r', encoding='utf8') as file:
    wines_raw_test = file.read().splitlines()
print("Loaded dataset")

#wines_raw_train, wines_raw_test = train_test_split(wines_raw,test_size=0.2)

#wine_encodings_train = tokenizer(wines_raw_train, max_length=200, truncation=True, padding=True)
wine_encodings_test = tokenizer(wines_raw_test, max_length=200, truncation=True, padding=True)
print("Encoded dataset")

#wine_dataset_train = wineDataset(wine_encodings_train)
wine_dataset_test = wineDataset(wine_encodings_test)
print("Created PyTorch DataSet")

#train_loader = torch.utils.data.DataLoader(wine_dataset_train)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>model = transformers.AutoModelForCausalLM.from_pretrained(MODEL_NAME)
#model.to('cuda')
model.resize_token_embeddings(len(tokenizer))

print(f"model parameters: {model.num_parameters():,}")

training_args = transformers.TrainingArguments(
    output_dir="data/modeling/trainer_42/",
    overwrite_output_dir=True,
    num_train_epochs=1,
    per_device_train_batch_size=2,
    save_steps=100,
    save_total_limit=2,
    fp16=True,
    #deepspeed='data/ds_config.json'
)

trainer = transformers.Trainer(
    model=model,
    args=training_args,
    train_dataset=wine_dataset_test,
)
trainer.train()<jupyter_output><empty_output><jupyter_text>---------------------  Notes<jupyter_code>Before Modifications:

distilgpt2 - single batch = 5.8GB - (2.1 at idle) = 3.7GB 
distilgpt2 - 2 batch = 5.5GB - (2.1 at idle) = 3.7GB 
distilgpt2 - 4 batch = 5.5GB - (2.1 at idle) = 3.7GB same??
distilgpt2 - 8 batch = 6.5GB - (2.1 at idle) = 
distilgpt2 - 20 batch = 10.4GB - (2.1 at idle) = 8.3GB
gpt2-small - 16 batch = oom - (2.1 at idle)
gpt2-small - 12 batch = 10.1 - (2.1 at idle) = 8GB
gpt2-medium - 2 batch = 9.1 - (2.1 at idle) = 7GB
gpt2-medium - 3 batch = 10.4 - (2.1 at idle) = 8.3GB
gpt2-large - 1 batch == oom

fp16
gpt2-medium - 3 batch = 10.4GB - (2.1 at idle) = 8.3GB

deepspeed<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14399509803921567, 'fraction_numerical': 0.035845588235294115, 'mean_word_length': 4.97985347985348, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '377239', 'license': 'Apache-2.0', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1253, 'n_words': 196, 'path': 'notebooks/4a_finetune_gpt2_Trainer.ipynb', 'repo_name': 'cipher982/Wine-o-matic', 'size': '3264'}