<jupyter_start><jupyter_text>**Hotels**<jupyter_code>import os

TASK = 'SDS-H'

for set_ in get_sets(TASK):
  os.system('wget https://wothub-data.s3.amazonaws.com/Corpus/%s -nc'%set_)
train_path, dev_path, test_path = get_sets(TASK)
# Load data as pandas dataframes with two columns -- sentences and labels
train_data = pd.read_csv(train_path, sep="__label__", header=None, names=["text", "label"], engine="python")
dev_data = pd.read_csv(dev_path, sep="__label__", header=None, names=["text", "label"], engine="python")
test_data = pd.read_csv(test_path, sep="__label__", header=None, names=["text", "label"], engine="python")
train_data = train_data[train_data.label != 'z_amb']
dev_data = dev_data[dev_data.label != 'z_amb']
test_data = test_data[test_data.label != 'z_amb']
# Convert to numpy arrays
train_sentences, dev_sentences, test_sentences = [data.iloc[:,0] for data in (train_data, dev_data, test_data)]
train_labels, dev_labels, test_labels = [data.iloc[:,1] for data in (train_data, dev_data, test_data)]
# Check for errors in data labeling, removing nans
def remove_nulls(sentences, labels):
  lab = pd.Series(labels)
  sen = pd.Series(sentences)
  lab_nuls = pd.isnull(lab)
  sen_nuls = pd.isnull(sen)
  not_nuls = ~(lab_nuls | sen_nuls)
  lab = lab.loc[not_nuls].to_numpy()
  sen = sen.loc[not_nuls].to_numpy()
  return [sen, lab]
train_sentences, train_labels = remove_nulls(train_sentences, train_labels)
dev_sentences, dev_labels = remove_nulls(dev_sentences, dev_labels)
test_sentences, test_labels = remove_nulls(test_sentences, test_labels)
len(np.concatenate((train_labels, dev_labels, test_labels)))
np.unique(np.concatenate((train_labels, dev_labels, test_labels)), return_counts=True)
from sklearn.preprocessing import LabelEncoder
# Encoder for labels
labelencoder = LabelEncoder()
train_labels = labelencoder.fit_transform(train_labels)
test_labels = labelencoder.transform(test_labels)
dev_labels = labelencoder.transform(dev_labels)
labelencoder.classes_
len(train_sentences)  * (1-0.875)
dev_sentences = np.append(dev_sentences, train_sentences[:2178])
dev_labels = np.append(dev_labels, train_labels[:2178])
train_sentences = train_sentences[2178:]
train_labels = train_labels[2178:]
print('Train data')
print(len(train_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
print('Dev data')
print(len(dev_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
print('Test data')
print(len(test_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
# Run this cell to mount your Google Drive.
from google.colab import drive
drive.mount('/content/drive')
dev_polemo_data = pd.DataFrame([dev_sentences,dev_labels]).T
test_polemo_data = pd.DataFrame([test_sentences,test_labels]).T
dev_polemo_data.to_csv('/content/drive/My Drive/dev_polemo_hotels_data_preprocessed.csv')
test_polemo_data.to_csv('/content/drive/My Drive/test_polemo_hotels_data_preprocessed.csv')
from sklearn.model_selection import StratifiedShuffleSplit

X = train_sentences
y = train_labels
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in sss.split(X, y):
  train_sentences, test_sentences = X[train_index], X[test_index]
  train_labels, test_labels = y[train_index], y[test_index]

X = test_sentences
y = test_labels
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=42)
for train_index, test_index in sss.split(X, y):
  dev_sentences, test_sentences = X[train_index], X[test_index]
  dev_labels, test_labels = y[train_index], y[test_index]
# Remove long sentences.
# TO-DO Possible cut?
def remove_big(sentences, labels):
  to_remove = []
  for i, sent in enumerate(sentences):
      input_ids = tokenizer.encode(sent, add_special_tokens=True) # TO-DO: add_special_tokens
      if len(input_ids) > MAX_LEN:
        to_remove.append(i)

  sentences = np.delete(sentences, to_remove)
  labels = np.delete(labels, to_remove) 

  print('{} samples removed.'.format(len(to_remove)))

  return sentences, labels
# Downloading tokenizer
# From Polbert - Polish BERT by <NAME>: https://github.com/kldarek/polbert
tokenizer = BertTokenizer.from_pretrained("dkleczek/bert-base-polish-uncased-v1")
MAX_LEN = 128

train_sentences, train_labels = remove_big(train_sentences, train_labels)
test_sentences, test_labels = remove_big(test_sentences, test_labels)
dev_sentences, dev_labels = remove_big(dev_sentences, dev_labels)
from torch.utils.data import TensorDataset
# Create TensorDatasets for train/dev/test sets
def tensor_dataset(sentences, labels):
  input_ids = []
  attention_masks = []

  for sent in sentences:
      encoded_dict = tokenizer.encode_plus(
                          sent,                     
                          add_special_tokens = True,
                          max_length = MAX_LEN,
                          pad_to_max_length = True,
                          return_attention_mask = True,
                          return_tensors = 'pt',     # Return pytorch tensors.
                    )
      
      input_ids.append(encoded_dict['input_ids'])
      attention_masks.append(encoded_dict['attention_mask'])

  # Convert the lists into tensors.
  input_ids = torch.cat(input_ids, dim=0)
  attention_masks = torch.cat(attention_masks, dim=0)
  labels = torch.tensor(labels)
  dataset = TensorDataset(input_ids, attention_masks, labels)
  return dataset
BATCH_SIZE = 8

train_dataset = tensor_dataset(train_sentences, train_labels)
test_dataset = tensor_dataset(test_sentences, test_labels)
dev_dataset = tensor_dataset(dev_sentences, dev_labels)

from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
# Create the DataLoaders for train/dev/test sets.
train_dataloader = DataLoader(train_dataset, sampler = RandomSampler(train_dataset), batch_size = BATCH_SIZE)
validation_dataloader = DataLoader(dev_dataset, sampler = SequentialSampler(dev_dataset), batch_size = BATCH_SIZE)
test_dataloader = DataLoader(test_dataset, sampler = SequentialSampler(test_dataset), batch_size = BATCH_SIZE)
batch_size = 32
# Load model with a sequence classification head
model = BertForSequenceClassification.from_pretrained(
    "dkleczek/bert-base-polish-uncased-v1", # Polbert - Polish BERT by <NAME>: https://github.com/kldarek/polbert
    num_labels = 3,
    output_attentions = False,
    output_hidden_states = False,
)
model.cuda()
import time, datetime
import numpy as np
from tqdm import tqdm
from transformers.optimization import AdamW
from transformers import AutoModelForSequenceClassification
from transformers import get_linear_schedule_with_warmup
from sklearn.metrics import f1_score


# Function to calculate the accuracy of our predictions vs labels
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)

# Takes a time in seconds and returns a string hh:mm:ss
def format_time(elapsed):
  elapsed_rounded = int(round((elapsed)))
  return str(datetime.timedelta(seconds=elapsed_rounded))

# Parameters:
epochs = 3
#lr = 1e-3 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
lr = 5e-5 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
adam_epsilon = 1e-8
WARM_UP = 0

optimizer = AdamW(model.parameters(), lr = lr, eps = adam_epsilon)

from transformers import get_linear_schedule_with_warmup
total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = WARM_UP, num_training_steps = total_steps)

train_loss_values = []
dev_acc_values = []

model.zero_grad()

t0 = time.time()
for epoch_i in range(0, epochs):  
  print("")
  print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
  print('Training...')

  # https://github.com/huggingface/transformers/blob/master/examples/run_glue.py
  # linie 168-183
  epoch_train_loss = 0 # Cumulative loss
  loss = 0 ;     batch_loss = 0
  model.train()

  for step, batch in enumerate(train_dataloader):

    # Progress update every 40 batches.
    if step % 40 == 0 and not step == 0:
      # Calculate elapsed time in minutes.
      elapsed = format_time(time.time() - t0)      
      # Report progress.
      print('  Batch {:>5,}  of  {:>5,}. Loss: {:.3f}  Elapsed: {:}.'.format(step, len(train_dataloader), batch_loss, elapsed))
    

    batch_loss = 0
    b_input_ids = batch[0].to(device)
    b_input_mask = batch[1].to(device)
    b_labels = batch[2].to(device)    

    # clear any previously calculated gradients before backward pass
    optimizer.zero_grad()

    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)

    loss = outputs[0]
    epoch_train_loss += loss.item()
    batch_loss += loss.item()
    loss.backward()

    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)  # Gradient clipping is not in AdamW anymore (so you can use amp without issue)
    optimizer.step()
    scheduler.step()  # Update learning rate schedule

  epoch_train_loss = epoch_train_loss / len(train_dataloader)          
  train_loss_values.append(epoch_train_loss)
  
  print('Average training loss: {0:.2f}'.format(epoch_train_loss))

  # Evaluation
  total_eval_accuracy = 0
  model.eval()

  for batch in validation_dataloader:
    
    input_ids = batch[0].to(device)
    attention_masks = batch[1].to(device)
    labels = batch[2].to('cpu').numpy()
                
    with torch.no_grad():
        outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)

    logits = outputs[0]
    logits = logits.detach().cpu().numpy()

    predictions = np.argmax(logits, axis=1).flatten()
    total_eval_accuracy += flat_accuracy(logits, labels)

  avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
  print("  Accuracy: {0:.4f}".format(avg_val_accuracy))

predicted_labels = [] ; true_labels = []; logits_list = []

for batch in test_dataloader:
  
  input_ids = batch[0].to(device)
  attention_masks = batch[1].to(device)
  labels = batch[2]
  
  with torch.no_grad():        
      outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)
                  
  logits = outputs[0]
  logits = logits.detach().cpu().numpy()
  logits_list.append(logits)
  
  predictions = np.argmax(logits, axis=1).flatten()
  labels = labels.numpy().flatten()

  predicted_labels.extend( predictions )
  true_labels.extend( labels )
  
def inverse_logit(x):
  return np.exp(x) / (1 + np.exp(x))
# Parameters:
#epochs = 2
#lr = 3e-5 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
#adam_epsilon = 1e-8
#WARM_UP = 0
#87 86 87
from sklearn.metrics import classification_report 
print( classification_report(y_true=true_labels, y_pred=predicted_labels, zero_division=0) )
# Run this cell to mount your Google Drive.
from google.colab import drive
drive.mount('/content/drive')
import os
from transformers import WEIGHTS_NAME, CONFIG_NAME
output_dir = "/content/drive/My Drive/model_bert_finetuned_1_1"

# Step 1: Save a model, configuration and vocabulary that you have fine-tuned
# If we have a distributed model, save only the encapsulated model
# (it was wrapped in PyTorch DistributedDataParallel or DataParallel)

model_to_save = model.module if hasattr(model, 'module') else model
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(output_dir, CONFIG_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_pretrained(output_dir)
# Step 2: Re-load the saved model and vocabulary
model = BertForSequenceClassification.from_pretrained(output_dir)
tokenizer = BertTokenizer.from_pretrained(output_dir)<jupyter_output><empty_output><jupyter_text>**Medicine**<jupyter_code>import os

TASK = 'SDS-M'

for set_ in get_sets(TASK):
  os.system('wget https://wothub-data.s3.amazonaws.com/Corpus/%s -nc'%set_)
train_path, dev_path, test_path = get_sets(TASK)
# Load data as pandas dataframes with two columns -- sentences and labels
train_data = pd.read_csv(train_path, sep="__label__", header=None, names=["text", "label"], engine="python")
dev_data = pd.read_csv(dev_path, sep="__label__", header=None, names=["text", "label"], engine="python")
test_data = pd.read_csv(test_path, sep="__label__", header=None, names=["text", "label"], engine="python")
train_data = train_data[train_data.label != 'z_amb']
dev_data = dev_data[dev_data.label != 'z_amb']
test_data = test_data[test_data.label != 'z_amb']
# Convert to numpy arrays
train_sentences, dev_sentences, test_sentences = [data.iloc[:,0] for data in (train_data, dev_data, test_data)]
train_labels, dev_labels, test_labels = [data.iloc[:,1] for data in (train_data, dev_data, test_data)]
# Check for errors in data labeling, removing nans
def remove_nulls(sentences, labels):
  lab = pd.Series(labels)
  sen = pd.Series(sentences)
  lab_nuls = pd.isnull(lab)
  sen_nuls = pd.isnull(sen)
  not_nuls = ~(lab_nuls | sen_nuls)
  lab = lab.loc[not_nuls].to_numpy()
  sen = sen.loc[not_nuls].to_numpy()
  return [sen, lab]
train_sentences, train_labels = remove_nulls(train_sentences, train_labels)
dev_sentences, dev_labels = remove_nulls(dev_sentences, dev_labels)
test_sentences, test_labels = remove_nulls(test_sentences, test_labels)
np.unique(np.concatenate((train_labels, dev_labels, test_labels)), return_counts=True)
len(train_sentences)  * (1-0.875)
from sklearn.preprocessing import LabelEncoder
# Encoder for labels
labelencoder = LabelEncoder()
train_labels = labelencoder.fit_transform(train_labels)
test_labels = labelencoder.transform(test_labels)
dev_labels = labelencoder.transform(dev_labels)
labelencoder.classes_
dev_sentences = np.append(dev_sentences, train_sentences[:2127])
dev_labels = np.append(dev_labels, train_labels[:2127])
train_sentences = train_sentences[2127:]
train_labels = train_labels[2127:]
print('Train data')
print(len(train_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
print('Dev data')
print(len(dev_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
print('Test data')
print(len(test_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
# Run this cell to mount your Google Drive.
from google.colab import drive
drive.mount('/content/drive')
dev_polemo_data = pd.DataFrame([dev_sentences,dev_labels]).T
test_polemo_data = pd.DataFrame([test_sentences,test_labels]).T
dev_polemo_data.to_csv('/content/drive/My Drive/dev_polemo_medicine_data_preprocessed.csv')
test_polemo_data.to_csv('/content/drive/My Drive/test_polemo_medicine_data_preprocessed.csv')
from sklearn.model_selection import StratifiedShuffleSplit

X = train_sentences
y = train_labels
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in sss.split(X, y):
  train_sentences, test_sentences = X[train_index], X[test_index]
  train_labels, test_labels = y[train_index], y[test_index]

X = test_sentences
y = test_labels
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=42)
for train_index, test_index in sss.split(X, y):
  dev_sentences, test_sentences = X[train_index], X[test_index]
  dev_labels, test_labels = y[train_index], y[test_index]
from sklearn.preprocessing import LabelEncoder
# Encoder for labels
labelencoder = LabelEncoder()
train_labels = labelencoder.fit_transform(train_labels)
test_labels = labelencoder.transform(test_labels)
dev_labels = labelencoder.transform(dev_labels)
# Remove long sentences.
# TO-DO Possible cut?
def remove_big(sentences, labels):
  to_remove = []
  for i, sent in enumerate(sentences):
      input_ids = tokenizer.encode(sent, add_special_tokens=True) # TO-DO: add_special_tokens
      if len(input_ids) > MAX_LEN:
        to_remove.append(i)

  sentences = np.delete(sentences, to_remove)
  labels = np.delete(labels, to_remove) 

  print('{} samples removed.'.format(len(to_remove)))

  return sentences, labels
# Downloading tokenizer
# From Polbert - Polish BERT by <NAME>: https://github.com/kldarek/polbert
tokenizer = BertTokenizer.from_pretrained("dkleczek/bert-base-polish-uncased-v1")
MAX_LEN = 128

train_sentences, train_labels = remove_big(train_sentences, train_labels)
test_sentences, test_labels = remove_big(test_sentences, test_labels)
dev_sentences, dev_labels = remove_big(dev_sentences, dev_labels)
from torch.utils.data import TensorDataset
# Create TensorDatasets for train/dev/test sets
def tensor_dataset(sentences, labels):
  input_ids = []
  attention_masks = []

  for sent in sentences:
      encoded_dict = tokenizer.encode_plus(
                          sent,                     
                          add_special_tokens = True,
                          max_length = MAX_LEN,
                          pad_to_max_length = True,
                          return_attention_mask = True,
                          return_tensors = 'pt',     # Return pytorch tensors.
                    )
      
      input_ids.append(encoded_dict['input_ids'])
      attention_masks.append(encoded_dict['attention_mask'])

  # Convert the lists into tensors.
  input_ids = torch.cat(input_ids, dim=0)
  attention_masks = torch.cat(attention_masks, dim=0)
  labels = torch.tensor(labels)
  dataset = TensorDataset(input_ids, attention_masks, labels)
  return dataset
BATCH_SIZE = 8

train_dataset = tensor_dataset(train_sentences, train_labels)
test_dataset = tensor_dataset(test_sentences, test_labels)
dev_dataset = tensor_dataset(dev_sentences, dev_labels)

from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
# Create the DataLoaders for train/dev/test sets.
train_dataloader = DataLoader(train_dataset, sampler = RandomSampler(train_dataset), batch_size = BATCH_SIZE)
validation_dataloader = DataLoader(dev_dataset, sampler = SequentialSampler(dev_dataset), batch_size = BATCH_SIZE)
test_dataloader = DataLoader(test_dataset, sampler = SequentialSampler(test_dataset), batch_size = BATCH_SIZE)
# Load model with a sequence classification head
model = BertForSequenceClassification.from_pretrained(
    "dkleczek/bert-base-polish-uncased-v1", # Polbert - Polish BERT by <NAME>: https://github.com/kldarek/polbert
    num_labels = 3,
    output_attentions = False,
    output_hidden_states = False,
)
model.cuda()
import time, datetime
import numpy as np
from tqdm import tqdm
from transformers.optimization import AdamW
from transformers import AutoModelForSequenceClassification
from transformers import get_linear_schedule_with_warmup
from sklearn.metrics import f1_score


# Function to calculate the accuracy of our predictions vs labels
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)

# Takes a time in seconds and returns a string hh:mm:ss
def format_time(elapsed):
  elapsed_rounded = int(round((elapsed)))
  return str(datetime.timedelta(seconds=elapsed_rounded))

# Parameters:
epochs = 3
#lr = 1e-3 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
lr = 5e-5 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
adam_epsilon = 1e-8
WARM_UP = 0

optimizer = AdamW(model.parameters(), lr = lr, eps = adam_epsilon)

from transformers import get_linear_schedule_with_warmup
total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = WARM_UP, num_training_steps = total_steps)

train_loss_values = []
dev_acc_values = []

model.zero_grad()

t0 = time.time()
for epoch_i in range(0, epochs):  
  print("")
  print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
  print('Training...')

  # https://github.com/huggingface/transformers/blob/master/examples/run_glue.py
  # linie 168-183
  epoch_train_loss = 0 # Cumulative loss
  loss = 0 ;     batch_loss = 0
  model.train()

  for step, batch in enumerate(train_dataloader):

    # Progress update every 40 batches.
    if step % 40 == 0 and not step == 0:
      # Calculate elapsed time in minutes.
      elapsed = format_time(time.time() - t0)      
      # Report progress.
      print('  Batch {:>5,}  of  {:>5,}. Loss: {:.3f}  Elapsed: {:}.'.format(step, len(train_dataloader), batch_loss, elapsed))
    

    batch_loss = 0
    b_input_ids = batch[0].to(device)
    b_input_mask = batch[1].to(device)
    b_labels = batch[2].to(device)    

    # clear any previously calculated gradients before backward pass
    optimizer.zero_grad()

    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)

    loss = outputs[0]
    epoch_train_loss += loss.item()
    batch_loss += loss.item()
    loss.backward()

    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)  # Gradient clipping is not in AdamW anymore (so you can use amp without issue)
    optimizer.step()
    scheduler.step()  # Update learning rate schedule

  epoch_train_loss = epoch_train_loss / len(train_dataloader)          
  train_loss_values.append(epoch_train_loss)
  
  print('Average training loss: {0:.2f}'.format(epoch_train_loss))

  # Evaluation
  total_eval_accuracy = 0
  model.eval()

  for batch in validation_dataloader:
    
    input_ids = batch[0].to(device)
    attention_masks = batch[1].to(device)
    labels = batch[2].to('cpu').numpy()
                
    with torch.no_grad():
        outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)

    logits = outputs[0]
    logits = logits.detach().cpu().numpy()

    predictions = np.argmax(logits, axis=1).flatten()
    total_eval_accuracy += flat_accuracy(logits, labels)

  avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
  print("  Accuracy: {0:.4f}".format(avg_val_accuracy))

predicted_labels = [] ; true_labels = []; logits_list = []

for batch in test_dataloader:
  
  input_ids = batch[0].to(device)
  attention_masks = batch[1].to(device)
  labels = batch[2]
  
  with torch.no_grad():        
      outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)
                  
  logits = outputs[0]
  logits = logits.detach().cpu().numpy()
  logits_list.append(logits)
  
  predictions = np.argmax(logits, axis=1).flatten()
  labels = labels.numpy().flatten()

  predicted_labels.extend( predictions )
  true_labels.extend( labels )
  
# Parameters:
#epochs = 2
#lr = 3e-5 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
#adam_epsilon = 1e-8
#WARM_UP = 0
#87 86 87
from sklearn.metrics import classification_report 
print( classification_report(y_true=true_labels, y_pred=predicted_labels, zero_division=0) )
import os
from transformers import WEIGHTS_NAME, CONFIG_NAME
output_dir = "/content/drive/My Drive/model_bert_finetuned_1_2"

# Step 1: Save a model, configuration and vocabulary that you have fine-tuned
# If we have a distributed model, save only the encapsulated model
# (it was wrapped in PyTorch DistributedDataParallel or DataParallel)

model_to_save = model.module if hasattr(model, 'module') else model
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(output_dir, CONFIG_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_pretrained(output_dir)
# Step 2: Re-load the saved model and vocabulary
model = BertForSequenceClassification.from_pretrained(output_dir)
tokenizer = BertTokenizer.from_pretrained(output_dir)<jupyter_output><empty_output><jupyter_text>**Products**<jupyter_code>import os

TASK = 'SDS-P'

for set_ in get_sets(TASK):
  os.system('wget https://wothub-data.s3.amazonaws.com/Corpus/%s -nc'%set_)
train_path, dev_path, test_path = get_sets(TASK)
# Load data as pandas dataframes with two columns -- sentences and labels
train_data = pd.read_csv(train_path, sep="__label__", header=None, names=["text", "label"], engine="python")
dev_data = pd.read_csv(dev_path, sep="__label__", header=None, names=["text", "label"], engine="python")
test_data = pd.read_csv(test_path, sep="__label__", header=None, names=["text", "label"], engine="python")
train_data = train_data[train_data.label != 'z_amb']
dev_data = dev_data[dev_data.label != 'z_amb']
test_data = test_data[test_data.label != 'z_amb']
# Convert to numpy arrays
train_sentences, dev_sentences, test_sentences = [data.iloc[:,0] for data in (train_data, dev_data, test_data)]
train_labels, dev_labels, test_labels = [data.iloc[:,1] for data in (train_data, dev_data, test_data)]
# Check for errors in data labeling, removing nans
def remove_nulls(sentences, labels):
  lab = pd.Series(labels)
  sen = pd.Series(sentences)
  lab_nuls = pd.isnull(lab)
  sen_nuls = pd.isnull(sen)
  not_nuls = ~(lab_nuls | sen_nuls)
  lab = lab.loc[not_nuls].to_numpy()
  sen = sen.loc[not_nuls].to_numpy()
  return [sen, lab]
train_sentences, train_labels = remove_nulls(train_sentences, train_labels)
dev_sentences, dev_labels = remove_nulls(dev_sentences, dev_labels)
test_sentences, test_labels = remove_nulls(test_sentences, test_labels)
np.unique(np.concatenate((train_labels, dev_labels, test_labels)), return_counts=True)
len(train_sentences)  * (1-0.875)
from sklearn.preprocessing import LabelEncoder
# Encoder for labels
labelencoder = LabelEncoder()
train_labels = labelencoder.fit_transform(train_labels)
test_labels = labelencoder.transform(test_labels)
dev_labels = labelencoder.transform(dev_labels)
labelencoder.classes_
dev_sentences = np.append(dev_sentences, train_sentences[:594])
dev_labels = np.append(dev_labels, train_labels[:594])
train_sentences = train_sentences[594:]
train_labels = train_labels[594:]
print('Train data')
print(len(train_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
print('Dev data')
print(len(dev_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
print('Test data')
print(len(test_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
# Run this cell to mount your Google Drive.
from google.colab import drive
drive.mount('/content/drive')
dev_polemo_data = pd.DataFrame([dev_sentences,dev_labels]).T
test_polemo_data = pd.DataFrame([test_sentences,test_labels]).T
dev_polemo_data.to_csv('/content/drive/My Drive/dev_polemo_products_data_preprocessed.csv')
test_polemo_data.to_csv('/content/drive/My Drive/test_polemo_products_data_preprocessed.csv')
from sklearn.model_selection import StratifiedShuffleSplit

X = train_sentences
y = train_labels
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in sss.split(X, y):
  train_sentences, test_sentences = X[train_index], X[test_index]
  train_labels, test_labels = y[train_index], y[test_index]

X = test_sentences
y = test_labels
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=42)
for train_index, test_index in sss.split(X, y):
  dev_sentences, test_sentences = X[train_index], X[test_index]
  dev_labels, test_labels = y[train_index], y[test_index]
from sklearn.preprocessing import LabelEncoder
# Encoder for labels
labelencoder = LabelEncoder()
train_labels = labelencoder.fit_transform(train_labels)
test_labels = labelencoder.transform(test_labels)
dev_labels = labelencoder.transform(dev_labels)
# Remove long sentences.
# TO-DO Possible cut?
def remove_big(sentences, labels):
  to_remove = []
  for i, sent in enumerate(sentences):
      input_ids = tokenizer.encode(sent, add_special_tokens=True) # TO-DO: add_special_tokens
      if len(input_ids) > MAX_LEN:
        to_remove.append(i)

  sentences = np.delete(sentences, to_remove)
  labels = np.delete(labels, to_remove) 

  print('{} samples removed.'.format(len(to_remove)))

  return sentences, labels
# Downloading tokenizer
# From Polbert - Polish BERT by <NAME>: https://github.com/kldarek/polbert
tokenizer = BertTokenizer.from_pretrained("dkleczek/bert-base-polish-uncased-v1")
MAX_LEN = 128

train_sentences, train_labels = remove_big(train_sentences, train_labels)
test_sentences, test_labels = remove_big(test_sentences, test_labels)
dev_sentences, dev_labels = remove_big(dev_sentences, dev_labels)
from torch.utils.data import TensorDataset
# Create TensorDatasets for train/dev/test sets
def tensor_dataset(sentences, labels):
  input_ids = []
  attention_masks = []

  for sent in sentences:
      encoded_dict = tokenizer.encode_plus(
                          sent,                     
                          add_special_tokens = True,
                          max_length = MAX_LEN,
                          pad_to_max_length = True,
                          return_attention_mask = True,
                          return_tensors = 'pt',     # Return pytorch tensors.
                    )
      
      input_ids.append(encoded_dict['input_ids'])
      attention_masks.append(encoded_dict['attention_mask'])

  # Convert the lists into tensors.
  input_ids = torch.cat(input_ids, dim=0)
  attention_masks = torch.cat(attention_masks, dim=0)
  labels = torch.tensor(labels)
  dataset = TensorDataset(input_ids, attention_masks, labels)
  return dataset
BATCH_SIZE = 8

train_dataset = tensor_dataset(train_sentences, train_labels)
test_dataset = tensor_dataset(test_sentences, test_labels)
dev_dataset = tensor_dataset(dev_sentences, dev_labels)

from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
# Create the DataLoaders for train/dev/test sets.
train_dataloader = DataLoader(train_dataset, sampler = RandomSampler(train_dataset), batch_size = BATCH_SIZE)
validation_dataloader = DataLoader(dev_dataset, sampler = SequentialSampler(dev_dataset), batch_size = BATCH_SIZE)
test_dataloader = DataLoader(test_dataset, sampler = SequentialSampler(test_dataset), batch_size = BATCH_SIZE)
# Load model with a sequence classification head
model = BertForSequenceClassification.from_pretrained(
    "dkleczek/bert-base-polish-uncased-v1", # Polbert - Polish BERT by <NAME>: https://github.com/kldarek/polbert
    num_labels = 3,
    output_attentions = False,
    output_hidden_states = False,
)
model.cuda()
import time, datetime
import numpy as np
from tqdm import tqdm
from transformers.optimization import AdamW
from transformers import AutoModelForSequenceClassification
from transformers import get_linear_schedule_with_warmup
from sklearn.metrics import f1_score


# Function to calculate the accuracy of our predictions vs labels
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)

# Takes a time in seconds and returns a string hh:mm:ss
def format_time(elapsed):
  elapsed_rounded = int(round((elapsed)))
  return str(datetime.timedelta(seconds=elapsed_rounded))

# Parameters:
epochs = 3
#lr = 1e-3 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
lr = 3e-5 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
adam_epsilon = 1e-8
WARM_UP = 0

optimizer = AdamW(model.parameters(), lr = lr, eps = adam_epsilon)

from transformers import get_linear_schedule_with_warmup
total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = WARM_UP, num_training_steps = total_steps)

train_loss_values = []
dev_acc_values = []

model.zero_grad()

t0 = time.time()
for epoch_i in range(0, epochs):  
  print("")
  print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
  print('Training...')

  # https://github.com/huggingface/transformers/blob/master/examples/run_glue.py
  # linie 168-183
  epoch_train_loss = 0 # Cumulative loss
  loss = 0 ;     batch_loss = 0
  model.train()

  for step, batch in enumerate(train_dataloader):

    # Progress update every 40 batches.
    if step % 40 == 0 and not step == 0:
      # Calculate elapsed time in minutes.
      elapsed = format_time(time.time() - t0)      
      # Report progress.
      print('  Batch {:>5,}  of  {:>5,}. Loss: {:.3f}  Elapsed: {:}.'.format(step, len(train_dataloader), batch_loss, elapsed))
    

    batch_loss = 0
    b_input_ids = batch[0].to(device)
    b_input_mask = batch[1].to(device)
    b_labels = batch[2].to(device)    

    # clear any previously calculated gradients before backward pass
    optimizer.zero_grad()

    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)

    loss = outputs[0]
    epoch_train_loss += loss.item()
    batch_loss += loss.item()
    loss.backward()

    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)  # Gradient clipping is not in AdamW anymore (so you can use amp without issue)
    optimizer.step()
    scheduler.step()  # Update learning rate schedule

  epoch_train_loss = epoch_train_loss / len(train_dataloader)          
  train_loss_values.append(epoch_train_loss)
  
  print('Average training loss: {0:.2f}'.format(epoch_train_loss))

  # Evaluation
  total_eval_accuracy = 0
  model.eval()

  for batch in validation_dataloader:
    
    input_ids = batch[0].to(device)
    attention_masks = batch[1].to(device)
    labels = batch[2].to('cpu').numpy()
                
    with torch.no_grad():
        outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)

    logits = outputs[0]
    logits = logits.detach().cpu().numpy()

    predictions = np.argmax(logits, axis=1).flatten()
    total_eval_accuracy += flat_accuracy(logits, labels)

  avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
  print("  Accuracy: {0:.4f}".format(avg_val_accuracy))

predicted_labels = [] ; true_labels = []; logits_list = []

for batch in test_dataloader:
  
  input_ids = batch[0].to(device)
  attention_masks = batch[1].to(device)
  labels = batch[2]
  
  with torch.no_grad():        
      outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)
                  
  logits = outputs[0]
  logits = logits.detach().cpu().numpy()
  logits_list.append(logits)
  
  predictions = np.argmax(logits, axis=1).flatten()
  labels = labels.numpy().flatten()

  predicted_labels.extend( predictions )
  true_labels.extend( labels )
  
# Parameters:
#epochs = 2
#lr = 3e-5 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
#adam_epsilon = 1e-8
#WARM_UP = 0
#87 86 87
from sklearn.metrics import classification_report 
print( classification_report(y_true=true_labels, y_pred=predicted_labels, zero_division=0) )
import os
from transformers import WEIGHTS_NAME, CONFIG_NAME
output_dir = "/content/drive/My Drive/model_bert_finetuned_1_3"

# Step 1: Save a model, configuration and vocabulary that you have fine-tuned
# If we have a distributed model, save only the encapsulated model
# (it was wrapped in PyTorch DistributedDataParallel or DataParallel)

model_to_save = model.module if hasattr(model, 'module') else model
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(output_dir, CONFIG_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_pretrained(output_dir)
# Step 2: Re-load the saved model and vocabulary
model = BertForSequenceClassification.from_pretrained(output_dir)
tokenizer = BertTokenizer.from_pretrained(output_dir)<jupyter_output><empty_output><jupyter_text>**Reviews**<jupyter_code>import os

TASK = 'SDS-R'

for set_ in get_sets(TASK):
  os.system('wget https://wothub-data.s3.amazonaws.com/Corpus/%s -nc'%set_)
train_path, dev_path, test_path = get_sets(TASK)
# Load data as pandas dataframes with two columns -- sentences and labels
train_data = pd.read_csv(train_path, sep="__label__", header=None, names=["text", "label"], engine="python")
dev_data = pd.read_csv(dev_path, sep="__label__", header=None, names=["text", "label"], engine="python")
test_data = pd.read_csv(test_path, sep="__label__", header=None, names=["text", "label"], engine="python")
train_data = train_data[train_data.label != 'z_amb']
dev_data = dev_data[dev_data.label != 'z_amb']
test_data = test_data[test_data.label != 'z_amb']
# Convert to numpy arrays
train_sentences, dev_sentences, test_sentences = [data.iloc[:,0] for data in (train_data, dev_data, test_data)]
train_labels, dev_labels, test_labels = [data.iloc[:,1] for data in (train_data, dev_data, test_data)]
# Check for errors in data labeling, removing nans
def remove_nulls(sentences, labels):
  lab = pd.Series(labels)
  sen = pd.Series(sentences)
  lab_nuls = pd.isnull(lab)
  sen_nuls = pd.isnull(sen)
  not_nuls = ~(lab_nuls | sen_nuls)
  lab = lab.loc[not_nuls].to_numpy()
  sen = sen.loc[not_nuls].to_numpy()
  return [sen, lab]
train_sentences, train_labels = remove_nulls(train_sentences, train_labels)
dev_sentences, dev_labels = remove_nulls(dev_sentences, dev_labels)
test_sentences, test_labels = remove_nulls(test_sentences, test_labels)
np.unique(np.concatenate((train_labels, dev_labels, test_labels)), return_counts=True)
len(train_sentences)  * (1-0.875)
from sklearn.preprocessing import LabelEncoder
# Encoder for labels
labelencoder = LabelEncoder()
train_labels = labelencoder.fit_transform(train_labels)
test_labels = labelencoder.transform(test_labels)
dev_labels = labelencoder.transform(dev_labels)
labelencoder.classes_
dev_sentences = np.append(dev_sentences, train_sentences[:160])
dev_labels = np.append(dev_labels, train_labels[:160])
train_sentences = train_sentences[160:]
train_labels = train_labels[160:]
print('Train data')
print(len(train_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
print('Dev data')
print(len(dev_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
print('Test data')
print(len(test_sentences) / ( len(train_sentences) + len(dev_sentences) + len(test_sentences) ))
dev_polemo_data = pd.DataFrame([dev_sentences,dev_labels]).T
test_polemo_data = pd.DataFrame([test_sentences,test_labels]).T
dev_polemo_data.to_csv('/content/drive/My Drive/dev_polemo_reviews_data_preprocessed.csv')
test_polemo_data.to_csv('/content/drive/My Drive/test_polemo_reviews_data_preprocessed.csv')
from sklearn.model_selection import StratifiedShuffleSplit

X = train_sentences
y = train_labels
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in sss.split(X, y):
  train_sentences, test_sentences = X[train_index], X[test_index]
  train_labels, test_labels = y[train_index], y[test_index]

X = test_sentences
y = test_labels
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=42)
for train_index, test_index in sss.split(X, y):
  dev_sentences, test_sentences = X[train_index], X[test_index]
  dev_labels, test_labels = y[train_index], y[test_index]
from sklearn.preprocessing import LabelEncoder
# Encoder for labels
labelencoder = LabelEncoder()
train_labels = labelencoder.fit_transform(train_labels)
test_labels = labelencoder.transform(test_labels)
dev_labels = labelencoder.transform(dev_labels)
# Remove long sentences.
# TO-DO Possible cut?
def remove_big(sentences, labels):
  to_remove = []
  for i, sent in enumerate(sentences):
      input_ids = tokenizer.encode(sent, add_special_tokens=True) # TO-DO: add_special_tokens
      if len(input_ids) > MAX_LEN:
        to_remove.append(i)

  sentences = np.delete(sentences, to_remove)
  labels = np.delete(labels, to_remove) 

  print('{} samples removed.'.format(len(to_remove)))

  return sentences, labels
# Downloading tokenizer
# From Polbert - Polish BERT by <NAME>: https://github.com/kldarek/polbert
tokenizer = BertTokenizer.from_pretrained("dkleczek/bert-base-polish-uncased-v1")
MAX_LEN = 128

train_sentences, train_labels = remove_big(train_sentences, train_labels)
test_sentences, test_labels = remove_big(test_sentences, test_labels)
dev_sentences, dev_labels = remove_big(dev_sentences, dev_labels)
from torch.utils.data import TensorDataset
# Create TensorDatasets for train/dev/test sets
def tensor_dataset(sentences, labels):
  input_ids = []
  attention_masks = []

  for sent in sentences:
      encoded_dict = tokenizer.encode_plus(
                          sent,                     
                          add_special_tokens = True,
                          max_length = MAX_LEN,
                          pad_to_max_length = True,
                          return_attention_mask = True,
                          return_tensors = 'pt',     # Return pytorch tensors.
                    )
      
      input_ids.append(encoded_dict['input_ids'])
      attention_masks.append(encoded_dict['attention_mask'])

  # Convert the lists into tensors.
  input_ids = torch.cat(input_ids, dim=0)
  attention_masks = torch.cat(attention_masks, dim=0)
  labels = torch.tensor(labels)
  dataset = TensorDataset(input_ids, attention_masks, labels)
  return dataset
BATCH_SIZE = 8

train_dataset = tensor_dataset(train_sentences, train_labels)
test_dataset = tensor_dataset(test_sentences, test_labels)
dev_dataset = tensor_dataset(dev_sentences, dev_labels)

from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
# Create the DataLoaders for train/dev/test sets.
train_dataloader = DataLoader(train_dataset, sampler = RandomSampler(train_dataset), batch_size = BATCH_SIZE)
validation_dataloader = DataLoader(dev_dataset, sampler = SequentialSampler(dev_dataset), batch_size = BATCH_SIZE)
test_dataloader = DataLoader(test_dataset, sampler = SequentialSampler(test_dataset), batch_size = BATCH_SIZE)
# Load model with a sequence classification head
model = BertForSequenceClassification.from_pretrained(
    "dkleczek/bert-base-polish-uncased-v1", # Polbert - Polish BERT by <NAME>: https://github.com/kldarek/polbert
    num_labels = 3,
    output_attentions = False,
    output_hidden_states = False,
)
model.cuda()
import time, datetime
import numpy as np
from tqdm import tqdm
from transformers.optimization import AdamW
from transformers import AutoModelForSequenceClassification
from transformers import get_linear_schedule_with_warmup
from sklearn.metrics import f1_score


# Function to calculate the accuracy of our predictions vs labels
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)

# Takes a time in seconds and returns a string hh:mm:ss
def format_time(elapsed):
  elapsed_rounded = int(round((elapsed)))
  return str(datetime.timedelta(seconds=elapsed_rounded))

# Parameters:
epochs = 3
#lr = 1e-3 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
lr = 3e-5 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
adam_epsilon = 1e-8
WARM_UP = 0

optimizer = AdamW(model.parameters(), lr = lr, eps = adam_epsilon)

from transformers import get_linear_schedule_with_warmup
total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = WARM_UP, num_training_steps = total_steps)

train_loss_values = []
dev_acc_values = []

model.zero_grad()

t0 = time.time()
for epoch_i in range(0, epochs):  
  print("")
  print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
  print('Training...')

  # https://github.com/huggingface/transformers/blob/master/examples/run_glue.py
  # linie 168-183
  epoch_train_loss = 0 # Cumulative loss
  loss = 0 ;     batch_loss = 0
  model.train()

  for step, batch in enumerate(train_dataloader):

    # Progress update every 40 batches.
    if step % 40 == 0 and not step == 0:
      # Calculate elapsed time in minutes.
      elapsed = format_time(time.time() - t0)      
      # Report progress.
      print('  Batch {:>5,}  of  {:>5,}. Loss: {:.3f}  Elapsed: {:}.'.format(step, len(train_dataloader), batch_loss, elapsed))
    

    batch_loss = 0
    b_input_ids = batch[0].to(device)
    b_input_mask = batch[1].to(device)
    b_labels = batch[2].to(device)    

    # clear any previously calculated gradients before backward pass
    optimizer.zero_grad()

    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)

    loss = outputs[0]
    epoch_train_loss += loss.item()
    batch_loss += loss.item()
    loss.backward()

    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)  # Gradient clipping is not in AdamW anymore (so you can use amp without issue)
    optimizer.step()
    scheduler.step()  # Update learning rate schedule

  epoch_train_loss = epoch_train_loss / len(train_dataloader)          
  train_loss_values.append(epoch_train_loss)
  
  print('Average training loss: {0:.2f}'.format(epoch_train_loss))

  # Evaluation
  total_eval_accuracy = 0
  model.eval()

  for batch in validation_dataloader:
    
    input_ids = batch[0].to(device)
    attention_masks = batch[1].to(device)
    labels = batch[2].to('cpu').numpy()
                
    with torch.no_grad():
        outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)

    logits = outputs[0]
    logits = logits.detach().cpu().numpy()

    predictions = np.argmax(logits, axis=1).flatten()
    total_eval_accuracy += flat_accuracy(logits, labels)

  avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
  print("  Accuracy: {0:.4f}".format(avg_val_accuracy))

predicted_labels = [] ; true_labels = []; logits_list = []

for batch in test_dataloader:
  
  input_ids = batch[0].to(device)
  attention_masks = batch[1].to(device)
  labels = batch[2]
  
  with torch.no_grad():        
      outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)
                  
  logits = outputs[0]
  logits = logits.detach().cpu().numpy()
  logits_list.append(logits)
  
  predictions = np.argmax(logits, axis=1).flatten()
  labels = labels.numpy().flatten()

  predicted_labels.extend( predictions )
  true_labels.extend( labels )
  
# Parameters:
#epochs = 2
#lr = 3e-5 # Learning rate (Adam): 5e-5, 3e-5, 2e-5
#adam_epsilon = 1e-8
#WARM_UP = 0
#87 86 87
from sklearn.metrics import classification_report 
print( classification_report(y_true=true_labels, y_pred=predicted_labels, zero_division=0) )
import os
from transformers import WEIGHTS_NAME, CONFIG_NAME
output_dir = "/content/drive/My Drive/model_bert_finetuned_1_4"

# Step 1: Save a model, configuration and vocabulary that you have fine-tuned
# If we have a distributed model, save only the encapsulated model
# (it was wrapped in PyTorch DistributedDataParallel or DataParallel)

model_to_save = model.module if hasattr(model, 'module') else model
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(output_dir, CONFIG_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_pretrained(output_dir)
# Step 2: Re-load the saved model and vocabulary
model = BertForSequenceClassification.from_pretrained(output_dir)
tokenizer = BertTokenizer.from_pretrained(output_dir)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10978964743725399, 'fraction_numerical': 0.011067846108265967, 'mean_word_length': 4.935812083909057, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 37, 'https://': 16, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '39181', 'license': 'MIT', 'n_tokens_mistral': 16436, 'n_tokens_neox': 15881, 'n_words': 3381, 'path': 'polemo_base_classifiers.ipynb', 'repo_name': 'antoni-jamiolkowski/Bachelor-thesis', 'size': '47310'}