<jupyter_start><jupyter_text>**Baselines**<jupyter_code>import numpy as np
import pandas as pd
from transformers import *
import torch
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader, SequentialSampler
from sklearn.metrics import classification_report 

MAX_LEN = 128
BATCH_SIZE = 8

def baseline_experiment(
    test_data_path,
    model_path,
    dropnas=True):
  
  def tensor_dataset(sentences, labels, tokenizer):
    input_ids = []
    attention_masks = []

    for sent in sentences:
        encoded_dict = tokenizer.encode_plus(
                            sent,                     
                            add_special_tokens = True,
                            max_length = MAX_LEN,
                            pad_to_max_length = True,
                            return_attention_mask = True,
                            return_tensors = 'pt',     # Return pytorch tensors.
                      )
        
        input_ids.append(encoded_dict['input_ids'])
        attention_masks.append(encoded_dict['attention_mask'])

    # Convert the lists into tensors.
    input_ids = torch.cat(input_ids, dim=0)
    attention_masks = torch.cat(attention_masks, dim=0)
    labels = torch.tensor(labels)
    dataset = TensorDataset(input_ids, attention_masks, labels)
    return dataset
  
  def BERT_inference(model, dataloader):
    predicted_labels = []; true_labels = []
    model.to(device)
    for batch in dataloader:
      
      input_ids = batch[0].to(device)
      attention_masks = batch[1].to(device)
      labels = batch[2]
      
      with torch.no_grad():        
          outputs = model(input_ids, token_type_ids=None, attention_mask=attention_masks)
                      
      logits = outputs[0]
      logits = logits.detach().cpu().numpy()

      predictions = np.argmax(logits, axis=1).flatten()
      labels = labels.numpy().flatten()

      predicted_labels.extend(predictions)
      true_labels.extend(labels)

    return np.array(predicted_labels), np.array(true_labels)

  
  if torch.cuda.is_available():    

      # Tell PyTorch to use the GPU.    
      device = torch.device("cuda")

      print('There are %d GPU(s) available.' % torch.cuda.device_count())

      print('We will use the GPU:', torch.cuda.get_device_name(0))

  else:
      print('No GPU available, using the CPU instead.')
      device = torch.device("cpu")

  device = torch.device("cuda")

  test_data = pd.read_csv(test_data_path, index_col=0)
  print("Loaded data ...")

  if dropnas:
      test_data = test_data.dropna()
      print("Dropped NAN's ...")

  tokenizer = BertTokenizer.from_pretrained(model_path)
  print("Tokenizer loaded!")
  
  model = BertForSequenceClassification.from_pretrained(model_path)
  print("Model loaded!")

  print("Data set preparation ...")
  test_dataset = tensor_dataset(test_data.iloc[:,0].to_numpy(), test_data.iloc[:,1].to_numpy(), tokenizer)
  test_dataloader = DataLoader(test_dataset, sampler = SequentialSampler(test_dataset), batch_size = BATCH_SIZE)
  print("Data set preparation end!")

  print("Model inference ...")
  model_output, test_labels = BERT_inference(model, test_dataloader)
  print("Model inference end!")
  
  print(classification_report(y_true=test_labels, y_pred=model_output, zero_division=0))
baseline_experiment(
    test_data_path = '/content/drive/My Drive/test_moto_data_preprocessed.csv',
    model_path = "/content/drive/My Drive/model_bert_finetuned_2"
)
baseline_experiment(
    test_data_path =  '/content/drive/My Drive/test_treebank_data_preprocessed.csv',
    model_path = "/content/drive/My Drive/model_bert_finetuned_4"
)
baseline_experiment(
    test_data_path = '/content/drive/My Drive/test_polemo_hotels_data_preprocessed.csv',
    model_path = "/content/drive/My Drive/model_bert_finetuned_1_1"
)
baseline_experiment(
    test_data_path = '/content/drive/My Drive/test_polemo_medicine_data_preprocessed.csv',
    model_path = "/content/drive/My Drive/model_bert_finetuned_1_2"
)
baseline_experiment(
    test_data_path = '/content/drive/My Drive/test_polemo_products_data_preprocessed.csv',
    model_path = "/content/drive/My Drive/model_bert_finetuned_1_3"
)
baseline_experiment(
    test_data_path = '/content/drive/My Drive/test_polemo_reviews_data_preprocessed.csv',
    model_path = "/content/drive/My Drive/model_bert_finetuned_1_4"
)
baseline_experiment(
    test_data_path = '/content/drive/My Drive/test_filmweb_data_preprocessed.csv',
    model_path = "/content/drive/My Drive/model_bert_finetuned_3"
)
# Accuracy
baseline_moto = 0.72
baseline_tw = 0.75
baseline_polemo_hotels = 0.9
baseline_polemo_medicine = 0.83
baseline_polemo_products = 0.85
baseline_polemo_reviews = 0.76
baseline_films = 0.75
# Accuracy
metaSVM_moto = 0.7
metaSVM_tw = 0.71
metaSVM_polemo_hotels = 0.89
metaSVM_polemo_medicine = 0.83
metaSVM_polemo_products = 0.86
metaSVM_polemo_reviews = 0.77
metaSVM_films = 0.7
# Accuracy
metaxgboost_moto = 0.71
metaxgboost_tw = 0.72
metaxgboost_polemo_hotels = 0.89
metaxgboost_polemo_medicine = 0.84
metaxgboost_polemo_products = 0.86
metaxgboost_polemo_reviews = 0.78
metaxgboost_films = 0.68
# Accuracy
min_moto, max_moto, median_moto = 0.46, 0.49, 0.56
min_tw, max_tw, median_tw = 0.38, 0.39, 0.4
min_polemo_hotels, max_polemo_hotels, median_polemo_hotels = 0.85, 0.86, 0.79
min_polemo_medicine, max_polemo_medicine, median_polemo_medicine = 0.79, 0.81, 0.77
min_polemo_products, max_polemo_products, median_polemo_products = 0.76, 0.70, 0.66
min_polemo_reviews, max_polemo_reviews, median_polemo_reviews = 0.62, 0.64, 0.59
min_films, max_films, median_films = 0.61, 0.59, 0.63
# Accuracy
hard_vote_moto = 0.57
hard_vote_tw = 0.37
hard_vote_polemo_hotels = 0.79
hard_vote_polemo_medicine = 0.76
hard_vote_polemo_products = 0.68
hard_vote_polemo_reviews = 0.64
hard_vote_films = 0.64
# Accuracy
soft_vote_moto = 0.55
soft_vote_tw = 0.39
soft_vote_polemo_hotels = 0.83
soft_vote_polemo_medicine = 0.79
soft_vote_polemo_products = 0.75
soft_vote_polemo_reviews = 0.64
soft_vote_films = 0.64

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
font = {'weight' : 'bold',
        'size'   : 14}

matplotlib.rc('font', **font)

labels = ['Motofakty', 'TW', 'PolEmo Hotels', 'PolEmo Medicine', 'PolEmo Products', 'PolEmo Reviews', 'Films']
baseline = [
        baseline_moto,
        baseline_tw,
        baseline_polemo_hotels,
        baseline_polemo_medicine,
        baseline_polemo_products,
        baseline_polemo_reviews,
        baseline_films 
]
metaSVM = [
          metaSVM_moto,
          metaSVM_tw,
          metaSVM_polemo_hotels,
          metaSVM_polemo_medicine,
          metaSVM_polemo_products,
          metaSVM_polemo_reviews,
          metaSVM_films
]
metaxgboost = [
              metaxgboost_moto,
              metaxgboost_tw,
              metaxgboost_polemo_hotels,
              metaxgboost_polemo_medicine,
              metaxgboost_polemo_products,
              metaxgboost_polemo_reviews,
              metaxgboost_films
]

x = np.arange(len(labels))  # the label locations
width = 0.3  # the width of the bars
fig, ax = plt.subplots(figsize=(20, 10))
plt.ylim(ymax = 0.95, ymin = 0.65)
rects1 = ax.bar(x - width, baseline, width, label='Baseline')
rects2 = ax.bar(x, metaSVM, width, label='SVM ML')
rects3 = ax.bar(x + width, metaxgboost, width, label='XGBoost ML')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('Accuracy')
ax.set_title('Scores by data set and method')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.legend()


def autolabel(rects):
    """Attach a text label above each bar in *rects*, displaying its height."""
    for rect in rects:
        height = rect.get_height()
        ax.annotate('{}'.format(height),
                    xy=(rect.get_x() + rect.get_width() / 2, height),
                    xytext=(0, 3),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom')


autolabel(rects1)
autolabel(rects2)
autolabel(rects3)

fig.tight_layout()

plt.show()
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
font = {'weight' : 'bold',
        'size'   : 14}

matplotlib.rc('font', **font)

labels = [
          'Motofakty', 'TW', 
          #'PolEmo Hotels', 'PolEmo Medicine', 'PolEmo Products', 'PolEmo Reviews', 
          'Films'
          ]
baseline = [
        baseline_moto,
        baseline_tw,
        #baseline_polemo_hotels,
        #baseline_polemo_medicine,
        #baseline_polemo_products,
        #baseline_polemo_reviews,
        baseline_films 
]
min = [
      #min_moto, 
      #min_tw, 
      min_polemo_hotels, 
      min_polemo_medicine,
      min_polemo_products, 
      min_polemo_reviews, 
      #min_films
]
max = [
      #max_moto,
      #max_tw,
      max_polemo_hotels,
      max_polemo_medicine,
      max_polemo_products,
      max_polemo_reviews,
      #max_films
]
median = [
        #median_moto,
        #median_tw,
        median_polemo_hotels, 
        median_polemo_medicine,
        median_polemo_products,
        median_polemo_reviews,
        #median_films 
]
hard_vote = [
            hard_vote_moto,
            hard_vote_tw,
            #hard_vote_polemo_hotels,
            #hard_vote_polemo_medicine,
            #hard_vote_polemo_products,
            #hard_vote_polemo_reviews,
            hard_vote_films
]
soft_vote = [
            soft_vote_moto,
            soft_vote_tw,
            #soft_vote_polemo_hotels,
            #soft_vote_polemo_medicine,
            #soft_vote_polemo_products,
            #soft_vote_polemo_reviews,
            soft_vote_films
]

x = np.arange(len(labels))  # the label locations
width = 0.2  # the width of the bars
fig, ax = plt.subplots(figsize=(20, 10))
plt.ylim(ymax = 0.95, ymin = 0.35)
rects1 = ax.bar(x - width, baseline, width, label='Baseline')
rects2 = ax.bar(x, hard_vote, width, label='Hard voting')
rects3 = ax.bar(x + width, soft_vote, width, label='Soft voting')
#rects4 = ax.bar(x + 1.5*width, median, width, label='Median aggregation')
#rects5 = ax.bar(x + 1.5*width, hard_vote, width, label='Hard voting')
#rects6 = ax.bar(x + 3*width, soft_vote, width, label='Soft voting')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('Accuracy')
ax.set_title('Scores by data set and method')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.legend()


def autolabel(rects):
    """Attach a text label above each bar in *rects*, displaying its height."""
    for rect in rects:
        height = rect.get_height()
        ax.annotate('{}'.format(height),
                    xy=(rect.get_x() + rect.get_width() / 2, height),
                    xytext=(0, 3),  # 3 points vertical offset
                    textcoords="offset points",
                    ha='center', va='bottom')


autolabel(rects1)
autolabel(rects2)
autolabel(rects3)
#autolabel(rects4)
#autolabel(rects5)
#autolabel(rects6)

fig.tight_layout()

plt.show()
from matplotlib import pyplot as plt


def bar_plot(ax, data, colors=None, total_width=0.8, single_width=1, legend=True, labels=None):
    """Draws a bar plot with multiple bars per data point.

    Parameters
    ----------
    ax : matplotlib.pyplot.axis
        The axis we want to draw our plot on.

    data: dictionary
        A dictionary containing the data we want to plot. Keys are the names of the
        data, the items is a list of the values.

        Example:
        data = {
            "x":[1,2,3],
            "y":[1,2,3],
            "z":[1,2,3],
        }

    colors : array-like, optional
        A list of colors which are used for the bars. If None, the colors
        will be the standard matplotlib color cyle. (default: None)

    total_width : float, optional, default: 0.8
        The width of a bar group. 0.8 means that 80% of the x-axis is covered
        by bars and 20% will be spaces between the bars.

    single_width: float, optional, default: 1
        The relative width of a single bar within a group. 1 means the bars
        will touch eachother within a group, values less than 1 will make
        these bars thinner.

    legend: bool, optional, default: True
        If this is set to true, a legend will be added to the axis.
    """

    # Check if colors where provided, otherwhise use the default color cycle
    if colors is None:
        colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

    # Number of bars per group
    n_bars = len(data)

    # The width of a single bar
    bar_width = total_width / n_bars

    # List containing handles for the drawn bars, used for the legend
    bars = []

    # Iterate over all data
    for i, (name, values) in enumerate(data.items()):
        # The offset in x direction of that bar
        x_offset = (i - n_bars / 2) * bar_width + bar_width / 2

        # Draw a bar for every value of that type
        for x, y in enumerate(values):
            bar = ax.bar(x + x_offset, y, width=bar_width * single_width, color=colors[i % len(colors)])

        # Add a handle to the last drawn bar, which we'll need for the legend
        bars.append(bar[0])
        
    # Draw legend if we need
    if legend:
        ax.legend(bars, data.keys())
        ax.set_xticklabels(labels)
baseline = [
        baseline_moto,
        baseline_tw,
        baseline_polemo_hotels,
        baseline_polemo_medicine,
        baseline_polemo_products,
        baseline_polemo_reviews,
        baseline_films 
]
metaSVM = [
          metaSVM_moto,
          metaSVM_tw,
          metaSVM_polemo_hotels,
          metaSVM_polemo_medicine,
          metaSVM_polemo_products,
          metaSVM_polemo_reviews,
          metaSVM_films
]
metaxgboost = [
              metaxgboost_moto,
              metaxgboost_tw,
              metaxgboost_polemo_hotels,
              metaxgboost_polemo_medicine,
              metaxgboost_polemo_products,
              metaxgboost_polemo_reviews,
              metaxgboost_films
]
min = [
      min_moto, 
      min_tw, 
      min_polemo_hotels, 
      min_medicine,
      min_products, 
      min_reviews, 
      min_films
]
max = [
      max_moto,
      max_tw,
      max_polemo_hotels,
      max_medicine,
      max_products,
      max_reviews,
      max_films
]
median = [
        median_moto,
        median_tw,
        median_polemo_hotels, 
        median_medicine,
        median_products,
        median_reviews,
        median_films 
]
hard_vote = [
            hard_vote_moto,
            hard_vote_tw,
            hard_vote_polemo_hotels,
            hard_vote_polemo_medicine,
            hard_vote_polemo_products,
            hard_vote_polemo_reviews,
            hard_vote_films
]
soft_vote = [
            soft_vote_moto,
            soft_vote_tw,
            soft_vote_polemo_hotels,
            soft_vote_polemo_medicine,
            soft_vote_polemo_products,
            soft_vote_polemo_reviews,
            soft_vote_films
]
labels = ['Motofakty', 'TW', 'PolEmo Hotels', 'PolEmo Medicine', 'PolEmo Products', 'PolEmo Reviews', 'Films']
data = {
        'Baseline': baseline, 
        'SVM ML': metaSVM,
        'XGBoost ML': metaxgboost,
        #'Min aggregation': min,
        #'Max aggregation': max,
        #'Median aggregation': median,
        #'Hard voting': hard_vote,
        #'Soft voting': soft_vote
    }

fig, ax = plt.subplots(figsize=(20, 10))
plt.ylim(ymax = 0.95, ymin = 0.65)
bar_plot(ax, data, total_width=.8, single_width=.9, labels = labels)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09710610932475884, 'fraction_numerical': 0.01852090032154341, 'mean_word_length': 2.7535602220613082, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '459494', 'license': 'MIT', 'n_tokens_mistral': 5710, 'n_tokens_neox': 5237, 'n_words': 1131, 'path': 'main_experiments.ipynb', 'repo_name': 'antoni-jamiolkowski/Bachelor-thesis', 'size': '15550'}