<jupyter_start><jupyter_text><jupyter_code>import requests, zipfile, io, os
import bz2,shutil
import xml.etree.ElementTree as ET
import numpy as np
import pandas as pd

!pip install sentencepiece
!pip install transformers
from transformers import XLNetTokenizer
from transformers import BertTokenizer
import re
from sklearn.model_selection import train_test_split
import tensorflow
from tensorflow import keras
from tensorflow.keras import Sequential
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.layers import Embedding, LSTM, Dropout, Dense, SpatialDropout1D, Conv1D, MaxPooling1D
zip_file_url = "http://groups.di.unipi.it/~gulli/newsspace200.xml.bz"

filename = zip_file_url.split("/")[-1]
with open(filename, "wb") as f:
    r = requests.get(zip_file_url)
    f.write(r.content)
dirName = 'data'
try:
    # Create target Directory
    os.mkdir(dirName)
    print("Directory " , dirName ,  " Created ") 
except FileExistsError:
    print("Directory " , dirName ,  " already exists")
    
    
with bz2.BZ2File("newsspace200.xml.bz") as fr, open("./data/input.xml","wb") as fw:
    shutil.copyfileobj(fr,fw)
tree = ET.parse("./data/input.xml")
root = tree.getroot()

titles = []
categories = []
descriptions = []
sources = []


for title in tree.findall('title'):
    titles.append(title.text)
    
for category in tree.findall('category'):
        categories.append(category.text)

for description in tree.findall('description'):
    descriptions.append(description.text)

for source in tree.findall('source'):
    sources.append(source.text)
titles = np.array(titles).T
categories = np.array(categories).T
descriptions = np.array(descriptions).T
sources = np.array(sources).T
data = np.dstack((sources,titles,descriptions,categories)).reshape(len(titles),4)
df = pd.DataFrame(data)
df.columns = ['source','title', 'desc', 'cat']
def tr(bt,reg,title,sz):
  a = bt.tokenize(title + " " + sz)
  r = []
  for s in a:
    if reg.match(s) is not None:     
      r.append(s)      
  return r

def bertenc(bt,title,sz):
  text_to_encode = title + " " + sz
  a = bt.encode_plus(
            text_to_encode, 
            max_length= 64, 
            add_special_tokens=True,
            return_token_type_ids=False, 
            padding="max_length",
            truncation = True,
            return_attention_mask=False
        )
  ids = a['input_ids']  
  return ids 

def dict_creater(arr):
  for a in arr:
    if a != 0:
      if a in bertencidvoc_dict:
        count = bertencidvoc_dict[a] + 1
        bertencidvoc_dict.update({a:count})
      else:
        bertencidvoc_dict.update({a:1})
  return

def dict_creater2(arr):
  for a in arr:
    if a != 0:
      if a in bertvoc_dict:
        count = bertvoc_dict[a] + 1
        bertvoc_dict.update({a:count})
      else:
        bertvoc_dict.update({a:1})
  return

def create_model(i):
  loaded_model = keras.models.load_model("best_LSTM_CONV1D_model.h5")
  model = tensorflow.keras.Sequential([
    keras.layers.Embedding(
        input_dim = len(bertvoc_dict),
        output_dim = loaded_model.layers[0].output_dim,
        input_length = 64
    ),
    tensorflow.keras.layers.Conv1D(
        filters=loaded_model.layers[1].filters,
        kernel_size=loaded_model.layers[1].kernel_size,
        padding='same',
        activation='relu'
    ),
    tensorflow.keras.layers.Dropout(
        rate = loaded_model.layers[2].rate
    ),
    tensorflow.keras.layers.LSTM(
        units=loaded_model.layers[3].units
    ),
    tensorflow.keras.layers.Dropout(
        rate = loaded_model.layers[4].rate
    ),
    tensorflow.keras.layers.Dense(
        units = loaded_model.layers[5].units,
        activation = 'relu'
    ),
    tensorflow.keras.layers.Dense(
        units = i,
        activation = 'softmax'
    )
  ])

  model.compile(optimizer=tensorflow.keras.optimizers.Adam(learning_rate=0.001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])
  
  return model
  

res = list()
for i in range(5,11):
  #select top i classes
  selected_cats = df['cat'].value_counts()[:i].index.tolist()
  df_selected = df.loc[df['cat'].isin(selected_cats)]
  #drop empty desc rows
  df_selected = df_selected.dropna()
  #load tokenizer
  regex = re.compile('^[a-zA-Z]{3,}$')
  xltokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased")
  btokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
  #tokenize
  df_selected['berttoken'] = df_selected.apply(lambda row: tr(btokenizer, regex, str(row['title']), str(row['desc'])), axis=1)
  df_selected['bertencodeplus'] = df_selected.apply(lambda row: bertenc(btokenizer, str(row['title']), str(row['desc'])), axis=1)
  #voc creation
  bertencidvoc_dict = {}
  bertvoc_dict = {}
  df_selected.apply(lambda row: dict_creater(row['bertencodeplus']), axis=1)
  df_selected.apply(lambda row: dict_creater2(row['berttoken']), axis=1)
  #label to numeric
  c = 0
  category_dictionary = {}
  cat_rev_dict= {}
  for s in selected_cats:
    category_dictionary.update({c:s})
    cat_rev_dict.update({s:i})
    c = c + 1
  df_selected['cat_enc'] = df_selected.apply(lambda row: cat_rev_dict[str(row['cat'])] , axis=1)
  #split data to X and y
  X = np.array(df_selected["bertencodeplus"].values.tolist())
  y = df_selected['cat_enc'].to_numpy()
  #split data to train, test, valid
  X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle = True)
  X_test, X_valid, y_test, y_valid = train_test_split(X_test, y_test, test_size=0.5, random_state=42, shuffle = True)
  #cat to onehotencode
  Y_train = to_categorical(y_train, num_classes=5, dtype='float32')
  Y_test = to_categorical(y_test, num_classes=5, dtype='float32')
  Y_valid = to_categorical(y_valid, num_classes=5, dtype='float32')       
  #load model
  model = create_model(i)
  #fit model
  print('Model for {0} classes started to fit'.format(i))
  model.fit(X_train,Y_train, epochs=10, validation_data=(X_valid,Y_valid))
  #evaluate model
  results = model.evaluate(X_test,Y_test)
  res.append(i,results)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13042060645582002, 'fraction_numerical': 0.011574828822954027, 'mean_word_length': 3.785491419656786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '509380', 'license': 'Unlicense', 'n_tokens_mistral': 2231, 'n_tokens_neox': 2173, 'n_words': 415, 'path': 'Different_class_number_testing.ipynb', 'repo_name': 'krisivarga/deep_learning_HW_big', 'size': '6134'}