<jupyter_start><jupyter_text>First enable parallel execution using TPU cores<jupyter_code>import tensorflow as tf
use_tpu = True
if use_tpu:
    # Create distribution strategy
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
    strategy = tf.distribute.TPUStrategy(tpu)<jupyter_output>INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.<jupyter_text>Import needed libs<jupyter_code>import numpy as np
import pandas as pd
import math, copy, time
import pickle 

import nltk
from nltk import word_tokenize
from nltk.corpus import stopwords

nltk.download('stopwords')
nltk.download('punkt')

from tqdm import trange
import random
import re

import tensorflow.keras.backend as K
from tensorflow.keras.layers import Layer
from tensorflow.keras import layers
from tensorflow.keras.callbacks import Callback
import keras

np.random.seed(0)<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Package punkt is already up-to-date!<jupyter_text>Mount dataset<jupyter_code>from google.colab import drive
drive.mount('/content/drive')

# special to google colab | put your 4 Datasets into this path 
folder_path = 'drive/MyDrive/logsy_data/dataset'<jupyter_output>Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).<jupyter_text>Read in the datasets using preprocessor<jupyter_code>"""
Standard tokenizer + do what the paper says
"""
"""
Do not run this script in Windows, it will not work due to use of signal

We also use authors preprocessor to evaluate our model just in case we made something wrong here
"""
import os
import pickle
import re

import signal


class TimeoutException(Exception):  # Custom exception class
    pass


def timeout_handler(signum, frame):  # Custom signal handler
    raise TimeoutException


# Change the behavior of SIGALRM
signal.signal(signal.SIGALRM, timeout_handler)

import numpy as np
from tqdm.auto import tqdm
class DataTokenizer:
    def __init__(self):
        self.word2index = {'[PAD]': 0, '[CLS]': 1, '[MASK]': 2}
        self.num_words = 3
        self.stop_words = set(stopwords.words('english'))

    def tokenize(self, message):
        # paper section IV: Tokenization processing
        message = message.lower()
        message = re.sub(r'/.*:', '', message, flags=re.MULTILINE)  # filter for endpoints
        message = re.sub(r'/.*', '', message, flags=re.MULTILINE)
        message = word_tokenize(message)  # remove non words
        message = (word for word in message if word.isalpha())  # generator  # remove numerical
        message = [word for word in message if word not in self.stop_words]  # remove nltk common stopwords
        message = ['[CLS]'] + message  # add embedding token
        for word_idx, word in enumerate(message):  # convert to value
            if word not in self.word2index:
                self.word2index[word] = self.num_words
                self.num_words += 1
            message[word_idx] = self.word2index[word]
        return message
tokenizer = DataTokenizer()


class DataImporter:
    """
    loads data set from the raw dataset
    """

    def __init__(self, log_template, dataset_folder_path, dataset_name, dataset_step=1,
                 dataset_limit=100000, dataset_type='main', normal_indicator: str = '-', aux_count=50000,
                 chunk: bool = True):
        self.log_template = log_template  # a template containing <Token{n}> and <Message>
        self.log_dataframe = None
        self.dataset_folder_path: str = dataset_folder_path  # path to the dataset folder
        self.dataset_name: str = dataset_name  # full name of raw dataset
        self.step: int = dataset_step  # step taken to sample auxiliary dataset
        self.log_template_regex: re = re.compile(r'')
        self.log_template_headers: list[str] = []
        self.limit: int = dataset_limit  # used for faster experiment only
        self.dataset_type: str = dataset_type
        self.normal_indicator: str = normal_indicator  # a sign indicating the log line is anomaly
        self.aux_count: int = aux_count
        self.chunk: bool = chunk

    def log_loader(self):
        """
        read from IO stream and only take the actual log message based on template
        :return:
        """
        log_messages = []
        true_labels = []
        with open(os.path.join(self.dataset_folder_path, self.dataset_name), 'r', encoding="latin-1") as ds:
            for line_no, line in enumerate(tqdm(ds, miniters=1)):
                if line_no != 1 and line_no % 20000000 == 1:  # 15 file chunks
                    with open(f'dataset/tbird/{self.dataset_name}_chunked_msg_line{line_no}', 'wb') as message_file:
                        # print(log_messages)
                        # log_messages_array = np.asanyarray(log_messages).reshape(-1, 1)
                        print(len(log_messages))
                        l = len(log_messages)
                        tokenized = [tokenizer.tokenize(log_message) for log_message in
                                     tqdm(log_messages, position=0, leave=True, total=l)]
                        tokenized_np = np.asanyarray(tokenized)
                        del tokenized
                        pickle.dump(tokenized_np, message_file)
                        log_messages = []  # reset
                # Start the timer. Once 10 seconds are over, a SIGALRM signal is sent.
                try:
                    try:
                        signal.alarm(30)

                        match = self.log_template_regex.search(line.strip())

                        if not match:
                            continue
                        label_decider = lambda x: 0 if x == self.normal_indicator else 1
                        true_labels.append(label_decider(match.group('Token0')))
                        # message = tokenizer.tokenize(match.group('Message'))
                        # log_messages.append(message)
                        log_messages.append(match.group('Message'))
                        # print('message after tokenize ', message, log_messages)
                        # print(self.log_template_headers)
                    except Exception as e:  # noqa
                        print(e)
                        # print(e) # will skip those without normal indications('-' OR 'warn')
                        pass
                except TimeoutException:
                    print("Regex hang detected, skipping")
                    continue  # catastrophic backtracking
                else:
                    signal.alarm(0)
                if line_no == self.limit:
                    break
        return log_messages, np.array(true_labels)  # remaining log_messages and all of labels

    def load(self):
        self.log_template_matcher()

        self.log_dataframe = self.log_loader()


    def log_template_matcher(self):
        headers = []
        splitters = re.split(r'(<[^<>]+>)', self.log_template)
        regex = ''
        for k in range(len(splitters)):
            if k % 2 == 0:
                splitter = re.sub(' +', '\\\s+', splitters[k])
                regex += splitter
            else:
                header = splitters[k].strip('<').strip('>')
                regex += '(?P<%s>.+?)' % header
                headers.append(header)
        print(regex)
        regex = re.compile('^' + regex + '$')

        self.log_template_headers, self.log_template_regex = headers, regex



# original authors' data processor
import numpy as np
import math, copy, time
import pandas as pd
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from tqdm import trange
import re


class LogReader:
    def __init__(self, log_format, log_name, indir='./', outdir='./result/', rex=[], every_n=10, max_lines=2000000):
    
        self.path = indir
       
        self.logName = log_name
        self.savePath = outdir
        self.df_log = None
        self.log_format = log_format
        self.rex = rex
        self.every_n = every_n
        self.max_lines = max_lines
    def log_to_dataframe(self, log_file, regex, headers, logformat):
            """ Function to transform log file to dataframe 
            """
            log_messages = []
            linecount = 0
            
            if self.max_lines:
                with open(log_file, 'r', encoding="latin-1") as fin:
                    for i,  line in enumerate(fin):
                        if i % self.every_n == 0:
                            try:
                                match = regex.search(line.strip())
                                message = [match.group(header) for header in headers]
                                log_messages.append(message)
                                linecount += 1
                            except Exception as e:
                                pass
                        if i==self.max_lines:
                            break
            else:
                with open(log_file, 'r', encoding="latin-1") as fin:
                    for i,  line in enumerate(fin):
                        if i % self.every_n == 0:
                            try:
                                match = regex.search(line.strip())
                                message = [match.group(header) for header in headers]
                                log_messages.append(message)
                                linecount += 1
                            except Exception as e:
                                pass
            logdf = pd.DataFrame(log_messages, columns=headers)
            logdf.insert(0, 'LineId', None)
            logdf['LineId'] = [i + 1 for i in range(linecount)]
            return logdf


    def generate_logformat_regex(self, logformat):
        """ Function to generate regular expression to split log messages
        """
        headers = []
        splitters = re.split(r'(<[^<>]+>)', logformat)
        regex = ''
        for k in range(len(splitters)):
            if k % 2 == 0:
                splitter = re.sub(' +', '\\\s+', splitters[k])
                regex += splitter
            else:
                header = splitters[k].strip('<').strip('>')
                regex += '(?P<%s>.*?)' % header
                headers.append(header)
        regex = re.compile('^' + regex + '$')
        return headers, regex

    def load_data(self):
        headers, regex = self.generate_logformat_regex(self.log_format)
        self.df_log = self.log_to_dataframe(os.path.join(self.path, self.logName), regex, headers, self.log_format)
        
        
class LogTokenizer:
    def __init__(self):
        self.word2index = {'[PAD]':0, '[CLS]':1, '[MASK]':2}
        self.index2word = {0:'[PAD]', 1:'[CLS]', 2:'[MASK]'}
        self.n_words = 3  # Count SOS and EOS
        self.stop_words = set(stopwords.words('english'))
        self.regextokenizer =  nltk.RegexpTokenizer('\w+|.|')
        
    def addWord(self, word):
        if word not in self.word2index:
            self.word2index[word] = self.n_words
            self.index2word[self.n_words] = word
            self.n_words += 1

    def tokenize(self, sent):
        sent = re.sub(r'\/.*:', '', sent, flags=re.MULTILINE)
        sent = re.sub(r'\/.*', '', sent, flags=re.MULTILINE)
        sent = self.regextokenizer.tokenize(sent)
        sent = [w.lower() for w in sent]
        sent = [word for word in sent if word.isalpha()]
        sent = [w for w in sent if not w in self.stop_words]
        sent = ['[CLS]'] + sent
        for w in range(len(sent)):
            self.addWord(sent[w])
            sent[w] = self.word2index[sent[w]]
        return sent
    
def get_data(log_file, input_dir, output_dir, log_format, regex=[], every_n=10, aux=0, max_lines=5000000):
    reader = LogReader(log_format, log_file, indir=input_dir, outdir=output_dir, rex=regex, every_n=every_n, max_lines=max_lines)
    reader.load_data()
    log_payload, true_labels = reader.df_log.Content, np.where(reader.df_log.t.values=='-',0,1)
    del reader
    if aux != 0:
        df_anomalies = log_payload.iloc[true_labels.flatten()==1].sample(n=aux).values
        df_normal = log_payload.iloc[true_labels.flatten()==0].sample(n=aux).values
        return df_normal, df_anomalies
    else:
        return log_payload, true_labels
    
def get_data_special(log_file, input_dir, output_dir, log_format, regex=[], every_n=10, aux=0, max_lines=2000000):
    reader = LogReader(log_format, log_file, indir=input_dir, outdir=output_dir, rex=regex, every_n=every_n, max_lines=max_lines)
    reader.load_data()
    log_payload, true_labels = reader.df_log.Content, np.where(reader.df_log.t.values!='FATAL',0,1)
    del reader

    return log_payload, true_labels<jupyter_output><empty_output><jupyter_text>First 3 aux datasets - switch with target if you need to evaluate different target<jupyter_code>#read the datasets
log_file = 'tbird2_small' # small meaning the dataset using only first 5m, note model doesn't converge well when sampling 5m instead of first 5m even if using original authors code
input_dir  = folder_path
log_format = '<t> <Timestamp> <Date> <User> <Month> <Day> <Time> <Location> <Component>(\[<PID>\])?: <Content>'  #thunderbird
output_dir = '.'
aux_normal, aux_anomalies = get_data(log_file, input_dir, output_dir, log_format, every_n=1, aux=200000, max_lines=False) # use 200000 when its tbird2 small (5m)
log_file = 'bgl2' # first 5 million, note bgl only has 5m when full dataset
input_dir  = folder_path  # The input directory of log file
output_dir = '.'  # The output directory of parsing results

log_format = '<t> <Timestamp> <Date> <Node> <Time> <NodeRepeat> <Type> <Component> <Level> <Content>'  #BGL
every_n = 1
aux_size = 200000
aux_normal1, aux_anomalies1 = get_data(log_file, input_dir, output_dir, log_format, every_n=1, aux=200000, max_lines=False)
log_file = 'Intrepid_RAS_0901_0908_scrubbed' # this one cannot be used as target, its too obvious
input_dir  = folder_path
output_dir = '.' 
log_format = '<f> <a>          <c>       <d>                  <e>    <t> <Content>'  #BGP
aux_anomalies_t, tl = get_data_special(log_file, input_dir, output_dir, log_format, every_n=1, aux=False, max_lines=False)
aux_normal2, aux_anomalies2  = aux_anomalies_t[tl==1], aux_anomalies_t[tl==0] # reverse this [TODO]<jupyter_output><empty_output><jupyter_text>Concat the auxiliary dataasets to one single piece, then sample 250000 when using 5m rows<jupyter_code># concatenate
aux_anomalies = np.append(aux_anomalies, aux_anomalies1)
aux_anomalies_full = np.append(aux_anomalies, aux_anomalies2)
# sample
aux_anomalies = np.random.choice(aux_anomalies_full, size=500000, replace=False)  # changing size doesn't seem to change results
print(f"Selected {aux_anomalies.shape} samples out of {len(aux_anomalies_full)} aux samples")<jupyter_output>Selected (500000,) samples out of 3310825 aux samples<jupyter_text>Then target dataset - switch with aux if you need to evaluate different target<jupyter_code>log_file = 'spirit_small'
input_dir  = folder_path
output_dir = '.' 
log_format = '<t> <Timestamp> <Date> <User> <Month> <Day> <Time> <Location> <Content>'  #spirit2

log_payload, true_labels = get_data(log_file, input_dir, output_dir, log_format, every_n=1, aux=False, max_lines=5000000)  # read at most 5m, else may crash
### Never shuffle the data here, or there will be info leak
df_size = len(log_payload)
df_size
true_labels = true_labels.reshape(-1,1)
#append the anomalies to the full data
log_payload = np.append(log_payload.values.reshape(-1,1), aux_anomalies.reshape(-1,1), axis=0)
true_labels = np.append(true_labels, np.ones(len(aux_anomalies)).reshape(-1,1), axis=0).flatten()
print(f'log shape {log_payload.shape}, log labels shape {true_labels.shape}')<jupyter_output>log shape (5499991, 1), log labels shape (5499991,)<jupyter_text>Tokenize!<jupyter_code>tokenizer = LogTokenizer()
df_len = int(log_payload.shape[0]) # the num of logs
data_tokenized = []
for i in trange(df_len):
    tokenized = tokenizer.tokenize(log_payload[i][0])
    data_tokenized.append(tokenized)
# pickle to file
with open('tokenized','wb') as tokenized_file:
  pickle.dump(data_tokenized, tokenized_file)
with open('tokenized','rb') as token:
  data_tokenized = pickle.load(token)
data_token_indexed = np.asanyarray(data_tokenized)<jupyter_output>/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:136: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
  return array(a, dtype, copy=False, order=order, subok=True)<jupyter_text>IMPORTANT: CHANGE TRAIN TEST SPLIT RATIO HERE<jupyter_code>ratio = 0.6 # change from 0.1 0.2 0.4 0.6 0.8 its not always 0.8 performas better than 0.1
train_size = round(df_size * ratio)
print(df_size, train_size)
print(f"train size proportional to total logs - {train_size/df_size}")
from collections import Counter
# perform train test split

# only take normal logs in target within ratio range 
data_token_indexed_train = np.append(data_token_indexed[:train_size][true_labels[:train_size]==0], data_token_indexed[df_size:],axis=0)

# put remaining to test
data_token_indexed_test = data_token_indexed[train_size:df_size]


# labels using same split
train_ground_labels = np.append(true_labels[:train_size][true_labels[:train_size]==0].flatten(), true_labels[df_size:].flatten(),axis=0)
print(f"train ground labels 0 1 counter {Counter(train_ground_labels)}")

test_ground_labels = true_labels[train_size:df_size] 
print(f"test ground labels 0 1 counter {Counter(test_ground_labels)}")

print(data_token_indexed_test.shape, data_token_indexed_train.shape, train_ground_labels.shape, test_ground_labels.shape)
# verify unique log counts consistent with paper count
# Commented because this line takes a long time to run
# unique_counter = np.intersect1d(np.unique(data_token_indexed_test), np.unique(data_token_indexed_train), assume_unique=True)
# print(f"Unique log message count - {np.unique(data_token_indexed_test).shape[0] - unique_counter.shape[0]}")
# assign set names
x_train = data_token_indexed_train
y_train =  train_ground_labels
x_test = data_token_indexed_test
y_test = test_ground_labels

# Add padding to sequences to maxlen 50 according to paper
from keras.preprocessing.sequence import pad_sequences

x_train = pad_sequences(x_train, maxlen=50, truncating="post", padding="post") 
x_test = pad_sequences(x_test, maxlen=50, truncating="post", padding="post") 
print(x_train.shape, x_test.shape)
print(x_train[0])
print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)

## padding masks
x_train_masks = tf.equal(x_train, 0)
x_test_masks = tf.equal(x_test, 0)
print(x_train_masks,x_test_masks)<jupyter_output>(2775123, 50) (1999996, 50)
[1 3 4 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 0 0 0 0 0 0 0 0 0 0 0 0 0]
(2775123, 50) (2775123,) (1999996, 50) (1999996,)
tf.Tensor(
[[False False False ...  True  True  True]
 [False False False ...  True  True  True]
 [False False False ...  True  True  True]
 ...
 [False False False ...  True  True  True]
 [False False  True ...  True  True  True]
 [False False False ...  True  True  True]], shape=(2775123, 50), dtype=bool) tf.Tensor(
[[False False False ...  True  True  True]
 [False False False ...  True  True  True]
 [False False False ...  True  True  True]
 ...
 [False False False ...  True  True  True]
 [False False False ...  True  True  True]
 [False False False ...  True  True  True]], shape=(1999996, 50), dtype=bool)<jupyter_text>Standard transformer encoder with multi-head *attention*<jupyter_code>class TransformerEncoder(layers.Layer): # standard tensformer encoder
    def __init__(self, embed_dim, num_heads, dense_dim, **kwargs):
        super(TransformerEncoder, self).__init__(**kwargs)
        self.embed_dim = embed_dim
        self.dense_dim = dense_dim
        self.num_heads = num_heads
        self.attention = layers.MultiHeadAttention(
            num_heads=num_heads, key_dim=embed_dim,dropout=0.1
        )
        self.dense_proj = keras.Sequential(
            [layers.Dense(dense_dim, activation="relu"), layers.Dense(embed_dim),]
        )
        self.layernorm_1 = layers.LayerNormalization()
        self.layernorm_2 = layers.LayerNormalization()
        self.supports_masking = True
        self.dropout1 = layers.Dropout(0.05)
        self.dropout2 = layers.Dropout(0.05)

    def call(self, inputs, training, mask=None):
        if mask is not None:
            padding_mask = tf.cast(mask[:, tf.newaxis, tf.newaxis, :], dtype="int32")
        attention_output = self.attention(
            query=inputs, value=inputs, key=inputs, attention_mask=padding_mask
        )
        attention_output = self.dropout1(attention_output, training=training)
        proj_input = self.layernorm_1(inputs + attention_output)
        proj_output = self.dense_proj(proj_input)
        proj_output = self.dropout2(proj_output, training=training)
        return self.layernorm_2(proj_input + proj_output)


class PositionalEmbedding(layers.Layer):
    def __init__(self, sequence_length, vocab_size, embed_dim, **kwargs):
        super(PositionalEmbedding, self).__init__(**kwargs)
        self.token_embeddings = layers.Embedding(
            input_dim=vocab_size, output_dim=embed_dim
        )
        self.position_embeddings = layers.Embedding(
            input_dim=sequence_length, output_dim=embed_dim
        )
        self.sequence_length = sequence_length
        self.vocab_size = vocab_size
        self.embed_dim = embed_dim

    def call(self, inputs):
        length = tf.shape(inputs)[-1]
        positions = tf.range(start=0, limit=length, delta=1)
        embedded_tokens = self.token_embeddings(inputs)
        embedded_positions = self.position_embeddings(positions)
        return embedded_tokens + embedded_positions

    def compute_mask(self, inputs, mask=None):
        return tf.math.not_equal(inputs, 0)<jupyter_output><empty_output><jupyter_text>Metrics and model building<jupyter_code>a = tf.metrics.BinaryAccuracy()
def recall_m(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    return true_positives / (possible_positives + K.epsilon())

def precision_m(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    return true_positives / (predicted_positives + K.epsilon())

def f1_m(y_true, y_pred):
    precision = precision_m(y_true, y_pred)
    recall = recall_m(y_true, y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))

def accuracy_m(y_true, y_pred):
    a.update_state(y_true,y_pred)
    return a.result()

def create_model():
  embed_dim = 16  # Embedding size for each token
  num_heads = 2  # Number of attention heads
  ff_dim = 16  # Hidden layer size in feed forward network inside transformer

  vocab_size = tokenizer.n_words  # Only consider the top 20k words
  maxlen = 50

  inputs = layers.Input(shape=(maxlen,))
  embedding_layer = PositionalEmbedding(maxlen, vocab_size, embed_dim)
  x = embedding_layer(inputs)
  transformer_block = TransformerEncoder(embed_dim, num_heads, ff_dim)
  transformer_block2 = TransformerEncoder(embed_dim, num_heads, ff_dim)

  x = transformer_block(x)
  x = transformer_block2(x)
  
  outputs = tf.gather(x, 0, axis=1) ## take the CLS token as embedding

  function = K.sum(K.square(outputs),axis=1)
  outputs = tf.keras.layers.Lambda(lambda x: K.sum(K.square(x),axis=1))(outputs)

  model = tf.keras.Model(inputs=inputs, outputs=outputs)
  
  return model

# learning rate decay for optmimizer
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.0001, # 0.0001,
    decay_steps=10000,
    decay_rate=1-0.001)

# optimizer
model_opt = tf.keras.optimizers.Adam(learning_rate=lr_schedule,beta_1=0.9, beta_2=0.999)

def custom_loss_function(y_true, y_pred):
   loss = K.mean((1-y_true)*K.sqrt(y_pred) - (y_true)*K.log(1-K.exp(-K.sqrt(y_pred))))
   #loss = K.mean((1-y_true)*K.square(dist) - (y_true)*K.log(1-K.exp(-K.square(dist))))

   return loss

# Create model with parallel execution - IMPORTANT!!
with strategy.scope():
  model = create_model()

# NOTE! the metrics here don't make sense, it requires threshold derivation
model.compile(optimizer=model_opt, loss=custom_loss_function, metrics=['binary_accuracy',recall_m,precision_m],loss_weights = [0.2, 1.0])  # this weight is [0.5, 1.0] in paper

print(model.summary())<jupyter_output>Model: "model_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_4 (InputLayer)        [(None, 50)]              0         
                                                                 
 positional_embedding_3 (Pos  (None, 50, 16)           45264     
 itionalEmbedding)                                               
                                                                 
 transformer_encoder_6 (Tran  (None, 50, 16)           2768      
 sformerEncoder)                                                 
                                                                 
 transformer_encoder_7 (Tran  (None, 50, 16)           2768      
 sformerEncoder)                                                 
                                                                 
 tf.compat.v1.gather_3 (TFOp  (None, 16)               0   [...]<jupyter_text>Threshold derivation! Evaluation step begins from fit We have no idea how original author got the thresholds, we use average of auc score<jupyter_code>from sklearn.metrics import f1_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score

def evaluation_threshold(threshold, y_test, distances):

  tr = [threshold]

  for t in tr:
    res=[]
    for d in distances:
      if d > t:
        res.append(1)
      else:
        res.append(0)

    print('threshold value: ', t)
    print('f1: ', f1_score(y_test.astype(np.int32), res))
    print('recall: ',  recall_score(y_test.astype(np.int32), res))
    print('precision: ', precision_score(y_test.astype(np.int32), res))
    print('Accuracy: ', accuracy_score(y_test.astype(np.int32), res))

    print('##################################')

# sometimes it can be a bad run, just rerun to see if results improve
for i in range(20):
  print(f"Currently Running Epoch - {i+1}")
  model.fit(x_train, y_train, batch_size=2048, epochs=1)

  test_distances = model.predict(x_test,batch_size=2048)
  test_auc = roc_auc_score(y_test.astype(np.int32), test_distances)

  train_distances = model.predict(x_train,batch_size=2048)
  train_auc = roc_auc_score(y_train.astype(np.int32), train_distances)

  fpr_test, tpr_test, thresholds_test = roc_curve(y_test.astype(np.int32), test_distances, pos_label=1)
  fpr_train, tpr_train, thresholds_train = roc_curve(y_train.astype(np.int32), train_distances, pos_label=1)

  print(f'Train AUC: {train_auc}')
  print(f'Test AUC: {test_auc}')

  if train_auc == 1: # prevent calculation go wrong, not impacting results
    train_auc = 0.99
  if test_auc == 1:
    test_auc = 0.99


  print('Using trainset to find threshold:') # this should be legit
  evaluation_threshold(np.average(thresholds_train[tpr_train>train_auc]), y_test, test_distances)
  print('Using testset to find threshold:') # this is what the authors used, which leaks info
  evaluation_threshold(np.average(thresholds_test[tpr_test>test_auc]), y_test, test_distances)<jupyter_output>Currently Running Epoch - 1
1356/1356 [==============================] - 46s 26ms/step - loss: 0.5030 - binary_accuracy: 0.2359 - recall_m: 0.9999 - precision_m: 0.1919
Train AUC: 0.9999505831544052
Test AUC: 0.99674711777883
Using trainset to find threshold:
threshold value:  7.584181
f1:  0.09255119470699871
recall:  0.9999000449797592
precision:  0.04852116783541558
Accuracy:  0.6076687153374307
##################################
Using testset to find threshold:
threshold value:  7.6317677
f1:  0.1101564500188577
recall:  0.9999000449797592
precision:  0.05828900063512781
Accuracy:  0.6767663535327071
##################################
Currently Running Epoch - 2
1356/1356 [==============================] - 35s 26ms/step - loss: 0.3608 - binary_accuracy: 0.4288 - recall_m: 0.9995 - precision_m: 0.2400
Train AUC: 0.9996841298786923
Test AUC: 0.9915002078633108
Using trainset to find threshold:
threshold value:  4.5026674
f1:  0.12521734231238135
recall:  0.9996751461842172
precision:[...]
{'fraction_non_alphanumeric': 0.11126712328767123, 'fraction_numerical': 0.030513698630136987, 'mean_word_length': 2.9540961408259987, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 90, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '478276', 'license': 'MIT', 'n_tokens_mistral': 9897, 'n_tokens_neox': 9075, 'n_words': 2270, 'path': 'src/model/anomaly_detection.ipynb', 'repo_name': 'Superskyyy/deep-log-unstructured', 'size': '29200'}