<commit_before>from nltk import corpus
from collections import Counter
from math import log, pow, sqrt
from re import sub


class PreprocessLog:
    def __init__(self, logfile):
        self.logfile = logfile
        self.logs = []
        self.loglength = 0
        self.events_list = []
        self.events_unique = []

    def read_log(self):
        with open(self.logfile, 'r') as f:
            logs = f.readlines()

        self.logs = logs
        self.loglength = len(logs)

    def get_logs(self):
        return self.logs

    def get_loglength(self):
        return self.loglength

    def get_wordindocs(self, word, docs):
        # find word occurence in all docs (logs)
        count = 0
        for doc in docs:
            if word in doc:
                count += 1

        return float(count)

    def get_tfidf(self, doc, total_docs, docs):
        # remove number, stopwords
        doc = sub('[^a-zA-Z]', ' ', doc)
        additional_stopwords = ['preauth', 'from', 'xxxxx', 'for', 'port', 'sshd', 'ssh']
        for a in additional_stopwords:
            doc = doc.replace(a, '')
        doc.replace('_', ' ')
        doc = ' '.join(doc.split())

        stopwords = corpus.stopwords.words('english')
        stopwords_result = [w.lower() for w in doc.split() if w.lower() not in stopwords]

        # count word frequency (tf)
        tf = Counter(stopwords_result)
        words_total = len(stopwords_result)
        tfidf = []
        for t in tf.most_common():
            normalized_tf = float(t[1]) / float(words_total)    # normalized word frequency
            wid = self.get_wordindocs(t[0], docs)               # calculate word occurrence in all documents
            idf = 1 + log(total_docs / wid)                     # calculate idf
            tfidf_val = normalized_tf * idf                     # calculate tf-idf
            tfidf.append((t[0], tfidf_val))

        return doc, tfidf

    def get_doclength(self, tfidf):
        # calculate doc's length for cosine similarity
        length = 0
        for ti in tfidf:
            length += pow(ti[1], 2)

        return sqrt(length)

    def do_preprocess(self):
        # read log file
        self.read_log()

        # convert to lower, count total logs
        logs_lower = [' '.join(l.lower().split()[5:]) for l in self.logs[:]]
        logs_total = self.loglength

        # preprocess logs, add to ordinary list and unique list
        events_list, events_unique = [], []
        index, index_log = 0, 0
        for l in logs_lower:
            auth_split = l.split()
            event_type, event_desc = auth_split[0].split('[')[0], ' '.join(auth_split[1:])
            event = event_type + ' ' + event_desc
            events_list.append(event)

            preprocessed_event, tfidf = self.get_tfidf(event, logs_total, logs_lower)
            check_events_unique = [e[1]['preprocessed_event'] for e in events_unique]

            # if not exist, add new element
            if preprocessed_event not in check_events_unique:
                length = self.get_doclength(tfidf)
                events_unique.append([index, {'event': event, 'tf-idf': tfidf, 'length': length, 'status': '',
                                              'cluster': 0, 'frequency': 1, 'member': [index_log],
                                              'preprocessed_event':preprocessed_event}])
                index += 1
            # if exist, increment the frequency
            else:
                for e in events_unique:
                    if preprocessed_event == e[1]['preprocessed_event']:
                        member = e[1]['member']
                        member.append(index_log)
                        e[1]['member'] = member
                        e[1]['frequency'] += 1<commit_msg>Add exception handling for idf calculation<commit_after>from nltk import corpus
from collections import Counter
from math import log, pow, sqrt
from re import sub


class PreprocessLog:
    def __init__(self, logfile):
        self.logfile = logfile
        self.logs = []
        self.loglength = 0
        self.events_list = []
        self.events_unique = []

    def read_log(self):
        with open(self.logfile, 'r') as f:
            logs = f.readlines()

        self.logs = logs
        self.loglength = len(logs)

    def get_logs(self):
        return self.logs

    def get_loglength(self):
        return self.loglength

    def get_wordindocs(self, word, docs):
        # find word occurence in all docs (logs)
        count = 0
        for doc in docs:
            if word in doc:
                count += 1

        return float(count)

    def get_tfidf(self, doc, total_docs, docs):
        # remove number, stopwords
        doc = sub('[^a-zA-Z]', ' ', doc)
        additional_stopwords = ['preauth', 'from', 'xxxxx', 'for', 'port', 'sshd', 'ssh']
        for a in additional_stopwords:
            doc = doc.replace(a, '')
        doc.replace('_', ' ')
        doc = ' '.join(doc.split())

        stopwords = corpus.stopwords.words('english')
        stopwords_result = [w.lower() for w in doc.split() if w.lower() not in stopwords]

        # count word frequency (tf)
        tf = Counter(stopwords_result)
        words_total = len(stopwords_result)
        tfidf = []
        for t in tf.most_common():
            normalized_tf = float(t[1]) / float(words_total)    # normalized word frequency
            wid = self.get_wordindocs(t[0], docs)               # calculate word occurrence in all documents
            try:
                idf = 1 + log(total_docs / wid)                     # calculate idf
            except ZeroDivisionError:
                idf = 1
            tfidf_val = normalized_tf * idf                     # calculate tf-idf
            tfidf.append((t[0], tfidf_val))

        return doc, tfidf

    def get_doclength(self, tfidf):
        # calculate doc's length for cosine similarity
        length = 0
        for ti in tfidf:
            length += pow(ti[1], 2)

        return sqrt(length)

    def do_preprocess(self):
        # read log file
        self.read_log()

        # convert to lower, count total logs
        logs_lower = [' '.join(l.lower().split()[5:]) for l in self.logs[:]]
        logs_total = self.loglength

        # preprocess logs, add to ordinary list and unique list
        events_list, events_unique = [], []
        index, index_log = 0, 0
        for l in logs_lower:
            auth_split = l.split()
            event_type, event_desc = auth_split[0].split('[')[0], ' '.join(auth_split[1:])
            event = event_type + ' ' + event_desc
            events_list.append(event)

            preprocessed_event, tfidf = self.get_tfidf(event, logs_total, logs_lower)
            check_events_unique = [e[1]['preprocessed_event'] for e in events_unique]

            # if not exist, add new element
            if preprocessed_event not in check_events_unique:
                print index, preprocessed_event
                length = self.get_doclength(tfidf)
                events_unique.append([index, {'event': event, 'tf-idf': tfidf, 'length': length, 'status': '',
                                              'cluster': 0, 'frequency': 1, 'member': [index_log],
                                              'preprocessed_event':preprocessed_event}])
                index += 1

            # if exist, increment the frequency
            else:
                for e in events_unique:
                    if preprocessed_event == e[1]['preprocessed_event']:
                        member = e[1]['member']
                        member.append(index_log)
                        e[1]['member'] = member
                        e[1]['frequency'] += 1
{'fraction_non_alphanumeric': 0.1071289946953034, 'fraction_numerical': 0.006598525035580282, 'mean_word_length': 1.7626876340243032, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '804616', 'n_tokens_mistral': 2288, 'n_tokens_neox': 2147, 'n_words': 594}