<commit_before>#!/usr/bin/env python3
# encoding: utf-8

import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import ModelCheckpoint
from config import DATASET, HDF5
from engine.datastore.structure.section import IMRaDType
from engine.preprocessing.chapter_classifier.bag_of_words import BagOfWords
from engine.preprocessing.chapter_classifier.classifier_base import ClassifierBase
from engine.preprocessing.chapter_classifier.classifier_utils.metrics import recall, precision, f1


class ClassifierNN(ClassifierBase):
    def __init__(self, load_weigths=True, size_input_layer=60, size_middle_layer=110, batch_size=10, num_epochs=80,
                 val_split=0.2):
        self.size_input_layer = size_input_layer
        self.size_middle_layer = size_middle_layer
        self.batch_size = batch_size
        self.num_epochs = num_epochs
        self.val_split = val_split
        self.bag = BagOfWords()
        self.__init_model(load_weigths)


    def __init_model(self, load_weights):
        self.model = Sequential()
        # Input Layer:
        self.model.add(Dense(self.size_input_layer, input_dim=len(self.bag.get_vocabulary()), activation='relu'))
        # Hidden Layer
        self.model.add(Dense(self.size_middle_layer, activation='relu'))
        # Output-Layer holds all members of the IMRaDTypes; softmax = give the actual output class label probabilities
        self.model.add(Dense(len(IMRaDType), activation='softmax'))

        if load_weights:
            self.model.load_weights(HDF5 + "weights_04.hdf5")
        self.model.compile(loss='categorical_crossentropy', optimizer='adam',
                           metrics=['accuracy', recall, precision, f1])


    def __load_dataset(self, filename):
        dataset = []
        with open(DATASET + filename) as f:
            content = f.readlines()

        for line in content:
            tokens = line.split(":")
            dataset.append([tokens[0].rstrip(), np.zeros(len(IMRaDType))])
            imrad_types = tokens[1].split(" ")

            for imrad in imrad_types:
                imrad = imrad.rstrip()
                if imrad == IMRaDType.INTRODUCTION.name:
                    dataset[-1][1][IMRaDType.INTRODUCTION.value] = 1
                elif imrad == IMRaDType.BACKGROUND.name:
                    dataset[-1][1][IMRaDType.BACKGROUND.value] = 1<commit_msg>Remove Problem with load weights - NN not used atm<commit_after>#!/usr/bin/env python3
# encoding: utf-8

import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import ModelCheckpoint
from config import DATASET, HDF5
from engine.datastore.structure.section import IMRaDType
from engine.preprocessing.chapter_classifier.bag_of_words import BagOfWords
from engine.preprocessing.chapter_classifier.classifier_base import ClassifierBase
from engine.preprocessing.chapter_classifier.classifier_utils.metrics import recall, precision, f1


class ClassifierNN(ClassifierBase):
    def __init__(self, load_weigths=True, size_input_layer=60, size_middle_layer=110, batch_size=10, num_epochs=80,
                 val_split=0.2):
        self.size_input_layer = size_input_layer
        self.size_middle_layer = size_middle_layer
        self.batch_size = batch_size
        self.num_epochs = num_epochs
        self.val_split = val_split
        self.bag = BagOfWords()
        self.__init_model(load_weigths)


    def __init_model(self, load_weights):
        self.model = Sequential()
        # Input Layer:
        self.model.add(Dense(self.size_input_layer, input_dim=len(self.bag.get_vocabulary()), activation='relu'))
        # Hidden Layer
        self.model.add(Dense(self.size_middle_layer, activation='relu'))
        # Output-Layer holds all members of the IMRaDTypes; softmax = give the actual output class label probabilities
        self.model.add(Dense(len(IMRaDType), activation='softmax'))

        # if load_weights:
        #    self.model.load_weights(HDF5 + "weights_04.hdf5")
        self.model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', recall, precision, f1])


    def __load_dataset(self, filename):
        dataset = []
        with open(DATASET + filename) as f:
            content = f.readlines()

        for line in content:
            tokens = line.split(":")
            dataset.append([tokens[0].rstrip(), np.zeros(len(IMRaDType))])
            imrad_types = tokens[1].split(" ")

            for imrad in imrad_types:
                imrad = imrad.rstrip()
                if imrad == IMRaDType.INTRODUCTION.name:
                    dataset[-1][1][IMRaDType.INTRODUCTION.value] = 1
                elif imrad == IMRaDType.BACKGROUND.name:
                    dataset[-1][1][IMRaDType.BACKGROUND.value] = 1
{'fraction_non_alphanumeric': 0.09745499689633767, 'fraction_numerical': 0.011587006000413822, 'mean_word_length': 3.0249791840133224, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3313690', 'n_tokens_mistral': 1531, 'n_tokens_neox': 1452, 'n_words': 295}