<commit_before># -*- coding: utf-8 -*-

import tensorflow as tf
import numpy as np
import yaml
import sys
import codecs

from WordEmbedding import WordEmbedding 

sys.stdin  = codecs.getreader('UTF-8')(sys.stdin)
sys.stdout = codecs.getwriter('UTF-8')(sys.stdout)
sys.stderr = codecs.getwriter('UTF-8')(sys.stderr)

class NeuralNetworkLanguageModel:
    def __init__(self, configPath="config.yml"):
        with codecs.open(configPath, "r", "UTF-8") as f:
            configString = f.read()
            data = yaml.load(configString) 

        self.history = data["history"]
        self.lineDim = data["lineDim"]
        self.learningRate = data["learningRate"]
        self.lineDelimiter = data["lineDelimiter"]
        self.wordDelimiter = data["wordDelimiter"]
        self.iteration = data["iteration"]
        self.supervisor_labels_placeholder = tf.placeholder("int32", [None])
        self.input_placeholder = tf.placeholder("float", [None, (self.history + 1)*(self.lineDim)*WordEmbedding.EMBEDDING_SIZE])
        self.layerSize = data["layerSize"]

    def _getWordVector(self, wordId, word):
        """
        To be written
        """
        with tf.name_scope("word%d" % wordId):
            # Hidden 1
            with tf.name_scope('hidden1'):
                weights = tf.Variable(tf.zeros([WordEmbedding.EMBEDDING_SIZE, self.layerSize["word"]["hidden1"]]), name="weights")
                biases = tf.Variable(tf.zeros([self.layerSize["word"]["hidden1"]]), name='biases')
                hidden1 = tf.nn.relu(tf.matmul(word, weights) + biases)

            # Hidden 2
            with tf.name_scope('hidden2'):
                weights = tf.Variable(tf.zeros([self.layerSize["word"]["hidden1"], self.layerSize["word"]["hidden2"]]), name="weights")
                biases = tf.Variable(tf.zeros([self.layerSize["word"]["hidden2"]]), name='biases')
                hidden2 = tf.nn.relu(tf.matmul(hidden1, weights) + biases) 

            # Word vector
            with tf.name_scope('wordVector'):
                weights = tf.Variable(tf.zeros([self.layerSize["word"]["hidden2"], WordEmbedding.EMBEDDING_SIZE]), name="weights")
                biases = tf.Variable(tf.zeros([WordEmbedding.EMBEDDING_SIZE]), name='biases')
                wordVector = tf.nn.relu(tf.matmul(hidden2, weights) + biases) 

        return wordVector

    def _getLineVector(self, lineId, line):
        """
        To be written
        """
        with tf.name_scope("line%d" % lineId):
            wordVectors = []
            splitted = tf.split(1, self.lineDim, line)
            for wordId, word in enumerate(splitted):
                wordVectors.append(self._getWordVector(wordId, word))

            wordVectors = tf.concat(1, wordVectors)
            lineVectorSize = (self.lineDim)*WordEmbedding.EMBEDDING_SIZE
            # Hidden 1
            with tf.name_scope('hidden1'):
                weights = tf.Variable(tf.zeros([lineVectorSize, self.layerSize["line"]["hidden1"]]), name="weights")
                biases = tf.Variable(tf.zeros([self.layerSize["line"]["hidden1"]]), name='biases')
                hidden1 = tf.nn.relu(tf.matmul(wordVectors, weights) + biases)

            # Line vector
            with tf.name_scope('lineVector'):
                weights = tf.Variable(tf.zeros([self.layerSize["line"]["hidden1"], lineVectorSize]), name="weights")
                biases = tf.Variable(tf.zeros([lineVectorSize]), name='biases')
                lineVector = tf.nn.relu(tf.matmul(hidden1, weights) + biases)

        return lineVector

    def _getTextVector(self, lines):
        """
        To be written
        """
        lineVectors = []
        for lineIndex, line in enumerate(tf.split(1, self.history + 1, lines)):
            lineVectors.append(self._getLineVector(lineIndex, line))

        lineVectors = tf.concat(1, lineVectors)
        textVectorSize = (self.history + 1) * (self.lineDim)*WordEmbedding.EMBEDDING_SIZE

        with tf.name_scope("text"):
            # Hidden 1
            with tf.name_scope('hidden1'):
                weights = tf.Variable(tf.zeros([textVectorSize, self.layerSize["text"]["hidden1"]]), name="weights") # TODO: what value should I use for stddev
                biases = tf.Variable(tf.zeros([self.layerSize["text"]["hidden1"]]), name='biases')
                hidden1 = tf.nn.relu(tf.matmul(lineVectors, weights) + biases)
            
            # Text vector
            with tf.name_scope('textVector'):
                weights = tf.Variable(tf.zeros([self.layerSize["text"]["hidden1"], textVectorSize]), name="weights") # TODO: what value should I use for stddev
                biases = tf.Variable(tf.zeros([textVectorSize]), name='biases')
                textVector = tf.nn.relu(tf.matmul(hidden1, weights) + biases)

        return textVector

    def inference(self, lines):
        textVector = self._getTextVector(lines)
        textVectorSize = (self.history + 1) * (self.lineDim)*WordEmbedding.EMBEDDING_SIZE
        # Linear
        with tf.name_scope('linear'):
            weights = tf.Variable(tf.zeros([textVectorSize, 2]), name="weights") # TODO: what value should I use for stddev
            biases = tf.Variable(tf.zeros([2]), name='biases')
            logits = tf.matmul(textVector, weights) + biases
        
        return logits

    def loss(self, logits, labels):
        """Calculates the loss from the logits and the labels.<commit_msg>Use unicode_literal<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import codecs
import tensorflow as tf
import numpy as np
import yaml
import sys

from WordEmbedding import WordEmbedding 

class NeuralNetworkLanguageModel:
    def __init__(self, configPath="config.yml"):
        with open(configPath, "r") as f:
            configString = f.read()
            data = yaml.load(configString) 

        self.history = data["history"]
        self.lineDim = data["lineDim"]
        self.learningRate = data["learningRate"]
        self.lineDelimiter = data["lineDelimiter"]
        self.wordDelimiter = data["wordDelimiter"]
        self.iteration = data["iteration"]
        self.supervisor_labels_placeholder = tf.placeholder("int32", [None])
        self.input_placeholder = tf.placeholder("float", [None, (self.history + 1)*(self.lineDim)*WordEmbedding.EMBEDDING_SIZE])
        self.layerSize = data["layerSize"]

    def _getWordVector(self, wordId, word):
        """
        To be written
        """
        with tf.name_scope("word%d" % wordId):
            # Hidden 1
            with tf.name_scope('hidden1'):
                weights = tf.Variable(tf.zeros([WordEmbedding.EMBEDDING_SIZE, self.layerSize["word"]["hidden1"]]), name="weights")
                biases = tf.Variable(tf.zeros([self.layerSize["word"]["hidden1"]]), name='biases')
                hidden1 = tf.nn.relu(tf.matmul(word, weights) + biases)

            # Hidden 2
            with tf.name_scope('hidden2'):
                weights = tf.Variable(tf.zeros([self.layerSize["word"]["hidden1"], self.layerSize["word"]["hidden2"]]), name="weights")
                biases = tf.Variable(tf.zeros([self.layerSize["word"]["hidden2"]]), name='biases')
                hidden2 = tf.nn.relu(tf.matmul(hidden1, weights) + biases) 

            # Word vector
            with tf.name_scope('wordVector'):
                weights = tf.Variable(tf.zeros([self.layerSize["word"]["hidden2"], WordEmbedding.EMBEDDING_SIZE]), name="weights")
                biases = tf.Variable(tf.zeros([WordEmbedding.EMBEDDING_SIZE]), name='biases')
                wordVector = tf.nn.relu(tf.matmul(hidden2, weights) + biases) 

        return wordVector

    def _getLineVector(self, lineId, line):
        """
        To be written
        """
        with tf.name_scope("line%d" % lineId):
            wordVectors = []
            splitted = tf.split(1, self.lineDim, line)
            for wordId, word in enumerate(splitted):
                wordVectors.append(self._getWordVector(wordId, word))

            wordVectors = tf.concat(1, wordVectors)
            lineVectorSize = (self.lineDim)*WordEmbedding.EMBEDDING_SIZE
            # Hidden 1
            with tf.name_scope('hidden1'):
                weights = tf.Variable(tf.zeros([lineVectorSize, self.layerSize["line"]["hidden1"]]), name="weights")
                biases = tf.Variable(tf.zeros([self.layerSize["line"]["hidden1"]]), name='biases')
                hidden1 = tf.nn.relu(tf.matmul(wordVectors, weights) + biases)

            # Line vector
            with tf.name_scope('lineVector'):
                weights = tf.Variable(tf.zeros([self.layerSize["line"]["hidden1"], lineVectorSize]), name="weights")
                biases = tf.Variable(tf.zeros([lineVectorSize]), name='biases')
                lineVector = tf.nn.relu(tf.matmul(hidden1, weights) + biases)

        return lineVector

    def _getTextVector(self, lines):
        """
        To be written
        """
        lineVectors = []
        for lineIndex, line in enumerate(tf.split(1, self.history + 1, lines)):
            lineVectors.append(self._getLineVector(lineIndex, line))

        lineVectors = tf.concat(1, lineVectors)
        textVectorSize = (self.history + 1) * (self.lineDim)*WordEmbedding.EMBEDDING_SIZE

        with tf.name_scope("text"):
            # Hidden 1
            with tf.name_scope('hidden1'):
                weights = tf.Variable(tf.zeros([textVectorSize, self.layerSize["text"]["hidden1"]]), name="weights")
                biases = tf.Variable(tf.zeros([self.layerSize["text"]["hidden1"]]), name='biases')
                hidden1 = tf.nn.relu(tf.matmul(lineVectors, weights) + biases)
            
            # Text vector
            with tf.name_scope('textVector'):
                weights = tf.Variable(tf.zeros([self.layerSize["text"]["hidden1"], textVectorSize]), name="weights")
                biases = tf.Variable(tf.zeros([textVectorSize]), name='biases')
                textVector = tf.nn.relu(tf.matmul(hidden1, weights) + biases)

        return textVector

    def inference(self, lines):
        textVector = self._getTextVector(lines)
        textVectorSize = (self.history + 1) * (self.lineDim)*WordEmbedding.EMBEDDING_SIZE
        # Linear
        with tf.name_scope('linear'):
            weights = tf.Variable(tf.zeros([textVectorSize, 2]), name="weights")
            biases = tf.Variable(tf.zeros([2]), name='biases')
            logits = tf.matmul(textVector, weights) + biases
        
        return logits

    def loss(self, logits, labels):
        """Calculates the loss from the logits and the labels.
{'fraction_non_alphanumeric': 0.13933345885897194, 'fraction_numerical': 0.008096403690453776, 'mean_word_length': 2.5469115191986647, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1437164', 'n_tokens_mistral': 3262, 'n_tokens_neox': 3157, 'n_words': 573}