<commit_before>'''@file trainer.py
neural network trainer environment'''

import os
from abc import ABCMeta, abstractmethod
from time import time, sleep
import tensorflow as tf
import numpy as np

class Trainer(object):
    '''General class outlining the training environment of a classifier.'''
    __metaclass__ = ABCMeta

    def __init__(self,
                 conf,
                 decoder,
                 classifier,
                 input_dim,
                 dispenser,
                 val_reader,
                 val_targets,
                 expdir,
                 server,
                 task_index):
        '''
        NnetTrainer constructor, creates the training graph

        Args:
            classifier: the neural net classifier that will be trained
            conf: the trainer config
            decoder: a callable that will create a decoder
            input_dim: the input dimension to the nnnetgraph
            num_steps: the total number of steps that will be taken
            dispenser: a Batchdispenser object
            val_reader: the feature reader for the validation data if None
                validation will not be used
            val_targets: a dictionary containing the targets of the validation
                set
            expdir: directory where the summaries will be written
            server: optional server to be used for distributed training
            task_index: optional index of the worker task in the cluster
        '''

        self.conf = conf
        self.dispenser = dispenser
        self.num_steps = int(dispenser.num_batches*int(conf['num_epochs'])
                             /max(1, int(conf['numbatches_to_aggregate'])))
        self.val_reader = val_reader
        self.val_targets = val_targets

        self.expdir = expdir
        self.server = server
        cluster = tf.train.ClusterSpec(server.server_def.cluster).as_dict()

        #save the max lengths
        self.max_target_length = dispenser.max_target_length
        self.max_input_length = dispenser.max_input_length

        #create the graph
        self.graph = tf.Graph()

        if 'local' in cluster:
            num_replicas = 1
        else:
            #distributed training
            num_replicas = len(cluster['worker'])

        self.is_chief = task_index == 0
        device = tf.train.replica_device_setter(cluster=cluster)

        #define the placeholders in the graph
        with self.graph.as_default():

            with tf.device(device):

                #create the inputs placeholder
                self.inputs = tf.placeholder(
                    dtype=tf.float32,
                    shape=[dispenser.size, self.max_input_length,<commit_msg>specified worker devices in device setter<commit_after>'''@file trainer.py
neural network trainer environment'''

import os
from abc import ABCMeta, abstractmethod
from time import time, sleep
import tensorflow as tf
import numpy as np

class Trainer(object):
    '''General class outlining the training environment of a classifier.'''
    __metaclass__ = ABCMeta

    def __init__(self,
                 conf,
                 decoder,
                 classifier,
                 input_dim,
                 dispenser,
                 val_reader,
                 val_targets,
                 expdir,
                 server,
                 task_index):
        '''
        NnetTrainer constructor, creates the training graph

        Args:
            classifier: the neural net classifier that will be trained
            conf: the trainer config
            decoder: a callable that will create a decoder
            input_dim: the input dimension to the nnnetgraph
            num_steps: the total number of steps that will be taken
            dispenser: a Batchdispenser object
            val_reader: the feature reader for the validation data if None
                validation will not be used
            val_targets: a dictionary containing the targets of the validation
                set
            expdir: directory where the summaries will be written
            server: optional server to be used for distributed training
            task_index: optional index of the worker task in the cluster
        '''

        self.conf = conf
        self.dispenser = dispenser
        self.num_steps = int(dispenser.num_batches*int(conf['num_epochs'])
                             /max(1, int(conf['numbatches_to_aggregate'])))
        self.val_reader = val_reader
        self.val_targets = val_targets

        self.expdir = expdir
        self.server = server
        cluster = tf.train.ClusterSpec(server.server_def.cluster).as_dict()

        #save the max lengths
        self.max_target_length = dispenser.max_target_length
        self.max_input_length = dispenser.max_input_length

        #create the graph
        self.graph = tf.Graph()

        if 'local' in cluster:
            num_replicas = 1
        else:
            #distributed training
            num_replicas = len(cluster['worker'])

        self.is_chief = task_index == 0
        device = tf.train.replica_device_setter(
            cluster=cluster,
            worker_device='/job:worker/task:%d' % task_index)

        print device
        print task_index

        #define the placeholders in the graph
        with self.graph.as_default():

            with tf.device(device):

                #create the inputs placeholder
                self.inputs = tf.placeholder(
                    dtype=tf.float32,
                    shape=[dispenser.size, self.max_input_length,
{'fraction_non_alphanumeric': 0.059012875536480686, 'fraction_numerical': 0.00178826895565093, 'mean_word_length': 1.823321554770318, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3616333', 'n_tokens_mistral': 1423, 'n_tokens_neox': 1356, 'n_words': 447}