def train(self):

    def get_optimizer(loss, global_step=None, var_list=None, is_gradient_clip=False):
        train_op = tf.train.AdamOptimizer(self.lr)
        if is_gradient_clip:
            grads_and_vars = train_op.compute_gradients(loss, var_list=var_list)
            unchanged_gvs = [(grad, var) for (grad, var) in grads_and_vars if (not ('LSTM' in var.name))]
            rnn_grad = [grad for (grad, var) in grads_and_vars if ('LSTM' in var.name)]
            rnn_var = [var for (grad, var) in grads_and_vars if ('LSTM' in var.name)]
            (capped_grad, _) = tf.clip_by_global_norm(rnn_grad, clip_norm=3)
            capped_gvs = list(zip(capped_grad, rnn_var))
            train_op = train_op.apply_gradients(grads_and_vars=(capped_gvs + unchanged_gvs), global_step=global_step)
        else:
            train_op = train_op.minimize(loss, global_step, var_list)
        return train_op
    global_step = tf.Variable(initial_value=0, dtype=tf.int32, trainable=False)
    self.global_step = global_step
    self.build_model()
    self.lr = tf.train.polynomial_decay(self.learning_rate, global_step, self.max_steps, end_learning_rate=0.0, power=0.3)
    tf.summary.scalar('learning_rate', self.lr)
    train_gnet = get_optimizer(self.loss_total, global_step, self.all_vars)
    gpu_options = tf.GPUOptions(allow_growth=True)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
    self.sess = sess
    sess.run(tf.global_variables_initializer())
    self.saver = tf.train.Saver(max_to_keep=50, keep_checkpoint_every_n_hours=1)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    summary_op = tf.summary.merge_all()
    summary_writer = tf.summary.FileWriter(self.train_dir, sess.graph, flush_secs=30)
    for step in xrange(sess.run(global_step), (self.max_steps + 1)):
        start_time = time.time()
        (_, loss_total_val) = sess.run([train_gnet, self.loss_total])
        duration = (time.time() - start_time)
        assert (not np.isnan(loss_total_val)), 'Model diverged with loss = NaN'
        if ((step % 5) == 0):
            num_examples_per_step = self.batch_size
            examples_per_sec = (num_examples_per_step / duration)
            sec_per_batch = float(duration)
            format_str = '%s: step %d, loss = (%.5f; %.5f, %.5f)(%.1f data/s; %.3f s/bch)'
            print((format_str % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), step, loss_total_val, 0.0, 0.0, examples_per_sec, sec_per_batch)))
        if ((step % 20) == 0):
            summary_str = sess.run(summary_op)
            summary_writer.add_summary(summary_str, global_step=step)
        if (((step % 1000) == 0) or (step == self.max_steps)):
            checkpoint_path = os.path.join(self.train_dir, 'checkpoints')
            self.save(sess, checkpoint_path, step)
def train(self):

    def get_optimizer(loss, global_step=None, var_list=None, is_gradient_clip=False):
        train_op = tf.train.AdamOptimizer(self.lr)
        if is_gradient_clip:
            grads_and_vars = train_op.compute_gradients(loss, var_list=var_list)
            unchanged_gvs = [(grad, var) for (grad, var) in grads_and_vars if (not ('LSTM' in var.name))]
            rnn_grad = [grad for (grad, var) in grads_and_vars if ('LSTM' in var.name)]
            rnn_var = [var for (grad, var) in grads_and_vars if ('LSTM' in var.name)]
            (capped_grad, _) = tf.clip_by_global_norm(rnn_grad, clip_norm=3)
            capped_gvs = list(zip(capped_grad, rnn_var))
            train_op = train_op.apply_gradients(grads_and_vars=(capped_gvs + unchanged_gvs), global_step=global_step)
        else:
            train_op = train_op.minimize(loss, global_step, var_list)
        return train_op
    global_step = tf.Variable(initial_value=0, dtype=tf.int32, trainable=False)
    self.global_step = global_step
    self.build_model()
    self.lr = tf.train.polynomial_decay(self.learning_rate, global_step, self.max_steps, end_learning_rate=0.0, power=0.3)
    tf.summary.scalar('learning_rate', self.lr)
    train_gnet = get_optimizer(self.loss_total, global_step, self.all_vars)
    gpu_options = tf.GPUOptions(allow_growth=True)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
    self.sess = sess
    sess.run(tf.global_variables_initializer())
    self.saver = tf.train.Saver(max_to_keep=50, keep_checkpoint_every_n_hours=1)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    summary_op = tf.summary.merge_all()
    summary_writer = tf.summary.FileWriter(self.train_dir, sess.graph, flush_secs=30)
    for step in xrange(sess.run(global_step), (self.max_steps + 1)):
        start_time = time.time()
        (_, loss_total_val) = sess.run([train_gnet, self.loss_total])
        duration = (time.time() - start_time)
        assert (not np.isnan(loss_total_val)), 'Model diverged with loss = NaN'
        if ((step % 5) == 0):
            num_examples_per_step = self.batch_size
            examples_per_sec = (num_examples_per_step / duration)
            sec_per_batch = float(duration)
            format_str = '%s: step %d, loss = (%.5f; %.5f, %.5f)(%.1f data/s; %.3f s/bch)'
            print((format_str % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), step, loss_total_val, 0.0, 0.0, examples_per_sec, sec_per_batch)))
        if ((step % 20) == 0):
            summary_str = sess.run(summary_op)
            summary_writer.add_summary(summary_str, global_step=step)
        if (((step % 1000) == 0) or (step == self.max_steps)):
            checkpoint_path = os.path.join(self.train_dir, 'checkpoints')
            self.save(sess, checkpoint_path, step)