I have used TensorFlow before for training a model to recognize images (something similar to this example: https://github.com/tensorflow/models/tree/master/inception). I was trying to do something similar with inception-v4 from scratch (using the model from https://github.com/tensorflow/models). But it was not working, it wasn't learning, so I decided to reduce my code and simplify everything. The model is a hidden layer with the same neurons as inputs and what the model has to learn is that input=output. Seems pretty simple and easy, but the results of my code are far from good.I have tried different things: sigmoid cross entropy instead of softmax cross entropy, other optimizers, different learning rates, models more comples with more neuros and more layers, reduce_sum instead of reduce mean for the loss ... But the only times I got good results (>90%) it seems because the model falls into a local minimum and I am not able to replicate them. Usually I see 70% precision with a high loss (>=0.7).My conclusion is that there is something wrong in my code that I am not able to spot and that is the reason the learning process is slow and get stuck at high losses. Otherwise the code with inception-v4 for images will at least learn something and don't get the loss stuck at the same value all the time.import osimport randomimport shutilimport numpy as npimport tensorflow as tfimport tensorflow.contrib.layers as layers# parametersmax_steps = 10000batch_size = 64output_size = 10summary_path = './summary'with tf.Graph().as_default() as graph:    shape = [batch_size, output_size]    inputs = tf.placeholder(dtype=tf.float32, shape=shape, name='inputs')    outputs = tf.placeholder(dtype=tf.float32, shape=shape, name='outputs')    global_step = tf.Variable(0, name='global_step', trainable=False)    logits = layers.fully_connected(inputs, output_size)    learning_rate = tf.train.exponential_decay(0.1, global_step, max_steps / 3, 0.1,                                               name='learning_rate')    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, outputs, name='loss'))    optimizer = tf.train.GradientDescentOptimizer(learning_rate)    train_op = optimizer.minimize(loss, global_step=global_step)    init_op = tf.global_variables_initializer()    input_label = tf.argmax(inputs, axis=1) # vector with the input labels    output_label = tf.argmax(logits, axis=1) # vector with the predicted labels    tf.summary.scalar('loss', loss)    tf.summary.scalar('learning_rate', learning_rate)    summary_op = tf.summary.merge_all()# prepare summaryif os.path.exists(summary_path):    shutil.rmtree(summary_path)os.makedirs(summary_path)summary_writer = tf.train.SummaryWriter(summary_path, graph=graph)with tf.Session(graph=graph).as_default() as session:    session.run(init_op)    to_run = {'step': global_step, 'loss': loss, 'logits': logits, 'train_op': train_op,              'learning_rate': learning_rate, 'summary': summary_op,              'in_label': input_label, 'out_label': output_label}    step = 0    while step < max_steps:        # fake input        data = np.zeros(shape, dtype=float)        for i in range(batch_size):            data[i, random.randint(0, output_size - 1)] = 1.0        # run        results = session.run(to_run, feed_dict={inputs: data, outputs: data})        # print data        step, in_label, out_label = results['step'],  results['in_label'], results['out_label']        precision = (in_label == out_label).sum() * 100.0 / batch_size        print('step: {0}/{1}  learning rate: {2:.4f}  loss: {3:.4f}  precision: {4:.1f}%'              .format(step, max_steps, results['learning_rate'], results['loss'], precision))        summary_writer.add_summary(results['summary'], step)    # print last batch    print(data)    print(results['logits'])    print(in_label)    print(out_label)    print('precision {}%'.format(precision))summary_writer.flush()
TensorFlow doesn't learn when input=output (or probably I am missing something)
python;neural network;deep learning;tensorflow