<jupyter_start><jupyter_text>Machine Learning<jupyter_code>num_unrollings=10

batch_size=32
hidden_size = 64


input_size = len(X_fields)
last_input_size = len(X_no_y_fields)
output_size = len(y_fields)


graph_rnn = tf.Graph()
with graph_rnn.as_default():
    
    # Inputs
    dropout_placeholder = tf.placeholder(tf.float32, name='Dropout')
    # Input data.
    train_inputs = list()
    train_inputs_last = list()
    train_labels = list()
    for _ in range(num_unrollings):
        train_inputs.append(tf.placeholder(tf.float32, shape=[batch_size,input_size]))  # I - I - I - I
        train_labels.append(tf.placeholder(tf.float32, shape=[batch_size,output_size])) #     O - O - O - O
        train_inputs_last.append(tf.placeholder(tf.float32, shape=[batch_size,last_input_size]))
    
    input_placeholder = tf.placeholder(
        tf.float64, shape=[batch_size, input_size], name='Input')
    labels_placeholder = tf.placeholder(
        tf.float64, shape=[batch_size, output_size], name='Target')
    
    with tf.variable_scope('InputDropout'):
        train_inputs = [tf.nn.dropout(x, dropout_placeholder) for x in train_inputs]
    # RNN
    rnn_outputs = []
    with tf.variable_scope('RNN') as scope:
        print("Shape one RNN input", train_inputs[0].get_shape())
        initial_state = tf.zeros([batch_size, hidden_size])
        state = initial_state
        
        for tstep, current_input in enumerate(train_inputs):
            if tstep > 0:
                scope.reuse_variables()
            RNN_H = tf.get_variable('HMatrix', [hidden_size, hidden_size], initializer=tf.truncated_normal_initializer(stddev=0.1))
            RNN_I = tf.get_variable('IMatrix', [input_size, hidden_size], initializer=tf.truncated_normal_initializer(stddev=0.1))
            RNN_b = tf.get_variable('B', [hidden_size], initializer=tf.constant_initializer(0.0))
            
            state = tf.nn.sigmoid(tf.matmul(state, RNN_H) + tf.matmul(current_input, RNN_I) + RNN_b)
            rnn_outputs.append(state)
        final_state = rnn_outputs[-1]
        print("Shape one RNN", final_state.get_shape())
    
    with tf.variable_scope('Last'):
        print("Shape one Last input", train_inputs_last[0].get_shape())
        W_last = tf.get_variable('W_last', [last_input_size, hidden_size], initializer=tf.truncated_normal_initializer(stddev=0.1))
        b_last = tf.get_variable('b_last', [hidden_size], initializer=tf.constant_initializer(0.0))
        tf.add_to_collection('total_loss', tf.nn.l2_loss(W_last))
        
        features_last = [tf.matmul(x, W_last) + b_last for x in train_inputs_last]
        print("Shape one Last", features_last[0].get_shape())
    
    #Projection
    with tf.variable_scope('Projection'):
        concatenated = [tf.concat(1, (features_last[i], rnn_outputs[i])) for i in range(num_unrollings)]
        print("Shape one concatenated (RNN + Last)", concatenated[0].get_shape())
        
        W_proj = tf.get_variable('W_proj', [hidden_size*2, output_size], initializer=tf.truncated_normal_initializer(stddev=0.1))
        b_proj = tf.get_variable('b_proj', [output_size], initializer=tf.constant_initializer(0.0))
        tf.add_to_collection('total_loss', tf.nn.l2_loss(W_proj))
        
        outputs = [tf.matmul(o, W_proj) + b_proj for o in concatenated]
        print("Shape one output", outputs[0].get_shape())
    
    cross_entropy = tf.reduce_mean(
      tf.nn.sigmoid_cross_entropy_with_logits(tf.concat(0, outputs), tf.concat(0, train_labels)))
    tf.add_to_collection('total_loss', cross_entropy)
    
    loss = tf.add_n(tf.get_collection('total_loss'))
    
    optimizer = tf.train.AdamOptimizer(0.0001).minimize(loss)
    
    predictions = [tf.nn.sigmoid(o) for o in outputs]
    
    auc = tf.contrib.metrics.streaming_auc(predictions[-1], train_labels[-1])
    
gc.collect() 
num_steps = len(csv)//batch_size//num_unrollings
print("steps", num_steps)

with tf.Session(graph=graph_rnn) as session:
    
    init_op_all = tf.initialize_all_variables()
    init_op_local = tf.initialize_local_variables()
    saver = tf.train.Saver()
    init_op_all.run()
    init_op_local.run()
    print('Initialized')
    
    g = BatchGenerator(group_idx, batch_size, num_unrollings+1)
    
    for i in range(num_steps):
        feed_dict = {dropout_placeholder: 1}
        
        batch_idx = next(g)
        for _i in range(num_unrollings):
            _X = csv.iloc[batch_idx[:,_i]][X_fields].values
            _y = csv.iloc[batch_idx[:,_i+1]][y_fields].values
            _X_no_y = csv.iloc[batch_idx[:,_i+1]][X_no_y_fields].values
            feed_dict[train_inputs[_i]] = _X
            feed_dict[train_inputs_last[_i]] = _X_no_y
            feed_dict[train_labels[_i]] = _y
        
        _ , state, y_pred, l, a = session.run(
            [optimizer, final_state, predictions[-1], loss, auc], feed_dict=feed_dict)
        if i%100 == 0:
            print(i, l, a)
        if i%10000 == 0:
            print(y_pred[0])
            print(_y[0])
    save_path = saver.save(session, "santander.ckpt")<jupyter_output>steps 42561
Initialized
0 44.8831 (0.0, 0.29694676)
[ 0.75921685  0.88538045  0.39598748  0.87144881  0.57983309  0.84354663
  0.32116571  0.8620556   0.77205902  0.87459809  0.83310044  0.54807383
  0.32306722  0.1362917   0.40194657  0.4615061   0.30799463  0.29567885
  0.44778401  0.40670407  0.82711935  0.84360921  0.37121505  0.30135393]
[0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
100 39.4164 (0.3255057, 0.32567453)
200 34.6528 (0.37174982, 0.37192219)
300 30.484 (0.38403448, 0.3844007)
400 26.8387 (0.41256714, 0.4124954)
500 23.6293 (0.42655647, 0.42667821)
600 20.8008 (0.4379406, 0.43809837)
700 18.3164 (0.44802818, 0.44812334)
800 16.1094 (0.46167871, 0.46178195)
900 14.1766 (0.47644117, 0.47650564)
1000 12.4762 (0.48841333, 0.48854458)
1100 10.9651 (0.50203037, 0.50218689)
1200 9.64948 (0.51974225, 0.51999235)
1300 8.46707 (0.54199684, 0.54220033)
1400 7.44058 (0.56156391, 0.56173557)
1500 6.53144 (0.57458305, 0.5745846)
1600 5.71062 (0.58420074, 0.58420038)
1700 5.01161[...]
{'fraction_non_alphanumeric': 0.12846595570139457, 'fraction_numerical': 0.12666119770303527, 'mean_word_length': 3.26592022393282, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '425319', 'license': 'MIT', 'n_tokens_mistral': 2762, 'n_tokens_neox': 2303, 'n_words': 400, 'path': 'kaggle/003_santander/003_santander.ipynb', 'repo_name': 'urakozz/ml-competitions', 'size': '6095'}