<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import scipy.io as cpio\n",
    "import os\n",
    "import time\n",
    "\n",
    "# Define the folder that contains all data files\n",
    "# Each data file contains the variables:\n",
    "#    s: The spectrogram [size = 513 x time_steps]\n",
    "#    f: Frequencies [size = 513]\n",
    "#    t: Time steps\n",
    "#    labels: The tagging data [size = time_steps]\n",
    "data_directory = '/Users/yardenc/Documents/Experiments/Imaging/CanaryData/lrb853_15/mat'\n",
    "# This folder must also contain a matlab file 'file_list.mat' with cell array 'keys' that holds the data file names\n",
    "data_list = cpio.loadmat(data_directory + '/file_list.mat')\n",
    "number_of_files = len(data_list['keys'][0])\n",
    "# The folder for saving training checkpoints\n",
    "training_records_dir = '/Users/yardenc/Documents/Experiments/Imaging/CanaryData/lrb853_15/training_records'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "input_vec_size = lstm_size = 513\n",
    "batch_size = 10\n",
    "n_lstm_layers = 2\n",
    "n_syllables = 28 #including zero\n",
    "learning_rate = 0.001\n",
    "n_max_iter = 10001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The inference graph\n",
    "def label_inference_graph(spectrogram, num_hidden, num_layers, seq_length):\n",
    "    # First the dynamic bi-directional, multi-layered LSTM\n",
    "    with tf.name_scope('biRNN'): \n",
    "        with tf.variable_scope('fwd'):\n",
    "            lstm_f1 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            #lstm_f2 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            #lstm_f3 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            #cells_f = tf.contrib.rnn.MultiRNNCell([lstm_f1,lstm_f2,lstm_f3], state_is_tuple=True)\n",
    "        with tf.variable_scope('bck'):\n",
    "            lstm_b1 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            #lstm_b2 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            #lstm_b3 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            #cells_b = tf.contrib.rnn.MultiRNNCell([lstm_b1,lstm_b2,lstm_b3], state_is_tuple=True)\n",
    "        outputs, _states = tf.nn.bidirectional_dynamic_rnn(lstm_f1,lstm_b1, spectrogram, time_major=False, dtype=tf.float32,sequence_length=seq_length)\n",
    "    # Second, projection on the number of syllables creates logits \n",
    "    with tf.name_scope('Projection'):\n",
    "        W_f = tf.Variable(tf.random_normal([num_hidden, n_syllables]))\n",
    "        W_b = tf.Variable(tf.random_normal([num_hidden, n_syllables]))\n",
    "        bias = tf.Variable(tf.random_normal([n_syllables]))\n",
    "    logits = tf.matmul(outputs[0][:,-1,:],W_f) + bias + tf.matmul(outputs[1][:,-1,:],W_b)# for a,b in zip(range(batch_size),range(batch_size))]\n",
    "    return logits,outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The training graph. Calculate cross entropy and loss function\n",
    "def training_graph(logits, labels, rate):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits = logits,labels = labels, name='xentropy')\n",
    "    cost = tf.reduce_mean(xentropy, name='cost')\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate = rate)\n",
    "    global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "    train_op = optimizer.minimize(cost, global_step=global_step)\n",
    "    return train_op, cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct the full graph and add saver\n",
    "full_graph = tf.Graph()\n",
    "with full_graph.as_default():\n",
    "    # Generate placeholders for the spectrograms and labels.\n",
    "    X = tf.placeholder(\"float\", [None,None,input_vec_size], name = \"Xdata\") # holds spectrograms\n",
    "    Y = tf.placeholder(\"int32\",[None],name = \"Ylabels\") # holds labels\n",
    "    lng = tf.placeholder(\"int32\",name = \"nSteps\") # holds the sequence length\n",
    "                                    \n",
    "    tf.add_to_collection(\"specs\", X)  # Remember this Op.\n",
    "    tf.add_to_collection(\"labels\", Y)  # Remember this Op.\n",
    "    tf.add_to_collection(\"lng\", lng)  # Remember this Op.\n",
    "\n",
    "    # Build a Graph that computes predictions from the inference model.\n",
    "    logits,outputs = label_inference_graph(X, lstm_size, n_lstm_layers, lng)\n",
    "    tf.add_to_collection(\"logits\", logits)  # Remember this Op.\n",
    "\n",
    "    # Add to the Graph the Ops that calculate and apply gradients.\n",
    "    train_op, cost = training_graph(logits, Y, learning_rate) \n",
    "    \n",
    "    # Add the variable initializer Op.\n",
    "    init = tf.global_variables_initializer() #initialize_all_variables()\n",
    "\n",
    "    # Create a saver for writing training checkpoints.\n",
    "    saver = tf.train.Saver(max_to_keep = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#debug\n",
    "file_num = 0\n",
    "fname = data_list['keys'][0][file_num][0][0:-3]+'mat'\n",
    "data = cpio.loadmat(data_directory + '/' + fname)\n",
    "data1 = np.transpose(data['s'])\n",
    "intY = data['labels'][0]\n",
    "for file_num in range(number_of_files-1):\n",
    "    # load current training file\n",
    "    fname = data_list['keys'][0][file_num+1][0][0:-3]+'mat'\n",
    "    bdata = cpio.loadmat(data_directory + '/' + fname)\n",
    "    bdata1 = np.transpose(bdata['s'])\n",
    "    bintY = bdata['labels'][0]\n",
    "    data1 = np.concatenate((data1,bdata1),axis = 0)\n",
    "    intY = np.concatenate((intY,bintY),axis = 0)\n",
    "temp_n = len(intY)/batch_size\n",
    "data1 = data1[0:temp_n*batch_size].reshape((batch_size,temp_n,-1))\n",
    "intY = intY[0:temp_n*batch_size].reshape((batch_size,-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#debug\n",
    "with tf.Session(graph=full_graph,config = tf.ConfigProto(\n",
    "    intra_op_parallelism_threads = batch_size)) as sess:\n",
    "    sess.run(init)\n",
    "    # Start the training loop.\n",
    "    costs = []\n",
    "    step = 1\n",
    "    # Go over all training files\n",
    "    \n",
    "    iternum = 0\n",
    "    d = {X: data1[:,iternum:iternum+30,:] ,Y: intY[:,iternum+30] ,lng:[30]*batch_size}\n",
    "    _logits,_outputs = sess.run((logits,outputs),feed_dict = d)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 30, 513)\n",
      "(3, 513)\n",
      "(30, 28)\n"
     ]
    }
   ],
   "source": [
    "# debug\n",
    "print _outputs[0].shape\n",
    "print _outputs[0][:,-1,:].shape\n",
    "print _logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 6266, 513) 5896\n",
      "[1, 1099, 4.2483702]\n",
      "[2, 5690, 4.7006426]\n",
      "[3, 3308, 2.3439364]\n",
      "[4, 4160, 5.7856154]\n",
      "[5, 4521, 2.461616]\n",
      "[6, 2633, 2.5212841]\n",
      "[7, 1572, 3.165504]\n",
      "[8, 1403, 2.184983]\n",
      "[9, 1866, 2.0820374]\n",
      "[10, 5641, 2.4901397]\n",
      "[11, 5235, 2.4759438]\n",
      "[12, 1685, 4.4641523]\n",
      "[13, 2872, 2.408258]\n",
      "[14, 1144, 2.7526615]\n",
      "[15, 3197, 2.6022811]\n",
      "[16, 961, 2.0234663]\n",
      "[17, 922, 1.8030649]\n",
      "[18, 3177, 2.3340554]\n",
      "[19, 5545, 0.53948194]\n",
      "[20, 1755, 3.0438879]\n",
      "[21, 3674, 1.150262]\n",
      "[22, 504, 2.0593922]\n",
      "[23, 3822, 2.3870995]\n",
      "[24, 4887, 2.1472015]\n",
      "[25, 5575, 0.98993462]\n",
      "[26, 1929, 1.6692593]\n",
      "[27, 3569, 1.0253913]\n",
      "[28, 3218, 1.2561661]\n",
      "[29, 1931, 1.693516]\n",
      "[30, 5852, 1.2745528]\n",
      "[31, 714, 0.82061148]\n",
      "[32, 5686, 0.84944791]\n",
      "[33, 1966, 0.77201557]\n",
      "[34, 171, 1.4239548]\n",
      "[35, 402, 2.2866788]\n",
      "[36, 2059, 1.9561851]\n",
      "[37, 2781, 0.36511284]\n",
      "[38, 4593, 3.5399048]\n",
      "[39, 3254, 1.2125297]\n",
      "[40, 4189, 1.9695721]\n",
      "[41, 2911, 0.681234]\n",
      "[42, 1670, 1.2766631]\n",
      "[43, 1184, 0.66253489]\n",
      "[44, 5499, 1.1228473]\n",
      "[45, 4660, 1.1882523]\n",
      "[46, 3839, 0.34203702]\n",
      "[47, 3032, 2.3019822]\n",
      "[48, 3147, 0.6117211]\n",
      "[49, 1813, 1.2307222]\n",
      "[50, 4520, 0.55905318]\n",
      "[51, 3405, 0.71087515]\n",
      "[52, 3430, 0.54497379]\n",
      "[53, 5518, 1.6505125]\n",
      "[54, 3646, 0.48300529]\n",
      "[55, 4957, 0.84135616]\n",
      "[56, 114, 0.72583139]\n",
      "[57, 3110, 0.68485284]\n",
      "[58, 2259, 1.5662203]\n",
      "[59, 1774, 1.7654114]\n",
      "[60, 739, 1.7347114]\n",
      "[61, 681, 0.83164853]\n",
      "[62, 2759, 1.0321684]\n",
      "[63, 5282, 0.77524674]\n",
      "[64, 2047, 0.69161671]\n",
      "[65, 791, 1.8794663]\n",
      "[66, 4417, 0.75301301]\n",
      "[67, 4499, 0.25130376]\n",
      "[68, 2241, 1.2955691]\n",
      "[69, 3178, 1.8557158]\n",
      "[70, 1823, 0.58212799]\n",
      "[71, 998, 0.64755768]\n",
      "[72, 2906, 0.84933221]\n",
      "[73, 3794, 1.7092375]\n",
      "[74, 4656, 0.22134785]\n",
      "[75, 722, 0.71812499]\n",
      "[76, 3418, 0.58940774]\n",
      "[77, 2001, 0.63717109]\n",
      "[78, 3654, 1.7937298]\n",
      "[79, 1202, 0.62768495]\n",
      "[80, 4042, 0.73183262]\n",
      "[81, 4798, 2.4900792]\n",
      "[82, 1072, 1.9436343]\n",
      "[83, 1343, 0.79695845]\n",
      "[84, 1247, 0.91509533]\n",
      "[85, 1616, 1.2775673]\n",
      "[86, 4703, 1.3808317]\n",
      "[87, 5705, 1.0097774]\n",
      "[88, 3716, 0.80575418]\n",
      "[89, 5720, 1.2930899]\n",
      "[90, 1475, 1.2582767]\n",
      "[91, 2388, 1.112993]\n",
      "[92, 5721, 1.0273987]\n",
      "[93, 3065, 0.91578323]\n",
      "[94, 2057, 0.92186606]\n",
      "[95, 5061, 1.6695429]\n",
      "[96, 1694, 0.52347326]\n",
      "[97, 1999, 0.06233732]\n",
      "[98, 4361, 2.830827]\n",
      "[99, 3807, 1.1048255]\n",
      "[100, 651, 1.7566061]\n",
      "[101, 2743, 0.8306303]\n",
      "[102, 231, 1.057826]\n",
      "[103, 3973, 0.30162615]\n",
      "[104, 1535, 0.38877279]\n",
      "[105, 2945, 0.76942682]\n",
      "[106, 4966, 1.5503477]\n",
      "[107, 2664, 0.22045031]\n",
      "[108, 5800, 1.1883607]\n",
      "[109, 3187, 1.0662024]\n",
      "[110, 3406, 0.36077261]\n",
      "[111, 5620, 2.0088871]\n",
      "[112, 5607, 0.4862102]\n",
      "[113, 4822, 0.95953673]\n",
      "[114, 3945, 1.4007645]\n",
      "[115, 3588, 0.41805497]\n",
      "[116, 2343, 0.61257857]\n",
      "[117, 4650, 1.2188551]\n",
      "[118, 4516, 0.907242]\n",
      "[119, 1512, 1.5608472]\n",
      "[120, 1113, 1.594229]\n",
      "[121, 5736, 0.35312468]\n",
      "[122, 3334, 0.079681635]\n",
      "[123, 1377, 1.4435952]\n",
      "[124, 4673, 0.55077785]\n",
      "[125, 2889, 0.67870748]\n",
      "[126, 2600, 0.31388503]\n",
      "[127, 2142, 0.59597635]\n",
      "[128, 2703, 1.739712]\n",
      "[129, 3299, 1.585441]\n",
      "[130, 5822, 1.137571]\n",
      "[131, 3104, 2.1606636]\n",
      "[132, 1459, 0.3220517]\n",
      "[133, 2956, 1.0957448]\n",
      "[134, 244, 0.81816709]\n",
      "[135, 3845, 0.11765306]\n",
      "[136, 1752, 0.66869843]\n",
      "[137, 2839, 1.7644733]\n",
      "[138, 304, 0.58144218]\n",
      "[139, 987, 2.3107414]\n",
      "[140, 3151, 1.1230924]\n",
      "[141, 5246, 0.26293159]\n",
      "[142, 4406, 1.379163]\n",
      "[143, 4204, 1.7068913]\n",
      "[144, 4359, 1.4086485]\n",
      "[145, 3951, 0.70551026]\n",
      "[146, 3990, 0.38408524]\n",
      "[147, 3195, 0.45905119]\n",
      "[148, 5597, 0.66843814]\n",
      "[149, 5464, 1.3519996]\n",
      "[150, 2716, 0.76328951]\n",
      "[151, 1170, 1.6837721]\n",
      "[152, 4555, 0.3864333]\n",
      "[153, 5257, 0.56942326]\n",
      "[154, 4800, 1.2272617]\n",
      "[155, 4244, 1.6821473]\n",
      "[156, 3320, 0.91891229]\n",
      "[157, 4081, 0.047415875]\n",
      "[158, 1748, 1.1771311]\n",
      "[159, 2705, 1.3796231]\n",
      "[160, 4262, 0.40870953]\n",
      "[161, 4438, 1.1938447]\n",
      "[162, 2808, 2.1695683]\n",
      "[163, 374, 0.81382215]\n",
      "[164, 629, 0.92803514]\n",
      "[165, 321, 0.33773237]\n",
      "[166, 39, 3.0752032]\n",
      "[167, 108, 1.7882004]\n",
      "[168, 4172, 0.73685235]\n",
      "[169, 5002, 1.3608745]\n",
      "[170, 5424, 1.1880319]\n",
      "[171, 5162, 0.76077932]\n",
      "[172, 3285, 0.39627835]\n",
      "[173, 5044, 1.0336157]\n",
      "[174, 2828, 0.52247578]\n",
      "[175, 3811, 1.2183011]\n",
      "[176, 3604, 0.49886528]\n",
      "[177, 4574, 1.0582141]\n",
      "[178, 2218, 0.99527806]\n",
      "[179, 5774, 1.7818944]\n",
      "[180, 4732, 0.67651868]\n",
      "[181, 4821, 1.1025653]\n",
      "[182, 5122, 0.76655596]\n",
      "[183, 3123, 0.7134487]\n",
      "[184, 625, 0.79466665]\n",
      "[185, 5124, 0.061476994]\n",
      "[186, 2814, 0.59954679]\n",
      "[187, 1600, 0.091705136]\n",
      "[188, 3472, 0.53338635]\n",
      "[189, 3862, 0.73569053]\n",
      "[190, 3004, 1.3208921]\n",
      "[191, 2046, 1.0748885]\n",
      "[192, 1871, 0.80845022]\n",
      "[193, 2351, 0.42919683]\n",
      "[194, 4728, 2.1518908]\n",
      "[195, 4476, 0.50665236]\n",
      "[196, 2859, 0.45944524]\n",
      "[197, 1243, 0.64081937]\n",
      "[198, 3836, 1.4948714]\n",
      "[199, 1130, 1.3746246]\n",
      "[200, 4517, 0.19705574]\n"
     ]
    }
   ],
   "source": [
    "# Train and save checkpoint at the end of each file.\n",
    "with tf.Session(graph=full_graph,config = tf.ConfigProto(\n",
    "    intra_op_parallelism_threads = batch_size)) as sess:\n",
    "    # Run the Op to initialize the variables.\n",
    "    sess.run(init)\n",
    "    # Start the training loop.\n",
    "    costs = []\n",
    "    step = 1\n",
    "    # Go over all training files\n",
    "    file_num = 0\n",
    "    fname = data_list['keys'][0][file_num][0][0:-3]+'mat'\n",
    "    data = cpio.loadmat(data_directory + '/' + fname)\n",
    "    data1 = np.transpose(data['s'])\n",
    "    intY = data['labels'][0]\n",
    "    for file_num in range(number_of_files-1):\n",
    "        # load current training file\n",
    "        fname = data_list['keys'][0][file_num+1][0][0:-3]+'mat'\n",
    "        bdata = cpio.loadmat(data_directory + '/' + fname)\n",
    "        bdata1 = np.transpose(bdata['s'])\n",
    "        bintY = bdata['labels'][0]\n",
    "        data1 = np.concatenate((data1,bdata1),axis = 0)\n",
    "        intY = np.concatenate((intY,bintY),axis = 0)\n",
    "    temp_n = len(intY)/batch_size\n",
    "    data1 = data1[0:temp_n*batch_size].reshape((batch_size,temp_n,-1))\n",
    "    intY = intY[0:temp_n*batch_size].reshape((batch_size,-1))\n",
    "    iter_order = np.random.permutation(data1.shape[1]-370)\n",
    "    if (len(iter_order) > n_max_iter):\n",
    "        iter_order = iter_order[0:n_max_iter]\n",
    "    print data1.shape, len(iter_order)\n",
    "    for iternum in iter_order:\n",
    "        d = {X: data1[:,iternum:iternum+100,:] ,Y: intY[:,iternum+100] ,lng:[100]*batch_size}\n",
    "        _cost,_ = sess.run((cost,train_op),feed_dict = d)\n",
    "        costs.append(_cost)\n",
    "        print([step,iternum,_cost])\n",
    "        step = step + 1\n",
    "            \n",
    "        if (step % 1000 == 0):    \n",
    "            checkpoint_file = os.path.join(training_records_dir, 'checkpoint')\n",
    "            saver.save(sess, checkpoint_file, global_step=step)\n",
    "            print np.mean(costs[-500:-1])\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 3, 513)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1[:,1:4,:].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate training set from a saved checkpoint\n",
    "with tf.Session(graph=tf.Graph()) as sess:\n",
    "    saver = tf.train.import_meta_graph(\n",
    "        os.path.join(training_records_dir, \"checkpoint-5001.meta\"))\n",
    "    saver.restore(\n",
    "        sess, os.path.join(training_records_dir, \"checkpoint-5001\"))\n",
    "\n",
    "    # Retrieve the Ops we 'remembered'.\n",
    "    logits = tf.get_collection(\"logits\")[0]\n",
    "    X = tf.get_collection(\"specs\")[0]\n",
    "    Y = tf.get_collection(\"labels\")[0]\n",
    "    lng = tf.get_collection(\"lng\")[0]\n",
    "    \n",
    "    # Add an Op that chooses the top k predictions.\n",
    "    eval_op = tf.nn.top_k(logits)\n",
    "    \n",
    "    # Run evaluation.\n",
    "    errors = []\n",
    "    for file_num in range(number_of_files):\n",
    "        # load current training file\n",
    "        fname = data_list['keys'][0][file_num][0][0:-3]+'mat'\n",
    "        data = cpio.loadmat(data_directory + '/' + fname)\n",
    "        data1 = np.transpose(data['s'])\n",
    "        intY = data['labels'][0]\n",
    "        d = {X: data1 ,Y: intY ,lng:[len(intY)]}\n",
    "        pred = sess.run(eval_op,feed_dict = d)\n",
    "        errors.append(np.abs(np.squeeze(pred[1])-intY) != 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.34048412813451223,\n",
       " 0.35495445763179684,\n",
       " 0.093001841620626149,\n",
       " 0.47276603993021904,\n",
       " 0.032034853921066121,\n",
       " 0.46023308116949502,\n",
       " 0.4215967834577829,\n",
       " 0.28068693693693691]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Results on training set\n",
    "[np.mean(err) for err in errors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",<commit_msg>small update<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import scipy.io as cpio\n",
    "import os\n",
    "import time\n",
    "\n",
    "# Define the folder that contains all data files\n",
    "# Each data file contains the variables:\n",
    "#    s: The spectrogram [size = 513 x time_steps]\n",
    "#    f: Frequencies [size = 513]\n",
    "#    t: Time steps\n",
    "#    labels: The tagging data [size = time_steps]\n",
    "data_directory = '/Users/yardenc/Documents/Experiments/Imaging/CanaryData/lrb853_15/mat'\n",
    "# This folder must also contain a matlab file 'file_list.mat' with cell array 'keys' that holds the data file names\n",
    "data_list = cpio.loadmat(data_directory + '/file_list.mat')\n",
    "number_of_files = len(data_list['keys'][0])\n",
    "# The folder for saving training checkpoints\n",
    "training_records_dir = '/Users/yardenc/Documents/Experiments/Imaging/CanaryData/lrb853_15/training_records'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "input_vec_size = lstm_size = 513\n",
    "batch_size = 101\n",
    "n_lstm_layers = 2\n",
    "n_syllables = 28 #including zero\n",
    "learning_rate = 0.001\n",
    "n_max_iter = 10001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The inference graph\n",
    "def label_inference_graph(spectrogram, num_hidden, num_layers, seq_length):\n",
    "    # First the dynamic bi-directional, multi-layered LSTM\n",
    "    with tf.name_scope('biRNN'): \n",
    "        with tf.variable_scope('fwd'):\n",
    "            lstm_f1 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            lstm_f2 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            #lstm_f3 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            cells_f = tf.contrib.rnn.MultiRNNCell([lstm_f1,lstm_f2], state_is_tuple=True)\n",
    "        with tf.variable_scope('bck'):\n",
    "            lstm_b1 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            lstm_b2 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            #lstm_b3 = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0, state_is_tuple=True,reuse=None)\n",
    "            cells_b = tf.contrib.rnn.MultiRNNCell([lstm_b1,lstm_b2], state_is_tuple=True)\n",
    "        outputs, _states = tf.nn.bidirectional_dynamic_rnn(cells_f,cells, spectrogram, time_major=False, dtype=tf.float32,sequence_length=seq_length)\n",
    "    # Second, projection on the number of syllables creates logits \n",
    "    with tf.name_scope('Projection'):\n",
    "        W_f = tf.Variable(tf.random_normal([num_hidden, n_syllables]))\n",
    "        W_b = tf.Variable(tf.random_normal([num_hidden, n_syllables]))\n",
    "        bias = tf.Variable(tf.random_normal([n_syllables]))\n",
    "    expr1 = tf.unstack(outputs[0],axis=0,num=batch_size)\n",
    "    expr2 = tf.unstack(outputs[1],axis=0,num=batch_size)\n",
    "    #logits = [tf.matmul(outputs[0][:,a,:],W_f) + bias + tf.matmul(outputs[1][:,a,:],W_b) for a in range(seq_length[0])]\n",
    "    logits = tf.concat([tf.matmul(ex1,W_f) + bias + tf.matmul(ex2,W_b) for ex1,ex2 in zip(expr1,expr2)],0)\n",
    "    return logits,outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The training graph. Calculate cross entropy and loss function\n",
    "def training_graph(logits, lbls, rate, lng):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits = logits,labels = tf.concat(tf.unstack(lbls,axis=0,num=batch_size),0), name='xentropy') \n",
    "    cost = tf.reduce_mean(xentropy, name='cost')\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate = rate)\n",
    "    global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "    train_op = optimizer.minimize(cost, global_step=global_step)\n",
    "    return train_op, cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct the full graph and add saver\n",
    "full_graph = tf.Graph()\n",
    "with full_graph.as_default():\n",
    "    # Generate placeholders for the spectrograms and labels.\n",
    "    X = tf.placeholder(\"float\", [None,None,input_vec_size], name = \"Xdata\") # holds spectrograms\n",
    "    Y = tf.placeholder(\"int32\",[None,None],name = \"Ylabels\") # holds labels\n",
    "    lng = tf.placeholder(\"int32\",name = \"nSteps\") # holds the sequence length\n",
    "                                    \n",
    "    tf.add_to_collection(\"specs\", X)  # Remember this Op.\n",
    "    tf.add_to_collection(\"labels\", Y)  # Remember this Op.\n",
    "    tf.add_to_collection(\"lng\", lng)  # Remember this Op.\n",
    "\n",
    "    # Build a Graph that computes predictions from the inference model.\n",
    "    logits,outputs = label_inference_graph(X, lstm_size, n_lstm_layers, lng)\n",
    "    tf.add_to_collection(\"logits\", logits)  # Remember this Op.\n",
    "\n",
    "    # Add to the Graph the Ops that calculate and apply gradients.\n",
    "    train_op, cost = training_graph(logits, Y, learning_rate, lng) \n",
    "    \n",
    "    # Add the variable initializer Op.\n",
    "    init = tf.global_variables_initializer() #initialize_all_variables()\n",
    "\n",
    "    # Create a saver for writing training checkpoints.\n",
    "    saver = tf.train.Saver(max_to_keep = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(101, 620, 513) 250\n",
      "[1, 133, 4.7702093]\n",
      "[2, 58, 6.2990441]\n",
      "[3, 115, 3.60216]\n",
      "[4, 144, 3.3844957]\n",
      "[5, 204, 2.5358846]\n",
      "[6, 38, 1.7358085]\n",
      "[7, 230, 1.3929291]\n",
      "[8, 8, 1.1815116]\n",
      "[9, 157, 1.2652613]\n",
      "[10, 112, 1.2231426]\n",
      "[11, 114, 1.0071509]\n",
      "[12, 192, 0.71131742]\n",
      "[13, 47, 0.74443352]\n",
      "[14, 113, 0.63870019]\n",
      "[15, 163, 0.56102383]\n",
      "[16, 239, 0.56968153]\n",
      "[17, 76, 0.57435787]\n",
      "[18, 11, 0.57224011]\n",
      "[19, 71, 0.49658978]\n",
      "[20, 40, 0.48458454]\n",
      "[21, 249, 0.44854602]\n",
      "[22, 194, 0.34426579]\n",
      "[23, 102, 0.41441154]\n",
      "[24, 215, 0.33483991]\n",
      "[25, 214, 0.30831599]\n",
      "[26, 16, 0.39117071]\n",
      "[27, 110, 0.32987255]\n",
      "[28, 186, 0.26855922]\n",
      "[29, 187, 0.25798756]\n",
      "[30, 150, 0.27858585]\n",
      "[31, 15, 0.30982912]\n",
      "[32, 106, 0.24929562]\n",
      "[33, 93, 0.23264398]\n",
      "[34, 117, 0.21557473]\n",
      "[35, 125, 0.20205656]\n",
      "[36, 217, 0.26682132]\n",
      "[37, 123, 0.18328482]\n",
      "[38, 61, 0.22719581]\n",
      "[39, 178, 0.18667486]\n",
      "[40, 68, 0.19575034]\n",
      "[41, 122, 0.18648686]\n",
      "[42, 160, 0.19411753]\n",
      "[43, 151, 0.18842819]\n",
      "[44, 248, 0.21507673]\n",
      "[45, 200, 0.1641589]\n",
      "[46, 205, 0.1552988]\n",
      "[47, 12, 0.22369331]\n",
      "[48, 167, 0.15969832]\n",
      "[49, 99, 0.17037526]\n",
      "[50, 177, 0.14575]\n",
      "[51, 100, 0.1521447]\n",
      "[52, 232, 0.19278154]\n",
      "[53, 129, 0.14426674]\n",
      "[54, 24, 0.18383618]\n",
      "[55, 190, 0.13531369]\n",
      "[56, 228, 0.17150973]\n",
      "[57, 156, 0.13991863]\n",
      "[58, 159, 0.13148805]\n",
      "[59, 147, 0.14307968]\n",
      "[60, 152, 0.13057436]\n",
      "[61, 183, 0.11360308]\n",
      "[62, 218, 0.13624339]\n",
      "[63, 138, 0.13126263]\n",
      "[64, 171, 0.10817586]\n",
      "[65, 63, 0.16675507]\n",
      "[66, 118, 0.1143341]\n",
      "[67, 213, 0.12293345]\n",
      "[68, 165, 0.11221323]\n",
      "[69, 116, 0.10264083]\n",
      "[70, 227, 0.13454704]\n",
      "[71, 131, 0.116336]\n",
      "[72, 191, 0.10630144]\n",
      "[73, 162, 0.095552787]\n",
      "[74, 145, 0.10644536]\n",
      "[75, 74, 0.1537918]\n",
      "[76, 87, 0.11686125]\n",
      "[77, 17, 0.1628273]\n",
      "[78, 105, 0.10257861]\n",
      "[79, 233, 0.14896195]\n",
      "[80, 238, 0.13980426]\n",
      "[81, 203, 0.11868022]\n",
      "[82, 30, 0.1688607]\n",
      "[83, 65, 0.14501049]\n",
      "[84, 79, 0.11512406]\n",
      "[85, 41, 0.13521934]\n",
      "[86, 26, 0.14444587]\n",
      "[87, 89, 0.13150936]\n",
      "[88, 60, 0.10691481]\n",
      "[89, 109, 0.094244972]\n",
      "[90, 35, 0.12285835]\n",
      "[91, 231, 0.15054934]\n",
      "[92, 80, 0.11104804]\n",
      "[93, 181, 0.104765]\n",
      "[94, 75, 0.10860685]\n",
      "[95, 234, 0.12266133]\n",
      "[96, 220, 0.10753977]\n",
      "[97, 211, 0.098386489]\n",
      "[98, 23, 0.13873006]\n",
      "[99, 212, 0.091272429]\n",
      "0.118349\n",
      "[100, 173, 0.096515372]\n",
      "[101, 86, 0.11484595]\n",
      "[102, 46, 0.11047448]\n",
      "[103, 4, 0.1313546]\n",
      "[104, 5, 0.12106537]\n",
      "[105, 28, 0.10190769]\n",
      "[106, 90, 0.092716888]\n",
      "[107, 179, 0.093859114]\n",
      "[108, 154, 0.10136502]\n",
      "[109, 206, 0.098572582]\n",
      "[110, 20, 0.1111227]\n",
      "[111, 223, 0.097958595]\n",
      "[112, 130, 0.093774229]\n",
      "[113, 137, 0.10531956]\n",
      "[114, 44, 0.096277267]\n",
      "[115, 64, 0.096843891]\n",
      "[116, 21, 0.10283798]\n",
      "[117, 161, 0.089114904]\n",
      "[118, 103, 0.085466072]\n",
      "[119, 69, 0.087299228]\n",
      "[120, 29, 0.089322984]\n",
      "[121, 229, 0.10678829]\n",
      "[122, 83, 0.082997516]\n",
      "[123, 52, 0.087722532]\n",
      "[124, 198, 0.1028951]\n",
      "[125, 208, 0.085396469]\n",
      "[126, 57, 0.081023991]\n",
      "[127, 34, 0.090353429]\n",
      "[128, 94, 0.089015119]\n",
      "[129, 98, 0.079200573]\n",
      "[130, 97, 0.072531715]\n",
      "[131, 6, 0.11219291]\n",
      "[132, 146, 0.094961792]\n",
      "[133, 9, 0.10120615]\n",
      "[134, 62, 0.092802189]\n",
      "[135, 170, 0.085262418]\n",
      "[136, 136, 0.088747196]\n",
      "[137, 25, 0.087162919]\n",
      "[138, 226, 0.10154906]\n",
      "[139, 174, 0.086397439]\n",
      "[140, 53, 0.088041797]\n",
      "[141, 3, 0.11196838]\n",
      "[142, 207, 0.081592388]\n",
      "[143, 67, 0.091620333]\n",
      "[144, 55, 0.092814639]\n",
      "[145, 195, 0.097389989]\n",
      "[146, 37, 0.089645296]\n",
      "[147, 111, 0.08701545]\n",
      "[148, 39, 0.088827856]\n",
      "[149, 1, 0.13522352]\n",
      "[150, 180, 0.083798103]\n",
      "[151, 32, 0.087651387]\n",
      "[152, 235, 0.11664248]\n",
      "[153, 96, 0.12953235]\n",
      "[154, 241, 0.11222978]\n",
      "[155, 201, 0.092652962]\n",
      "[156, 88, 0.099219121]\n",
      "[157, 169, 0.09344615]\n",
      "[158, 236, 0.10519038]\n",
      "[159, 247, 0.10184075]\n",
      "[160, 127, 0.088119201]\n",
      "[161, 128, 0.087905906]\n",
      "[162, 120, 0.083181702]\n",
      "[163, 101, 0.082984038]\n",
      "[164, 196, 0.095967241]\n",
      "[165, 158, 0.089932136]\n",
      "[166, 50, 0.1040526]\n",
      "[167, 244, 0.10124219]\n",
      "[168, 33, 0.10110624]\n",
      "[169, 182, 0.084684089]\n",
      "[170, 237, 0.093763538]\n",
      "[171, 43, 0.089666896]\n",
      "[172, 242, 0.095209762]\n",
      "[173, 246, 0.081305943]\n",
      "[174, 188, 0.086291395]\n",
      "[175, 91, 0.081929639]\n",
      "[176, 140, 0.10040722]\n",
      "[177, 199, 0.086171798]\n",
      "[178, 70, 0.095234692]\n",
      "[179, 82, 0.081976086]\n",
      "[180, 219, 0.07912793]\n",
      "[181, 49, 0.085340396]\n",
      "[182, 73, 0.09195143]\n",
      "[183, 209, 0.081822656]\n",
      "[184, 185, 0.096604064]\n",
      "[185, 240, 0.090528987]\n",
      "[186, 18, 0.10894047]\n",
      "[187, 66, 0.09914843]\n",
      "[188, 166, 0.087044276]\n",
      "[189, 107, 0.07752873]\n",
      "[190, 245, 0.091634884]\n",
      "[191, 104, 0.064945363]\n",
      "[192, 164, 0.081555732]\n",
      "[193, 51, 0.075884365]\n",
      "[194, 108, 0.069505349]\n",
      "[195, 142, 0.091015048]\n",
      "[196, 14, 0.10005478]\n",
      "[197, 155, 0.073494025]\n",
      "[198, 27, 0.0779863]\n",
      "[199, 175, 0.070071429]\n",
      "0.0806751\n",
      "[200, 149, 0.074140817]\n",
      "[201, 189, 0.07170859]\n",
      "[202, 193, 0.074470207]\n",
      "[203, 72, 0.10027562]\n",
      "[204, 13, 0.089425027]\n",
      "[205, 124, 0.069423698]\n",
      "[206, 78, 0.073509604]\n",
      "[207, 84, 0.068006091]\n",
      "[208, 148, 0.074570589]\n",
      "[209, 19, 0.089090005]\n",
      "[210, 243, 0.10162806]\n",
      "[211, 95, 0.06160143]\n",
      "[212, 132, 0.071476467]\n",
      "[213, 10, 0.089530513]\n",
      "[214, 81, 0.067935236]\n",
      "[215, 42, 0.072607793]\n",
      "[216, 221, 0.073103078]\n",
      "[217, 184, 0.065916769]\n",
      "[218, 225, 0.068978041]\n",
      "[219, 48, 0.076696075]\n",
      "[220, 135, 0.072094977]\n",
      "[221, 22, 0.080180004]\n",
      "[222, 168, 0.067285746]\n",
      "[223, 134, 0.063576452]\n",
      "[224, 2, 0.085048862]\n",
      "[225, 153, 0.06311696]\n",
      "[226, 92, 0.062919766]\n",
      "[227, 222, 0.066657968]\n",
      "[228, 36, 0.07697767]\n",
      "[229, 56, 0.06556128]\n",
      "[230, 126, 0.062072296]\n",
      "[231, 77, 0.067988776]\n",
      "[232, 7, 0.079530038]\n",
      "[233, 197, 0.070407905]\n",
      "[234, 176, 0.062743768]\n",
      "[235, 210, 0.061725106]\n",
      "[236, 119, 0.062183995]\n",
      "[237, 45, 0.067004368]\n",
      "[238, 224, 0.063575871]\n",
      "[239, 202, 0.057945412]\n",
      "[240, 31, 0.070534296]\n",
      "[241, 0, 0.078507349]\n",
      "[242, 139, 0.072480135]\n",
      "[243, 143, 0.063806772]\n",
      "[244, 121, 0.054559872]\n",
      "[245, 141, 0.062780485]\n",
      "[246, 54, 0.06071182]\n",
      "[247, 172, 0.065349579]\n",
      "[248, 216, 0.059548195]\n",
      "[249, 59, 0.063648194]\n",
      "[250, 85, 0.10232221]\n"
     ]
    }
   ],
   "source": [
    "# Train and save checkpoint at the end of each file.\n",
    "with tf.Session(graph=full_graph,config = tf.ConfigProto(\n",
    "    intra_op_parallelism_threads = batch_size)) as sess:\n",
    "    # Run the Op to initialize the variables.\n",
    "    sess.run(init)\n",
    "    # Start the training loop.\n",
    "    costs = []\n",
    "    step = 1\n",
    "    # Go over all training files\n",
    "    file_num = 0\n",
    "    fname = data_list['keys'][0][file_num][0][0:-3]+'mat'\n",
    "    data = cpio.loadmat(data_directory + '/' + fname)\n",
    "    data1 = np.transpose(data['s'])\n",
    "    intY = data['labels'][0]\n",
    "    for file_num in range(number_of_files-1):\n",
    "        # load current training file\n",
    "        fname = data_list['keys'][0][file_num+1][0][0:-3]+'mat'\n",
    "        bdata = cpio.loadmat(data_directory + '/' + fname)\n",
    "        bdata1 = np.transpose(bdata['s'])\n",
    "        bintY = bdata['labels'][0]\n",
    "        data1 = np.concatenate((data1,bdata1),axis = 0)\n",
    "        intY = np.concatenate((intY,bintY),axis = 0)\n",
    "    temp_n = len(intY)/batch_size\n",
    "    data1 = data1[0:temp_n*batch_size].reshape((batch_size,temp_n,-1))\n",
    "    intY = intY[0:temp_n*batch_size].reshape((batch_size,-1))\n",
    "    iter_order = np.random.permutation(data1.shape[1]-370)\n",
    "    if (len(iter_order) > n_max_iter):\n",
    "        iter_order = iter_order[0:n_max_iter]\n",
    "    print data1.shape, len(iter_order)\n",
    "    for iternum in iter_order:\n",
    "        d = {X: data1[:,iternum:iternum+100,:] ,Y: intY[:,iternum:iternum+100] ,lng:[100]*batch_size}\n",
    "        _cost,_ = sess.run((cost,train_op),feed_dict = d)\n",
    "        costs.append(_cost)\n",
    "        print([step,iternum,_cost])\n",
    "        step = step + 1\n",
    "            \n",
    "        if (step % 100 == 0):    \n",
    "            checkpoint_file = os.path.join(training_records_dir, 'checkpoint')\n",
    "            saver.save(sess, checkpoint_file, global_step=step)\n",
    "            print np.mean(costs[-10:-1])\n",
    "    checkpoint_file = os.path.join(training_records_dir, 'checkpoint')\n",
    "    saver.save(sess, checkpoint_file, global_step=step)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "<KEY>",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1151b7150>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "251"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pylab\n",
    "pylab.figure()\n",
    "pylab.plot(costs)\n",
    "pylab.show()\n",
    "step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /Users/yardenc/Documents/Experiments/Imaging/CanaryData/lrb853_15/training_records/checkpoint-200\n"
     ]
    }
   ],
   "source": [
    "# Evaluate training set from a saved checkpoint\n",
    "with tf.Session(graph=tf.Graph()) as sess:\n",
    "    saver = tf.train.import_meta_graph(\n",
    "        os.path.join(training_records_dir, \"checkpoint-200.meta\"))\n",
    "    saver.restore(\n",
    "        sess, os.path.join(training_records_dir, \"checkpoint-200\"))\n",
    "\n",
    "    # Retrieve the Ops we 'remembered'.\n",
    "    logits = tf.get_collection(\"logits\")[0]\n",
    "    X = tf.get_collection(\"specs\")[0]\n",
    "    Y = tf.get_collection(\"labels\")[0]\n",
    "    lng = tf.get_collection(\"lng\")[0]\n",
    "    \n",
    "    # Add an Op that chooses the top k predictions.\n",
    "    eval_op = tf.nn.top_k(logits)\n",
    "    \n",
    "    # Run evaluation.\n",
    "    errors = []\n",
    "    for file_num in range(number_of_files):\n",
    "        # load current training file\n",
    "        fname = data_list['keys'][0][file_num][0][0:-3]+'mat'\n",
    "        data = cpio.loadmat(data_directory + '/' + fname)\n",
    "        data1 = np.transpose(data['s'])\n",
    "        intY = data['labels'][0]\n",
    "        temp_n = len(intY)/batch_size\n",
    "        data1 = data1[0:temp_n*batch_size].reshape((batch_size,temp_n,-1))\n",
    "        intY = intY[0:temp_n*batch_size].reshape((batch_size,-1))\n",
    "        d = {X: data1 ,Y: intY ,lng:[temp_n]*batch_size}\n",
    "        pred = sess.run(eval_op,feed_dict = d)\n",
    "        errors.append(np.abs(np.squeeze(pred[1]).reshape(-1)-intY.reshape(-1)) != 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.040914385556202679,\n",
       " 0.041018387553041019,\n",
       " 0.048583928160257889,\n",
       " 0.049504950495049507,\n",
       " 0.04064616988014591,\n",
       " 0.062293729372937291,\n",
       " 0.060764900019413703,\n",
       " 0.035502121640735504,\n",
       " 0.050647372429550648,\n",
       " 0.037438118811881187,\n",
       " 0.042772277227722776]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Results on training set\n",
    "[np.mean(err) for err in errors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "<KEY>",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11c0763d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pylab.plot(intY.reshape(-1))\n",
    "pylab.plot(np.squeeze(pred[1]).reshape(-1))\n",
    "np.mean(np.abs(np.squeeze(pred[1]).reshape(-1)-intY.reshape(-1)) != 0)\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
{'fraction_non_alphanumeric': 0.22017358897660963, 'fraction_numerical': 0.18121414210758593, 'mean_word_length': 2.362489694971146, 'pattern_counts': {'":': 194, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4925666', 'n_tokens_mistral': 21851, 'n_tokens_neox': 16963, 'n_words': 2890}