<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Consume the TFrecord data\n",
    "\n",
    "*  https://www.tensorflow.org/programmers_guide/datasets\n",
    "*  https://github.com/Tony607/Keras_catVSdog_tf_estimator/blob/master/keras_estimator_vgg16-cat_vs_dog-TFRecord.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = [ './librivox/guidetomen_%02d_rowland_64kb.mp3' % (i,) for i in [1,2,3]]\n",
    "filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps_total, steps_leadin = 1024, 64\n",
    "mel_bins, spectra_bins = 80, 1025\n",
    "\n",
    "#batch_size, num_epochs = 8, 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataset_from_mp3(filenames, stub='train'):\n",
    "    dataset = tf.data.TFRecordDataset([f.replace('.mp3', '_%s.tfrecords') % stub \n",
    "                                       for f in filenames])\n",
    "\n",
    "    features = {\n",
    "      \"mel\":          tf.FixedLenFeature([mel_bins*steps_total], tf.float32),\n",
    "      \"spectra_real\": tf.FixedLenFeature([spectra_bins*steps_total], tf.float32),\n",
    "      \"spectra_imag\": tf.FixedLenFeature([spectra_bins*steps_total], tf.float32),\n",
    "    }\n",
    "    \n",
    "    def _parse_function(example_proto):\n",
    "        parsed_features = tf.parse_single_example(example_proto, features)\n",
    "        mel         = parsed_features[\"mel\"].reshape( (mel_bins, steps_total) )\n",
    "        specta_real = parsed_features[\"spectra_real\"].reshape( (spectra_bins, steps_total) )\n",
    "        specta_imag = parsed_features[\"spectra_imag\"].reshape( (spectra_bins, steps_total) )\n",
    "        spectra = tf.stack( [specta_real, specta_imag, ] ) # Should be (spectra_bins, steps_total, 2)\n",
    "        return tf.transpose(mel), tf.transpose(spectra, perm=[1,0,2])\n",
    "\n",
    "    dataset = dataset.map(_parse_function)\n",
    "    return dataset\n",
    "\n",
    "def input_fn_from(filenames, stub='train', batch_size=1, shuffle=False, repeats=1):\n",
    "    dataset = dataset_from_mp3(filenames, stub=stub)\n",
    "    if shuffle:\n",
    "        dataset = dataset.shuffle(buffer_size=100)\n",
    "    dataset = dataset_train.batch(batch_size).repeat(repeats)\n",
    "    \n",
    "    iterator_train = dataset.make_one_shot_iterator()    \n",
    "    \n",
    "    batch_features, batch_labels = iterator.get_next()\n",
    "    return batch_features, batch_labels    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keras model \n",
    "\n",
    "Keras-WaveNet : \n",
    "\n",
    "\n",
    "*  https://github.com/usernaamee/keras-wavenet/blob/master/simple-generative-model-regressor.py \n",
    "*  Beware of GPL3 license!  Code independently...\n",
    "*  But, also, the number of convolutional channels seems pretty arbitrary, so may not be fleshed-out yet\n",
    "\n",
    "TF-WaveNet :\n",
    "*  https://github.com/ibab/tensorflow-wavenet\n",
    "*  MIT license : Feel free to look + adapt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# See : https://github.com/tensorflow/tensorflow/issues/14933\n",
    "#   to understand how broken Google is\n",
    "#from tensorflow.python import keras\n",
    "#from tensorflow.python.keras import backend as K\n",
    "\n",
    "# Use 'real keras' to get the actual documented functionality\n",
    "import keras\n",
    "from keras import backend as K\n",
    "\n",
    "def wavenet_layer(channels, kernel_size, dilation_rate):\n",
    "    def f(input_):\n",
    "        filter_out = keras.layers.Conv1D(channels, kernel_size,\n",
    "                                       strides=1, dilation_rate=dilation_rate,\n",
    "                                       padding='causal', use_bias=True, \n",
    "                                       activation='tanh')(input_)\n",
    "        gate_out   = keras.layers.Conv1D(channels, kernel_size,\n",
    "                                       strides=1, dilation_rate=dilation_rate,\n",
    "                                       padding='causal', use_bias=True, \n",
    "                                       activation='sigmoid')(input_)\n",
    "        mult = keras.layers.Multiply()( [filter_out, gate_out] )\n",
    "        \n",
    "        # Probably need to pad this result back out to input_ size...\n",
    "        # ... except : https://github.com/keras-team/keras/blob/master/keras/utils/conv_utils.py#L119\n",
    "\n",
    "        transformed = keras.layers.Conv1D(channels, 1, \n",
    "                                          padding='same', use_bias=True, \n",
    "                                          activation='linear')(mult)\n",
    "        skip_out    = keras.layers.Conv1D(channels, 1, \n",
    "                                          padding='same', use_bias=True, \n",
    "                                          activation='relu')(mult)\n",
    "        \n",
    "        return keras.layers.Add()( [transformed, input_] ), skip_out\n",
    "      \n",
    "    return f\n",
    "\n",
    "hidden_channels=128\n",
    "def model_mel_to_spec( input_shape=(steps_total, mel_bins) ):\n",
    "    mel = keras.layers.Input(shape=input_shape)\n",
    "    mel_floored = K.maximum(0.00001, mel)\n",
    "    mel_log     = K.log(mel_floored)  # This is (batch, T. channels)\n",
    "\n",
    "    x = mel_log\n",
    "    x,s1 = wavenet_layer(hidden_channels, 3, 1)(x)\n",
    "    x,s2 = wavenet_layer(hidden_channels, 3, 2)(x)\n",
    "    x,s3 = wavenet_layer(hidden_channels, 3, 4)(x)\n",
    "    x,s4 = wavenet_layer(hidden_channels, 3, 8)(x)\n",
    "    _,s5 = wavenet_layer(hidden_channels, 3, 16)(x)  # Total footprint is ~64 0.75secs\n",
    "    # x is now irrelevant\n",
    "    \n",
    "    skip_overall = keras.layers.Concatenate( axis=-1 )( [s1,s2,s3,s4,s5] )\n",
    "    \n",
    "    log_amp = keras.layers.Conv1D(spectra_bins, 1, padding='same', activation='linear')(skip_overall)\n",
    "    phase   = keras.layers.Conv1D(spectra_bins, 1, padding='same', activation='linear')(skip_overall)\n",
    "    \n",
    "    amp = K.exp(log_amp)\n",
    "    \n",
    "    spec_real = Multiply( [amp, K.cos(phase)] )\n",
    "    spec_imag = Multiply( [amp, K.sin(phase)] )\n",
    "    \n",
    "    spec = keras.layers.Stack( [spec_real, spec_imag] )\n",
    "    \n",
    "    return keras.models.Model(inputs=mel, outputs=spec)\n",
    "\n",
    "keras_model = model_mel_to_spec()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,<commit_msg>On to next generation of errors...<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Consume the TFrecord data\n",
    "\n",
    "*  https://www.tensorflow.org/programmers_guide/datasets\n",
    "*  https://github.com/Tony607/Keras_catVSdog_tf_estimator/blob/master/keras_estimator_vgg16-cat_vs_dog-TFRecord.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = [ './librivox/guidetomen_%02d_rowland_64kb.mp3' % (i,) for i in [1,2,3]]\n",
    "filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps_total, steps_leadin = 1024, 64\n",
    "mel_bins, spectra_bins = 80, 1025\n",
    "\n",
    "#batch_size, num_epochs = 8, 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataset_from_mp3(filenames, stub='train'):\n",
    "    dataset = tf.data.TFRecordDataset([f.replace('.mp3', '_%s.tfrecords') % stub \n",
    "                                       for f in filenames])\n",
    "\n",
    "    features = {\n",
    "      \"mel\":          tf.FixedLenFeature([mel_bins*steps_total], tf.float32),\n",
    "      \"spectra_real\": tf.FixedLenFeature([spectra_bins*steps_total], tf.float32),\n",
    "      \"spectra_imag\": tf.FixedLenFeature([spectra_bins*steps_total], tf.float32),\n",
    "    }\n",
    "    \n",
    "    def _parse_function(example_proto):\n",
    "        parsed_features = tf.parse_single_example(example_proto, features)\n",
    "        mel         = parsed_features[\"mel\"].reshape( (mel_bins, steps_total) )\n",
    "        specta_real = parsed_features[\"spectra_real\"].reshape( (spectra_bins, steps_total) )\n",
    "        specta_imag = parsed_features[\"spectra_imag\"].reshape( (spectra_bins, steps_total) )\n",
    "        spectra = tf.stack( [specta_real, specta_imag, ] ) # Should be (spectra_bins, steps_total, 2)\n",
    "        return tf.transpose(mel), tf.transpose(spectra, perm=[1,0,2])\n",
    "\n",
    "    dataset = dataset.map(_parse_function)\n",
    "    return dataset\n",
    "\n",
    "def input_fn_from(filenames, stub='train', batch_size=1, shuffle=False, repeats=1):\n",
    "    dataset = dataset_from_mp3(filenames, stub=stub)\n",
    "    if shuffle:\n",
    "        dataset = dataset.shuffle(buffer_size=100)\n",
    "    dataset = dataset_train.batch(batch_size).repeat(repeats)\n",
    "    \n",
    "    iterator_train = dataset.make_one_shot_iterator()    \n",
    "    \n",
    "    batch_features, batch_labels = iterator.get_next()\n",
    "    return batch_features, batch_labels    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keras model \n",
    "\n",
    "Keras-WaveNet : \n",
    "\n",
    "\n",
    "*  https://github.com/usernaamee/keras-wavenet/blob/master/simple-generative-model-regressor.py \n",
    "*  Beware of GPL3 license!  Code independently...\n",
    "*  But, also, the number of convolutional channels seems pretty arbitrary, so may not be fleshed-out yet\n",
    "\n",
    "TF-WaveNet :\n",
    "*  https://github.com/ibab/tensorflow-wavenet\n",
    "*  MIT license : Feel free to look + adapt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# See : https://github.com/tensorflow/tensorflow/issues/14933\n",
    "#   to understand how broken Google is\n",
    "#from tensorflow.python import keras\n",
    "#from tensorflow.python.keras import backend as K\n",
    "\n",
    "# Use 'real keras' to get the actual documented functionality\n",
    "import keras\n",
    "from keras import backend as K\n",
    "\n",
    "def wavenet_layer(channels, hidden_channels, kernel_size, dilation_rate):\n",
    "    def f(input_):\n",
    "        filter_out = keras.layers.Conv1D(hidden_channels, kernel_size,\n",
    "                                       strides=1, dilation_rate=dilation_rate,\n",
    "                                       padding='causal', use_bias=True, \n",
    "                                       activation='tanh')(input_)\n",
    "        gate_out   = keras.layers.Conv1D(hidden_channels, kernel_size,\n",
    "                                       strides=1, dilation_rate=dilation_rate,\n",
    "                                       padding='causal', use_bias=True, \n",
    "                                       activation='sigmoid')(input_)\n",
    "        mult = keras.layers.Multiply()( [filter_out, gate_out] )\n",
    "        \n",
    "        # Probably need to pad this result back out to input_ size...\n",
    "        # ... except : https://github.com/keras-team/keras/blob/master/keras/utils/conv_utils.py#L119\n",
    "\n",
    "        transformed = keras.layers.Conv1D(channels, 1, \n",
    "                                          padding='same', use_bias=True, \n",
    "                                          activation='linear')(mult)\n",
    "        skip_out    = keras.layers.Conv1D(channels, 1, \n",
    "                                          padding='same', use_bias=True, \n",
    "                                          activation='relu')(mult)\n",
    "        \n",
    "        return keras.layers.Add()( [transformed, input_] ), skip_out\n",
    "      \n",
    "    return f\n",
    "\n",
    "io_channels, hidden_channels = 128,128\n",
    "def model_mel_to_spec( input_shape=(steps_total, mel_bins) ):\n",
    "    mel = keras.layers.Input(shape=input_shape)\n",
    "    mel_floored = K.maximum(0.00001, mel)\n",
    "    mel_log     = K.log(mel_floored)  # This is (batch, T. channels)\n",
    "\n",
    "    # 'Resize' to make everything 'io_channels' big at the layer interfaces\n",
    "    x = keras.layers.Conv1D(io_channels, 1, \n",
    "                          padding='same', use_bias=True, \n",
    "                          activation='linear')(mel_log)\n",
    "    \n",
    "    x,s1 = wavenet_layer(io_channels, hidden_channels*1, 3, 1)(x)\n",
    "    x,s2 = wavenet_layer(io_channels, hidden_channels*1, 3, 2)(x)\n",
    "    x,s3 = wavenet_layer(io_channels, hidden_channels*1, 3, 4)(x)\n",
    "    x,s4 = wavenet_layer(io_channels, hidden_channels*1, 3, 8)(x)\n",
    "    _,s5 = wavenet_layer(io_channels, hidden_channels*1, 3, 16)(x)  # Total footprint is ~64 0.75secs\n",
    "    # x is now irrelevant\n",
    "    \n",
    "    skip_overall = keras.layers.Concatenate( axis=-1 )( [s1,s2,s3,s4,s5] )\n",
    "    \n",
    "    log_amp = keras.layers.Conv1D(spectra_bins, 1, padding='same', activation='linear')(skip_overall)\n",
    "    phase   = keras.layers.Conv1D(spectra_bins, 1, padding='same', activation='linear')(skip_overall)\n",
    "    \n",
    "    amp = K.exp(log_amp)\n",
    "    \n",
    "    return keras.models.Model(inputs=[mel], outputs=[amp, phase])\n",
    "\n",
    "    #spec_real = keras.layers.Multiply()( [amp, K.cos(phase)] )\n",
    "    #spec_imag = keras.layers.Multiply()( [amp, K.sin(phase)] )\n",
    "    \n",
    "    #spec = keras.layers.Stack( [spec_real, spec_imag] )\n",
    "    \n",
    "    #return keras.models.Model(inputs=mel, outputs=[spec_real, spec_imag])\n",
    "\n",
    "keras_model = model_mel_to_spec()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
{'fraction_non_alphanumeric': 0.17830195092107948, 'fraction_numerical': 0.014682890354156753, 'mean_word_length': 2.1982608695652175, 'pattern_counts': {'":': 76, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 12, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2404498', 'n_tokens_mistral': 5452, 'n_tokens_neox': 5241, 'n_words': 871}