<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create dataset from audio files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.misc  # for image resizing\n",
    "\n",
    "#import scipy.io.wavfile\n",
    "\n",
    "# pip install soundfile\n",
    "import soundfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#f = './data/raw-from-phone.wav'\n",
    "\n",
    "f = './data/num_phone_en-UK_m_Martin01.wav'\n",
    "#f = './data/num_Bing_en-UK_f_Susan.wav'\n",
    "\n",
    "#f = './data/num_phone_en-UK_m_Martin00.ogg'\n",
    "#f = './data/num_Bing_en-UK_f_Susan.ogg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Clip the audio region for a long file\n",
    "\n",
    "samples, sample_rate = soundfile.read(f)\n",
    "\n",
    "def show_waveform(sound):\n",
    "    n_samples = sound.shape[0]\n",
    "\n",
    "    plt.figure(figsize=(12,1))\n",
    "    plt.plot(np.arange(0.0, n_samples)/sample_rate, sound)\n",
    "    plt.xticks( np.arange(0.0, n_samples/sample_rate, 0.5) )\n",
    "\n",
    "    plt.grid(True)\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "show_waveform(samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "crop = (3.25, 16.25)  # in seconds from above\n",
    "\n",
    "cropped = samples[ int(crop[0]*sample_rate):int(crop[1]*sample_rate) ]\n",
    "\n",
    "show_waveform(cropped)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Only do this (set it to 1) if you want to replace the file with the cropped version...\n",
    "if 1:\n",
    "    #f = './data/cropped-raw-from-phone.wav'  \n",
    "    soundfile.write(f, cropped, samplerate=sample_rate)\n",
    "    print(\"Wrote '%s'\" % (f,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def spectrogram(wav_filepath):\n",
    "    # https://mail.python.org/pipermail/chicago/2010-December/007314.html\n",
    "    \n",
    "    #sample_rate, samples = scipy.io.wavfile.read(wav_filepath)\n",
    "    samples, sample_rate = soundfile.read(wav_filepath)\n",
    "    \n",
    "    # Rescale so that max/min are ~ +/- 1 around 0\n",
    "    data_av = np.mean(samples)\n",
    "    data_max = np.max(np.absolute(samples-data_av))\n",
    "    sound_data = (samples - data_av)/data_max\n",
    "    \n",
    "    ## Parameters: 10ms step, 30ms window\n",
    "    nstep = int(sample_rate * 0.01)\n",
    "    nwin  = int(sample_rate * 0.03)\n",
    "    nfft = 2*int(nwin/2)\n",
    "\n",
    "    window = np.hamming(nwin)\n",
    "\n",
    "    # will take windows x[n1:n2].  generate and loop over \n",
    "    # n2 such that all frames fit within the waveform\n",
    "    nn = range(nwin, len(sound_data), nstep)\n",
    "\n",
    "    X = np.zeros( (len(nn), nfft//2) )\n",
    "\n",
    "    for i,n in enumerate(nn):\n",
    "        segment = sound_data[ n-nwin:n ]\n",
    "        z = np.fft.fft(window * segment, nfft)\n",
    "        X[i,:] = np.log(np.absolute(z[:nfft//2]))\n",
    "        \n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This is a function that smooths a time-series\n",
    "#   which enables us to segment the input into words by looking at the 'energy' profile\n",
    "def smooth(x, window_len=21):  # , window='hanning'\n",
    "    # http://scipy-cookbook.readthedocs.io/items/SignalSmooth.html\n",
    "    #s = np.r_[ x[window_len-1:0:-1], x, x[-1:-window_len:-1]]\n",
    "    s = np.r_[ np.zeros( ((window_len-1)//2,) ), x, np.zeros( ((window_len-1)//2,) ) ]\n",
    "    w=np.hamming(window_len)\n",
    "    return np.convolve(w/w.sum(), s, mode='valid') #[window_len-1 : -(window_len-1) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = spectrogram(f)\n",
    "print(\"X.shape=\", X.shape)\n",
    "\n",
    "Y = np.std(X, axis=1)\n",
    "#Y = np.max(X, axis=1)\n",
    "Y_min = np.min(Y)\n",
    "Y_range = Y.max()-Y_min\n",
    "Y = (Y - Y_min)/Y_range\n",
    "\n",
    "print(\"Y.shape=\", Y.shape)\n",
    "\n",
    "Y_crop = np.where(Y>0.25, 1.0, 0.0)\n",
    "# Apply some smoothing\n",
    "\n",
    "Y_crop = smooth(Y_crop)\n",
    "Y_crop = np.where(Y_crop>0.01, 1.0, 0.0)\n",
    "print(\"Y_crop.shape=\", Y_crop.shape)\n",
    "\n",
    "plt.imshow(X.T, interpolation='nearest',\n",
    "    origin='lower',\n",
    "    aspect='auto')\n",
    "\n",
    "plt.plot(Y * X.shape[1])\n",
    "\n",
    "plt.plot(Y_crop * X.shape[1])\n",
    "\n",
    "plt.show()\n",
    "#Y.min(), Y.max()\n",
    "#X[100,:]\n",
    "np.argmin(X)/248, np.argmax(X)/248"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Split the file into voiced segments\n",
    "\n",
    "#http://stackoverflow.com/questions/4494404/find-large-number-of-consecutive-values-fulfilling-condition-in-a-numpy-array\n",
    "def contiguous_regions(condition):\n",
    "    idx = []\n",
    "    i = 0\n",
    "    while i < len(condition):\n",
    "        x1 = i + condition[i:].argmax()\n",
    "        try:\n",
    "            x2 = x1 + condition[x1:].argmin()\n",
    "        except:\n",
    "            x2 = x1 + 1\n",
    "        if x1 == x2:\n",
    "            if condition[x1] == True:\n",
    "                x2 = len(condition)\n",
    "            else:\n",
    "                break\n",
    "        idx.append( [x1,x2] )\n",
    "        i = x2\n",
    "    return idx\n",
    "\n",
    "contiguous_regions(Y_crop>0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "remove_punc = re.compile('[\\,\\.\\?\\!]')\n",
    "squash_spaces = re.compile('\\s+')\n",
    "def words(s):\n",
    "    s = remove_punc.sub(' ', s)\n",
    "    s = squash_spaces.sub(' ', s)\n",
    "    return s.strip().lower()\n",
    "\n",
    "sentences=dict(\n",
    "    num=words(\"zero one two three four five six seven eight nine.\"),\n",
    "    \n",
    "    # https://www.quora.com/Is-there-a-text-that-covers-the-entire-English-phonetic-range/\n",
    "    qbf=words(\"That quick beige fox jumped in the air over each thin dog.  \"+\n",
    "              \"Look out, I shout, for he's foiled you again, creating chaos.\"),\n",
    "    shy=words(\"Are those shy Eurasian footwear, cowboy chaps, \"+\n",
    "              \"or jolly earthmoving headgear?\"),\n",
    "    ate=words(\"The hungry purple dinosaur ate the kind, zingy fox, the jabbering crab, \"+\n",
    "              \"and the mad whale and started vending and quacking.\"),\n",
    "    suz=words(\"With tenure, Suzie'd have all the more leisure for yachting, \"+\n",
    "              \"but her publications are no good.\"),\n",
    "    tbh=words(\"Shaw, those twelve beige hooks are joined if I patch a young, gooey mouth.\"),\n",
    "    \n",
    "    #  https://en.wikipedia.org/wiki/The_North_Wind_and_the_Sun          #594\n",
    "    #  http://videoweb.nie.edu.sg/phonetic/courses/aae103-web/wolf.html  #1111\n",
    ")\n",
    "sentences['num']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def for_msft(prefixes):  # comma separated\n",
    "    # https://www.microsoft.com/cognitive-services/en-us/speech-api\n",
    "    return ' '.join([sentences[a] for a in prefixes.split(',')]).replace(' ', '\\n') \n",
    "\"\"\"\n",
    "This is the SSML that will be sent to the service:\n",
    "<speak version=\"1.0\" xmlns=\"http://www.w3.org/2001/10/synthesis\" \n",
    "      xmlns:mstts=\"http://www.w3.org/2001/mstts\" xml:lang=\"en-GB\">\n",
    "  <voice xml:lang=\"en-GB\" name=\"Microsoft Server Speech Text to Speech Voice (en-GB, Susan, Apollo)\">\n",
    "zero\n",
    "one\n",
    "two\n",
    "three\n",
    "four\n",
    "five\n",
    "six\n",
    "seven\n",
    "eight\n",
    "nine\n",
    "  </voice>\n",
    "</speak>\n",
    "\"\"\"\n",
    "\n",
    "# https://www.microsoft.com/cognitive-services/en-us/Speech-api/documentation/API-Reference-REST/BingVoiceOutput\n",
    "a=for_msft('num')  # 49 long...\n",
    "#a=for_msft('qbf,shy,ate,suz,tbh')  # 474 long...\n",
    "\n",
    "print(\"length_in_chars=%d\\n%s\" % (len(a),a,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# sox_ogg_param='--rate 16000 --channels 1'\n",
    "# sox_wav_param=\"${sox_ogg_param} --encoding signed-integer\"\n",
    "\n",
    "# sox english.au ${sox_wav_param} english.wav norm -3\n",
    "# sox english.au ${sox_ogg_param} english.ogg norm -3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pip install python_speech_features\n",
    "import python_speech_features\n",
    "\n",
    "sample_window_step = 0.01 # in seconds (10ms)\n",
    "\n",
    "def get_sample_features(samples, sample_rate):\n",
    "    #sample_feat = python_speech_features.mfcc(samples, sample_rate, numcep=13, nfilt=26, appendEnergy=True)\n",
    "    #sample_feat = python_speech_features.mfcc(samples, sample_rate, numcep=28, nfilt=56, appendEnergy=True)\n",
    "\n",
    "    #sample_feat, e = python_speech_features.fbank(samples,samplerate=sample_rate,\n",
    "    #      winlen=0.025,winstep=0.01,nfilt=26,nfft=512,\n",
    "    #      lowfreq=0,highfreq=None,preemph=0.97, winfunc=lambda x:np.ones((x,)))\n",
    "\n",
    "    features, energy = python_speech_features.fbank(samples, samplerate=sample_rate, \n",
    "                            winlen=0.025, winstep=sample_window_step, \n",
    "                            nfilt=32,nfft=512,\n",
    "                            lowfreq=0,highfreq=None,preemph=0.25,\n",
    "                            winfunc=lambda x:np.hamming( x ))\n",
    "    return features, energy\n",
    "    \n",
    "def get_sample_isolated_words(energy, plot=False):\n",
    "    log_e = np.log(energy)\n",
    "    if plot: plt.plot(log_e)\n",
    "\n",
    "    log_e_hurdle = (log_e.max() - log_e.min())*0.25 + log_e.min()\n",
    "\n",
    "    log_e_crop = np.where(log_e>log_e_hurdle, 1.0, 0.0)\n",
    "    if plot: plt.plot(log_e_crop * 25)\n",
    "\n",
    "    # By smoothing, and applying a very low hurdle, we expand the crop area safely\n",
    "    log_e_crop_expanded = np.where(smooth(log_e_crop)>0.01, 1.0, 0.0)\n",
    "    if plot: plt.plot(log_e_crop_expanded * 25)\n",
    "        \n",
    "    return contiguous_regions(log_e_crop_expanded>0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "samples, sample_rate = soundfile.read(f)\n",
    "\n",
    "sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "\n",
    "plt.imshow(np.log(sample_feat.T), interpolation='nearest',\n",
    "    origin='lower',\n",
    "    aspect='auto')\n",
    "\n",
    "word_ranges = get_sample_isolated_words(energy, plot=True)\n",
    "\n",
    "plt.show()\n",
    "sample_feat.shape, energy.shape, energy[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Break sound into separate WAVs in word-based directories\n",
    "def split_combined_file_into_wavs(f, prefix='num'):\n",
    "    # f ~ './data/num_Bing_en-UK_f_Susan.wav'\n",
    "    f_base_orig = os.path.basename( f )\n",
    "    if not f_base_orig.startswith(prefix+\"_\"): \n",
    "        print(\"Wrong prefix for '%s'\" % (f_base_orig,))\n",
    "        return\n",
    "    \n",
    "    # Here's the new filename (directory to be calculated per-word)\n",
    "    f_base = os.path.splitext(f_base_orig)[0][len(prefix)+1:] + '.wav'\n",
    "    \n",
    "    samples, sample_rate = soundfile.read(f)\n",
    "    sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "    word_ranges = get_sample_isolated_words(energy, plot=False)\n",
    "    #print(word_ranges)\n",
    "    \n",
    "    words = sentences[prefix].split(' ')\n",
    "    if len(word_ranges) != len(words):\n",
    "        print(\"Found %d segments, rather than %d, in '%s'\" % (len(word_ranges), len(words), f,))\n",
    "        return\n",
    "    \n",
    "    for i, word in enumerate(words):\n",
    "        word_path = os.path.join('data', prefix, word)\n",
    "        os.makedirs(word_path, exist_ok=True)\n",
    "        \n",
    "        wr = word_ranges[i]\n",
    "        fac = int(sample_window_step*sample_rate)\n",
    "        soundfile.write(os.path.join(word_path, f_base), samples[ wr[0]*fac:wr[1]*fac ], samplerate=sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "split_combined_file_into_wavs('./data/num_Bing_en-UK_f_Susan.wav')\n",
    "#split_combined_file_into_wavs('./data/num_phone_en-UK_m_Martin00.wav')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Iterate through all the audio files with a given prefix, and unfold them\n",
    "\n",
    "def split_all_combined_files_into_wavs(prefix='num'):\n",
    "    for audio_file in os.listdir( 'data' ):\n",
    "        filename_stub, ext = os.path.splitext(audio_file)\n",
    "        if not (ext=='.wav' or ext=='.ogg'): continue\n",
    "        if not filename_stub.startswith( prefix+'_'): continue\n",
    "    \n",
    "        print(\"Splitting %s\" % (audio_file,))\n",
    "        split_combined_file_into_wavs(audio_file, prefix=prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_all_combined_files_into_wavs(prefix='num')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert a given (isolated word) WAV into a 'stamp' - using a helper function\n",
    "\n",
    "def samples_to_stamp(samples, sample_rate):\n",
    "    sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "    \n",
    "    data = np.log(sample_feat)\n",
    "    \n",
    "    # Force the data into the 'stamp size' as an image (implicit range normalization occurs)\n",
    "    stamp = scipy.misc.imresize(data, (64, 32), 'bilinear')\n",
    "    \n",
    "    # https://github.com/scipy/scipy/issues/4458 :: The stamps are stored as uint8...\n",
    "    return stamp\n",
    "\n",
    "def wav_to_stamp(prefix, word, wav):\n",
    "    samples, sample_rate = soundfile.read( os.path.join('data', prefix, word, wav) )\n",
    "    return samples_to_stamp(samples, sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Show what the 'visual stamp' for a given word looks like\n",
    "stamp = wav_to_stamp('num', 'six', 'phone_en-UK_m_Martin0.wav')\n",
    "\n",
    "plt.imshow(stamp.T, interpolation='nearest', origin='lower', aspect='auto')\n",
    "plt.show()\n",
    "\n",
    "np.min(stamp), np.max(stamp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# combine all words from a given prefix into a dataset of 'stamps'\n",
    "import pickle\n",
    "\n",
    "def create_dataset_from_folders(prefix, save_as='.pkl'):\n",
    "    words = sentences[prefix].split(' ')\n",
    "    stamps, labels = [], []\n",
    "    \n",
    "    for label_i, word in enumerate( words ):\n",
    "        # Find all the files for this word\n",
    "        for stamp_file in os.listdir( os.path.join('data', prefix, word )):\n",
    "            if not f.endswith('.wav'): continue\n",
    "            #print(stamp_file)\n",
    "            stamp = wav_to_stamp(prefix, word, stamp_file)\n",
    "            \n",
    "            stamps.append(stamp)\n",
    "            labels.append(label_i)\n",
    "\n",
    "    if save_as is None: # Return the data directly\n",
    "        return stamps, labels, words\n",
    "    \n",
    "    data_dictionary = dict(\n",
    "        stamp=stamps, label=labels, \n",
    "        rand=np.random.rand( len(labels) ), # This is to enable us to sample the data (based on hurdles)\n",
    "        words=words, \n",
    "    )\n",
    "    pickle.dump(data_dictionary, open(os.path.join('data', prefix+save_as), 'wb'), \n",
    "                protocol=pickle.HIGHEST_PROTOCOL)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#if not os.path.exists('data/num.pkl'):\n",
    "if True:\n",
    "    create_dataset_from_folders('num')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read in the dataset\n",
    "dataset = pickle.load(open(os.path.join('data', 'num.pkl'), 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Plot all of a given 'word'\n",
    "indices = [ i for i,label in enumerate(dataset['label']) if dataset['words'][label]=='six']\n",
    "\n",
    "plt.figure(figsize=(12,2))\n",
    "for i in indices[0:16]:  # at most 16\n",
    "    plt.subplot(2, 8, i+1)  # nrows, ncols, subplot#\n",
    "    plt.imshow(dataset['stamp'][i].T, cmap='gray', origin='lower', interpolation='nearest')\n",
    "    plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now do something similar for 'test files', create a dataset for all the audio files in the given folder\n",
    "\n",
    "def create_dataset_from_adhoc_wavs(prefix, save_as='.pkl'):\n",
    "    stamps, labels, words = [], [], []\n",
    "    \n",
    "    for audio_file in os.listdir( os.path.join('data', prefix) ):\n",
    "        filename_stub, ext = os.path.splitext(audio_file)\n",
    "        if not (ext=='.wav' or ext=='.ogg'): continue\n",
    "            \n",
    "        samples, sample_rate = soundfile.read(audio_file)\n",
    "        sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "        word_ranges = get_sample_isolated_words(energy, plot=False)\n",
    "    \n",
    "        for i, wr in enumerate(word_ranges):\n",
    "            wr = word_ranges[i]\n",
    "            fac = int(sample_window_step*sample_rate)\n",
    "            segment = samples[ wr[0]*fac:wr[1]*fac ]\n",
    "\n",
    "            stamp = samples_to_stamp(segment, sample_rate)\n",
    "            \n",
    "            stamps.append(stamp)\n",
    "            labels.append(-1)\n",
    "            words.append(\"%s_%d\" % (filename_stub, i))\n",
    "    \n",
    "    data_dictionary = dict(\n",
    "        stamp=stamps, label=labels, \n",
    "        rand=np.random.rand( len(labels) ),\n",
    "        words=words, \n",
    "    )\n",
    "    pickle.dump(data_dictionary, open(os.path.join('data', prefix+save_as), 'wb'), \n",
    "                protocol=pickle.HIGHEST_PROTOCOL)            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_prefix = 'test-'+'num'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "create_dataset_from_adhoc_wavs(test_prefix)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}<commit_msg>Use crop tool for real - and return to pre-usage<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create dataset from audio files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.misc  # for image resizing\n",
    "\n",
    "#import scipy.io.wavfile\n",
    "\n",
    "# pip install soundfile\n",
    "import soundfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = './data/raw-from-phone.wav'\n",
    "\n",
    "#f = './data/num_phone_en-UK_m_Martin15.wav'\n",
    "#f = './data/num_Bing_en-UK_f_Susan.wav'\n",
    "\n",
    "#f = './data/num_phone_en-UK_m_Martin00.ogg'\n",
    "#f = './data/num_Bing_en-UK_f_Susan.ogg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Clip the audio region for a long file\n",
    "\n",
    "samples, sample_rate = soundfile.read(f)\n",
    "\n",
    "def show_waveform(sound):\n",
    "    n_samples = sound.shape[0]\n",
    "\n",
    "    plt.figure(figsize=(12,1))\n",
    "    plt.plot(np.arange(0.0, n_samples)/sample_rate, sound)\n",
    "    plt.xticks( np.arange(0.0, n_samples/sample_rate, 0.5), rotation=90 )\n",
    "\n",
    "    plt.grid(True)\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "show_waveform(samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "crop = (3.25, 16.25)  # in seconds from above\n",
    "\n",
    "cropped = samples[ int(crop[0]*sample_rate):int(crop[1]*sample_rate) ]\n",
    "\n",
    "show_waveform(cropped)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Only do this (set it to 1) if you want to replace the file with the cropped version...\n",
    "if 0:\n",
    "    #f = './data/cropped-raw-from-phone.wav'  \n",
    "    soundfile.write(f, cropped, samplerate=sample_rate)\n",
    "    print(\"Wrote '%s'\" % (f,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def spectrogram(wav_filepath):\n",
    "    # https://mail.python.org/pipermail/chicago/2010-December/007314.html\n",
    "    \n",
    "    #sample_rate, samples = scipy.io.wavfile.read(wav_filepath)\n",
    "    samples, sample_rate = soundfile.read(wav_filepath)\n",
    "    \n",
    "    # Rescale so that max/min are ~ +/- 1 around 0\n",
    "    data_av = np.mean(samples)\n",
    "    data_max = np.max(np.absolute(samples-data_av))\n",
    "    sound_data = (samples - data_av)/data_max\n",
    "    \n",
    "    ## Parameters: 10ms step, 30ms window\n",
    "    nstep = int(sample_rate * 0.01)\n",
    "    nwin  = int(sample_rate * 0.03)\n",
    "    nfft = 2*int(nwin/2)\n",
    "\n",
    "    window = np.hamming(nwin)\n",
    "\n",
    "    # will take windows x[n1:n2].  generate and loop over \n",
    "    # n2 such that all frames fit within the waveform\n",
    "    nn = range(nwin, len(sound_data), nstep)\n",
    "\n",
    "    X = np.zeros( (len(nn), nfft//2) )\n",
    "\n",
    "    for i,n in enumerate(nn):\n",
    "        segment = sound_data[ n-nwin:n ]\n",
    "        z = np.fft.fft(window * segment, nfft)\n",
    "        X[i,:] = np.log(np.absolute(z[:nfft//2]))\n",
    "        \n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This is a function that smooths a time-series\n",
    "#   which enables us to segment the input into words by looking at the 'energy' profile\n",
    "def smooth(x, window_len=21):  # , window='hanning'\n",
    "    # http://scipy-cookbook.readthedocs.io/items/SignalSmooth.html\n",
    "    #s = np.r_[ x[window_len-1:0:-1], x, x[-1:-window_len:-1]]\n",
    "    s = np.r_[ np.zeros( ((window_len-1)//2,) ), x, np.zeros( ((window_len-1)//2,) ) ]\n",
    "    w=np.hamming(window_len)\n",
    "    return np.convolve(w/w.sum(), s, mode='valid') #[window_len-1 : -(window_len-1) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = spectrogram(f)\n",
    "print(\"X.shape=\", X.shape)\n",
    "\n",
    "Y = np.std(X, axis=1)\n",
    "#Y = np.max(X, axis=1)\n",
    "Y_min = np.min(Y)\n",
    "Y_range = Y.max()-Y_min\n",
    "Y = (Y - Y_min)/Y_range\n",
    "\n",
    "print(\"Y.shape=\", Y.shape)\n",
    "\n",
    "Y_crop = np.where(Y>0.25, 1.0, 0.0)\n",
    "# Apply some smoothing\n",
    "\n",
    "Y_crop = smooth(Y_crop)\n",
    "Y_crop = np.where(Y_crop>0.01, 1.0, 0.0)\n",
    "print(\"Y_crop.shape=\", Y_crop.shape)\n",
    "\n",
    "plt.imshow(X.T, interpolation='nearest',\n",
    "    origin='lower',\n",
    "    aspect='auto')\n",
    "\n",
    "plt.plot(Y * X.shape[1])\n",
    "\n",
    "plt.plot(Y_crop * X.shape[1])\n",
    "\n",
    "plt.show()\n",
    "#Y.min(), Y.max()\n",
    "#X[100,:]\n",
    "np.argmin(X)/248, np.argmax(X)/248"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Split the file into voiced segments\n",
    "\n",
    "#http://stackoverflow.com/questions/4494404/find-large-number-of-consecutive-values-fulfilling-condition-in-a-numpy-array\n",
    "def contiguous_regions(condition):\n",
    "    idx = []\n",
    "    i = 0\n",
    "    while i < len(condition):\n",
    "        x1 = i + condition[i:].argmax()\n",
    "        try:\n",
    "            x2 = x1 + condition[x1:].argmin()\n",
    "        except:\n",
    "            x2 = x1 + 1\n",
    "        if x1 == x2:\n",
    "            if condition[x1] == True:\n",
    "                x2 = len(condition)\n",
    "            else:\n",
    "                break\n",
    "        idx.append( [x1,x2] )\n",
    "        i = x2\n",
    "    return idx\n",
    "\n",
    "contiguous_regions(Y_crop>0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "remove_punc = re.compile('[\\,\\.\\?\\!]')\n",
    "squash_spaces = re.compile('\\s+')\n",
    "def words(s):\n",
    "    s = remove_punc.sub(' ', s)\n",
    "    s = squash_spaces.sub(' ', s)\n",
    "    return s.strip().lower()\n",
    "\n",
    "sentences=dict(\n",
    "    num=words(\"zero one two three four five six seven eight nine.\"),\n",
    "    \n",
    "    # https://www.quora.com/Is-there-a-text-that-covers-the-entire-English-phonetic-range/\n",
    "    qbf=words(\"That quick beige fox jumped in the air over each thin dog.  \"+\n",
    "              \"Look out, I shout, for he's foiled you again, creating chaos.\"),\n",
    "    shy=words(\"Are those shy Eurasian footwear, cowboy chaps, \"+\n",
    "              \"or jolly earthmoving headgear?\"),\n",
    "    ate=words(\"The hungry purple dinosaur ate the kind, zingy fox, the jabbering crab, \"+\n",
    "              \"and the mad whale and started vending and quacking.\"),\n",
    "    suz=words(\"With tenure, Suzie'd have all the more leisure for yachting, \"+\n",
    "              \"but her publications are no good.\"),\n",
    "    tbh=words(\"Shaw, those twelve beige hooks are joined if I patch a young, gooey mouth.\"),\n",
    "    \n",
    "    #  https://en.wikipedia.org/wiki/The_North_Wind_and_the_Sun          #594\n",
    "    #  http://videoweb.nie.edu.sg/phonetic/courses/aae103-web/wolf.html  #1111\n",
    ")\n",
    "sentences['num']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def for_msft(prefixes):  # comma separated\n",
    "    # https://www.microsoft.com/cognitive-services/en-us/speech-api\n",
    "    return ' '.join([sentences[a] for a in prefixes.split(',')]).replace(' ', '\\n') \n",
    "\"\"\"\n",
    "This is the SSML that will be sent to the service:\n",
    "<speak version=\"1.0\" xmlns=\"http://www.w3.org/2001/10/synthesis\" \n",
    "      xmlns:mstts=\"http://www.w3.org/2001/mstts\" xml:lang=\"en-GB\">\n",
    "  <voice xml:lang=\"en-GB\" name=\"Microsoft Server Speech Text to Speech Voice (en-GB, Susan, Apollo)\">\n",
    "zero\n",
    "one\n",
    "two\n",
    "three\n",
    "four\n",
    "five\n",
    "six\n",
    "seven\n",
    "eight\n",
    "nine\n",
    "  </voice>\n",
    "</speak>\n",
    "\"\"\"\n",
    "\n",
    "# https://www.microsoft.com/cognitive-services/en-us/Speech-api/documentation/API-Reference-REST/BingVoiceOutput\n",
    "a=for_msft('num')  # 49 long...\n",
    "#a=for_msft('qbf,shy,ate,suz,tbh')  # 474 long...\n",
    "\n",
    "print(\"length_in_chars=%d\\n%s\" % (len(a),a,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# sox_ogg_param='--rate 16000 --channels 1'\n",
    "# sox_wav_param=\"${sox_ogg_param} --encoding signed-integer\"\n",
    "\n",
    "# sox english.au ${sox_wav_param} english.wav norm -3\n",
    "# sox english.au ${sox_ogg_param} english.ogg norm -3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pip install python_speech_features\n",
    "import python_speech_features\n",
    "\n",
    "sample_window_step = 0.01 # in seconds (10ms)\n",
    "\n",
    "def get_sample_features(samples, sample_rate):\n",
    "    #sample_feat = python_speech_features.mfcc(samples, sample_rate, numcep=13, nfilt=26, appendEnergy=True)\n",
    "    #sample_feat = python_speech_features.mfcc(samples, sample_rate, numcep=28, nfilt=56, appendEnergy=True)\n",
    "\n",
    "    #sample_feat, e = python_speech_features.fbank(samples,samplerate=sample_rate,\n",
    "    #      winlen=0.025,winstep=0.01,nfilt=26,nfft=512,\n",
    "    #      lowfreq=0,highfreq=None,preemph=0.97, winfunc=lambda x:np.ones((x,)))\n",
    "\n",
    "    features, energy = python_speech_features.fbank(samples, samplerate=sample_rate, \n",
    "                            winlen=0.025, winstep=sample_window_step, \n",
    "                            nfilt=32,nfft=512,\n",
    "                            lowfreq=0,highfreq=None,preemph=0.25,\n",
    "                            winfunc=lambda x:np.hamming( x ))\n",
    "    return features, energy\n",
    "    \n",
    "def get_sample_isolated_words(energy, plot=False):\n",
    "    log_e = np.log(energy)\n",
    "    if plot: plt.plot(log_e)\n",
    "\n",
    "    log_e_hurdle = (log_e.max() - log_e.min())*0.25 + log_e.min()\n",
    "\n",
    "    log_e_crop = np.where(log_e>log_e_hurdle, 1.0, 0.0)\n",
    "    if plot: plt.plot(log_e_crop * 25)\n",
    "\n",
    "    # By smoothing, and applying a very low hurdle, we expand the crop area safely\n",
    "    log_e_crop_expanded = np.where(smooth(log_e_crop)>0.01, 1.0, 0.0)\n",
    "    if plot: plt.plot(log_e_crop_expanded * 25)\n",
    "        \n",
    "    return contiguous_regions(log_e_crop_expanded>0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "samples, sample_rate = soundfile.read(f)\n",
    "\n",
    "sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "\n",
    "plt.imshow(np.log(sample_feat.T), interpolation='nearest',\n",
    "    origin='lower',\n",
    "    aspect='auto')\n",
    "\n",
    "word_ranges = get_sample_isolated_words(energy, plot=True)\n",
    "\n",
    "plt.show()\n",
    "sample_feat.shape, energy.shape, energy[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Break sound into separate WAVs in word-based directories\n",
    "def split_combined_file_into_wavs(f, prefix='num'):\n",
    "    # f ~ './data/num_Bing_en-UK_f_Susan.wav'\n",
    "    f_base_orig = os.path.basename( f )\n",
    "    if not f_base_orig.startswith(prefix+\"_\"): \n",
    "        print(\"Wrong prefix for '%s'\" % (f_base_orig,))\n",
    "        return\n",
    "    \n",
    "    # Here's the new filename (directory to be calculated per-word)\n",
    "    f_base = os.path.splitext(f_base_orig)[0][len(prefix)+1:] + '.wav'\n",
    "    \n",
    "    samples, sample_rate = soundfile.read(f)\n",
    "    sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "    word_ranges = get_sample_isolated_words(energy, plot=False)\n",
    "    #print(word_ranges)\n",
    "    \n",
    "    words = sentences[prefix].split(' ')\n",
    "    if len(word_ranges) != len(words):\n",
    "        print(\"Found %d segments, rather than %d, in '%s'\" % (len(word_ranges), len(words), f,))\n",
    "        return\n",
    "    \n",
    "    for i, word in enumerate(words):\n",
    "        word_path = os.path.join('data', prefix, word)\n",
    "        os.makedirs(word_path, exist_ok=True)\n",
    "        \n",
    "        wr = word_ranges[i]\n",
    "        fac = int(sample_window_step*sample_rate)\n",
    "        soundfile.write(os.path.join(word_path, f_base), samples[ wr[0]*fac:wr[1]*fac ], samplerate=sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "split_combined_file_into_wavs('./data/num_Bing_en-UK_f_Susan.wav')\n",
    "#split_combined_file_into_wavs('./data/num_phone_en-UK_m_Martin00.wav')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Iterate through all the audio files with a given prefix, and unfold them\n",
    "\n",
    "def split_all_combined_files_into_wavs(prefix='num'):\n",
    "    for audio_file in os.listdir( 'data' ):\n",
    "        filename_stub, ext = os.path.splitext(audio_file)\n",
    "        if not (ext=='.wav' or ext=='.ogg'): continue\n",
    "        if not filename_stub.startswith( prefix+'_'): continue\n",
    "    \n",
    "        print(\"Splitting %s\" % (audio_file,))\n",
    "        split_combined_file_into_wavs(audio_file, prefix=prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_all_combined_files_into_wavs(prefix='num')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert a given (isolated word) WAV into a 'stamp' - using a helper function\n",
    "\n",
    "def samples_to_stamp(samples, sample_rate):\n",
    "    sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "    \n",
    "    data = np.log(sample_feat)\n",
    "    \n",
    "    # Force the data into the 'stamp size' as an image (implicit range normalization occurs)\n",
    "    stamp = scipy.misc.imresize(data, (64, 32), 'bilinear')\n",
    "    \n",
    "    # https://github.com/scipy/scipy/issues/4458 :: The stamps are stored as uint8...\n",
    "    return stamp\n",
    "\n",
    "def wav_to_stamp(prefix, word, wav):\n",
    "    samples, sample_rate = soundfile.read( os.path.join('data', prefix, word, wav) )\n",
    "    return samples_to_stamp(samples, sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Show what the 'visual stamp' for a given word looks like\n",
    "stamp = wav_to_stamp('num', 'six', 'phone_en-UK_m_Martin0.wav')\n",
    "\n",
    "plt.imshow(stamp.T, interpolation='nearest', origin='lower', aspect='auto')\n",
    "plt.show()\n",
    "\n",
    "np.min(stamp), np.max(stamp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# combine all words from a given prefix into a dataset of 'stamps'\n",
    "import pickle\n",
    "\n",
    "def create_dataset_from_folders(prefix, save_as='.pkl'):\n",
    "    words = sentences[prefix].split(' ')\n",
    "    stamps, labels = [], []\n",
    "    \n",
    "    for label_i, word in enumerate( words ):\n",
    "        # Find all the files for this word\n",
    "        for stamp_file in os.listdir( os.path.join('data', prefix, word )):\n",
    "            if not f.endswith('.wav'): continue\n",
    "            #print(stamp_file)\n",
    "            stamp = wav_to_stamp(prefix, word, stamp_file)\n",
    "            \n",
    "            stamps.append(stamp)\n",
    "            labels.append(label_i)\n",
    "\n",
    "    if save_as is None: # Return the data directly\n",
    "        return stamps, labels, words\n",
    "    \n",
    "    data_dictionary = dict(\n",
    "        stamp=stamps, label=labels, \n",
    "        rand=np.random.rand( len(labels) ), # This is to enable us to sample the data (based on hurdles)\n",
    "        words=words, \n",
    "    )\n",
    "    pickle.dump(data_dictionary, open(os.path.join('data', prefix+save_as), 'wb'), \n",
    "                protocol=pickle.HIGHEST_PROTOCOL)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#if not os.path.exists('data/num.pkl'):\n",
    "if True:\n",
    "    create_dataset_from_folders('num')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read in the dataset\n",
    "dataset = pickle.load(open(os.path.join('data', 'num.pkl'), 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Plot all of a given 'word'\n",
    "indices = [ i for i,label in enumerate(dataset['label']) if dataset['words'][label]=='six']\n",
    "\n",
    "plt.figure(figsize=(12,2))\n",
    "for i in indices[0:16]:  # at most 16\n",
    "    plt.subplot(2, 8, i+1)  # nrows, ncols, subplot#\n",
    "    plt.imshow(dataset['stamp'][i].T, cmap='gray', origin='lower', interpolation='nearest')\n",
    "    plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now do something similar for 'test files', create a dataset for all the audio files in the given folder\n",
    "\n",
    "def create_dataset_from_adhoc_wavs(prefix, save_as='.pkl'):\n",
    "    stamps, labels, words = [], [], []\n",
    "    \n",
    "    for audio_file in os.listdir( os.path.join('data', prefix) ):\n",
    "        filename_stub, ext = os.path.splitext(audio_file)\n",
    "        if not (ext=='.wav' or ext=='.ogg'): continue\n",
    "            \n",
    "        samples, sample_rate = soundfile.read(audio_file)\n",
    "        sample_feat, energy = get_sample_features(samples, sample_rate)\n",
    "        word_ranges = get_sample_isolated_words(energy, plot=False)\n",
    "    \n",
    "        for i, wr in enumerate(word_ranges):\n",
    "            wr = word_ranges[i]\n",
    "            fac = int(sample_window_step*sample_rate)\n",
    "            segment = samples[ wr[0]*fac:wr[1]*fac ]\n",
    "\n",
    "            stamp = samples_to_stamp(segment, sample_rate)\n",
    "            \n",
    "            stamps.append(stamp)\n",
    "            labels.append(-1)\n",
    "            words.append(\"%s_%d\" % (filename_stub, i))\n",
    "    \n",
    "    data_dictionary = dict(\n",
    "        stamp=stamps, label=labels, \n",
    "        rand=np.random.rand( len(labels) ),\n",
    "        words=words, \n",
    "    )\n",
    "    pickle.dump(data_dictionary, open(os.path.join('data', prefix+save_as), 'wb'), \n",
    "                protocol=pickle.HIGHEST_PROTOCOL)            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_prefix = 'test-'+'num'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "create_dataset_from_adhoc_wavs(test_prefix)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{'fraction_non_alphanumeric': 0.20698949716516404, 'fraction_numerical': 0.012036434612882238, 'mean_word_length': 2.3225507604415965, 'pattern_counts': {'":': 378, '<': 13, '<?xml version=': 0, '>': 23, 'https://': 12, 'lorem ipsum': 0, 'www.': 10, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7580351', 'n_tokens_mistral': 17258, 'n_tokens_neox': 16727, 'n_words': 2943}