<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fft_step   = 12.5/1000. # 12.5ms\n",
    "fft_window = 50.0/1000.  # 50ms\n",
    "\n",
    "audio_filenames = [ './librivox/guidetomen_%02d_rowland_64kb.mp3' % (i,) for i in [1,2,3]]\n",
    "audio_filenames\n",
    "\n",
    "mel_filenames = [ f.replace('.mp3', '.melspectra.hkl') for f in audio_filenames ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_filename_test_idx = 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mel_filename_test = mel_filenames[audio_filename_test_idx]\n",
    "\n",
    "#with open(mel_filename_test.replace('.hkl', '.16_k2.sym'), 'rt') as f:\n",
    "with open(mel_filename_test.replace('.hkl', '.64_k2.sym'), 'rt') as f:\n",
    "    mel_sym_str = f.read()\n",
    "    \n",
    "mel_sym_chars = set(mel_sym_str)\n",
    "mel_sym_dict  = { c:i for i,c in enumerate(sorted(list(mel_sym_chars))) }\n",
    "mel_sym = np.array( [mel_sym_dict[c] for c in mel_sym_str] )\n",
    "\n",
    "mel_sym_silence = mel_sym_dict[' ']\n",
    "\n",
    "mel_sym_str[100:115], mel_sym[100:115], mel_sym.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_one_sec_per_line(s, t_min=0., t_max=None):\n",
    "    #each_line = int(1/fft_step)\n",
    "    if t_max is None: t_max = len(s)*fft_step\n",
    "    for t in np.arange(t_min, t_max, 1.):\n",
    "        i_min = int(t/fft_step)\n",
    "        i_max = int((t+1.)/fft_step)\n",
    "        if i_max>t_max/fft_step: \n",
    "            i_max = int(t_max/fft_step)\n",
    "        print(\"%6.2f %s\" % (t, s[i_min:i_max]) )\n",
    "        \n",
    "print_one_sec_per_line(mel_sym_str, 0.0, 10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compress the sym data, by counting the duplicates, \n",
    "# and storing 'initial char', 'char count' and 'initial char idx'\n",
    "mel_sym_ct, mel_sym_cc, mel_sym_cn = [], [], []\n",
    "prev_c, prev_n = '', 0\n",
    "for t, c in enumerate(mel_sym_str):\n",
    "    if c==prev_c: \n",
    "        prev_n+=1 # Add one to count\n",
    "    else:\n",
    "        mel_sym_cn.append(prev_n)  # Store count of previous char\n",
    "        mel_sym_ct.append(t)       # Start on new char's index\n",
    "        mel_sym_cc.append(c)       # Start on new char's value\n",
    "        prev_c, prev_n = c, 1\n",
    "mel_sym_cn.append(prev_n)  # Store last count value\n",
    "\n",
    "mel_sym_ct = np.array( mel_sym_ct )\n",
    "mel_sym_ci = np.array( [mel_sym_dict[c] for c in mel_sym_cc] )\n",
    "mel_sym_cn = np.array( mel_sym_cn[1:])   # Kill the first value, and convert to numpy\n",
    "\n",
    "(mel_sym_str[66:95],   # Original String \n",
    " mel_sym_cc[0:10],     # Distinct characters\n",
    " mel_sym_ci[0:10],     # Distinct symbol indicies\n",
    " mel_sym_cn[0:10],     #   x number of repetitions\n",
    " mel_sym_ct[0:10],     # mel time\n",
    " mel_sym_ci.shape[0],) # total number of mels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in text as words\n",
    "\n",
    "# Create initial array of word starts\n",
    "#   with initial guess of maximum error bars\n",
    "# Create map of word -> word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set #FILE: = #FILE:  guidetomen_02_rowland_64kb.mp3\n",
    "#Set #OFFSET_START: = #OFFSET_START: 7.0\n",
    "#Set #OFFSET_END: = #OFFSET_END: 613.0\n",
    "offset_start, offset_end = 7.0, 613.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(mel_filename_test.replace('.melspectra.hkl', '.txt'), 'rt') as f:\n",
    "    mel_txt = f.read()\n",
    "\n",
    "txt_arr = mel_txt.replace('\\n', ' ').split(' ')\n",
    "txt_arr.insert(0, '#EOS') # Extra one at start\n",
    "txt_arr.insert(0, '') # Helps start process\n",
    "len(txt_arr), ','.join( txt_arr[0:10] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_spans = []\n",
    "def add_sentence_span(span_start_index, span):\n",
    "    if len(span)>0:\n",
    "        sentence_spans.append(dict(\n",
    "            t_start= span_start_index,\n",
    "            t_end  = span_start_index+len(span),\n",
    "            span   = span,\n",
    "        ))\n",
    "\n",
    "span_start, span_words = -1, []  # Indices and text\n",
    "for idx, w in enumerate(txt_arr):\n",
    "    if span_start<0: \n",
    "        span_start = idx\n",
    "    if w=='#EOS':\n",
    "        add_sentence_span(span_start, span_words)\n",
    "        span_start, span_words = -1, []\n",
    "        continue \n",
    "    span_words.append(w)\n",
    "add_sentence_span(span_start, span_words)\n",
    "\n",
    "len(sentence_spans), sentence_spans[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up some matrices to fill in - these are timings in seconds \n",
    "txt_length_est = np.array( [ len(s) for s in txt_arr ] )\n",
    "#txt_err = np.zeros_like( txt_starts )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Total up the lengths of the words (in characters) \n",
    "#  - the initial timing guess is going to be proportional \n",
    "txt_length_est[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_err_min = 5. # Seconds\n",
    "txt_err_pct = 0.10 # i.e. plus or minus this amount of 'unknown length'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need function to fix starts and errs for every word in txt_arr\n",
    "#   given a dict of word_index to known starts+errs\n",
    "known_starts = {\n",
    "    0 : (offset_start, txt_err_min),\n",
    "    (txt_length_est.shape[0]-1) : (offset_end, txt_err_min),\n",
    "}\n",
    "known_starts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_starts_and_errs(known_starts):\n",
    "    txt_starts  = np.zeros( txt_length_est.shape )\n",
    "    txt_err_fwd = np.zeros_like( txt_starts )\n",
    "    txt_err_bwd = np.zeros_like( txt_starts )\n",
    "    \n",
    "    known_start_i = sorted( known_starts.keys() )\n",
    "    for i_start, i_end in  zip( known_start_i[:-1], known_start_i[1:]):\n",
    "        v_start, v_end = known_starts[i_start], known_starts[i_end]\n",
    "        \n",
    "        # Create a span including updated duration estimates\n",
    "        actual_duration = (v_end[0]-v_start[0])\n",
    "        length_est = txt_length_est[i_start:i_end] # Copy range\n",
    "        length_adj = length_est/length_est.sum()*actual_duration\n",
    "        \n",
    "        # Put the span into the txt_starts array\n",
    "        cs_fwd = np.cumsum(length_adj)\n",
    "        cs_bwd = np.flip(np.flip(length_adj, 0).cumsum(), 0)\n",
    "        \n",
    "        txt_starts[i_start] = v_start[0]\n",
    "        txt_starts[i_start+1:i_end+1] = v_start[0] + cs_fwd\n",
    "\n",
    "        txt_err_fwd[i_start] = v_start[1]\n",
    "        txt_err_fwd[i_start+1:i_end+1] = v_start[1] + cs_fwd*txt_err_pct\n",
    "\n",
    "        txt_err_bwd[i_end] = v_end[1]\n",
    "        txt_err_bwd[i_start:i_end] = v_end[1] + cs_bwd*txt_err_pct\n",
    "    \n",
    "    return txt_starts, np.minimum(txt_err_fwd, txt_err_bwd)\n",
    "\n",
    "#np.cumsum( np.array( [66,3,72,12,42] ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_starts, txt_err = create_starts_and_errs(known_starts)\n",
    "\n",
    "txt_length_est[0:10], txt_starts[0:10], txt_starts[-10:]\n",
    "#txt_err[0:5], txt_err[ txt_starts.shape[0]//2-5:txt_starts.shape[0]//2 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Old version\n",
    "#txt_starts = txt_length_est.cumsum()\n",
    "#txt_starts = offset_start + txt_starts*(offset_end-offset_start)/txt_starts[-1]\n",
    "#txt_starts[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Old version\n",
    "#txt_err = np.array( [ (i-txt_starts.shape[0]) for i, txt in enumerate(txt_arr) ] )\n",
    "#txt_err = np.square(txt_err)\n",
    "#txt_err = txt_err.max() - txt_err # Now a parabola peaking in the middle\n",
    "#\n",
    "#txt_err_scale = (txt_err_max - txt_err_min) / txt_err[ txt_starts.shape[0]//2 ]\n",
    "#\n",
    "#txt_err = txt_err*txt_err_scale + txt_err_min\n",
    "#\n",
    "#txt_err[0:5], txt_err[ txt_starts.shape[0]//2:txt_starts.shape[0]//2+5 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Global alignment\n",
    "\n",
    "# Idea : Train up a word embedding based on range (within error bars)\n",
    "# Function to map word to set of ranges\n",
    "# Function to convert ranges into a % of whole\n",
    "# Table of words with % coverage (to check whether that's a doable idea)\n",
    "# Find word-range average vector (vs. not-in-word-range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_to_idx={}\n",
    "for i, w in enumerate(txt_arr):\n",
    "    if w not in word_to_idx: word_to_idx[w]=[]\n",
    "    word_to_idx[w].append( i )\n",
    "word_to_idx['heart'], len(txt_arr), len(word_to_idx)  #bachelor mere"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_range_mask(w):\n",
    "    mask = np.zeros_like(mel_sym)\n",
    "    for i in word_to_idx.get(w, []):\n",
    "        i_min = (txt_starts[i]-txt_err[i])/fft_step\n",
    "        if i_min<0: i_min=0/fft_step\n",
    "            \n",
    "        i_next = i+1 if i<txt_starts.shape[0]-1 else i\n",
    "        i_max = (txt_starts[i_next]+txt_err[i])/fft_step\n",
    "        if i_max>mask.shape[0]: i_max=mask.shape[0]/fft_step\n",
    "            \n",
    "        #print(\"(i_min, i_max) = \", i_min, i_max)\n",
    "        mask[ int(i_min):int(i_max) ] = 1.\n",
    "    return mask\n",
    "\n",
    "word_mask=word_range_mask('love')  # bachelor mere woman man the\n",
    "np.sum(word_mask) / word_mask.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's look at word frequencies, and mask coverage\n",
    "words_freq_ordered = sorted(word_to_idx.keys(), key=lambda k: -len(word_to_idx[k]))\n",
    "len(words_freq_ordered), words_freq_ordered[0], words_freq_ordered[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for w in words_freq_ordered:\n",
    "    n = len(word_to_idx[w])\n",
    "    if n<2: continue # Not enough for anything...\n",
    "    word_mask=word_range_mask(w)\n",
    "    coverage=np.sum(word_mask) / word_mask.shape[0]\n",
    "    print(\"%6.2f%%, %4d, %s\" % (coverage*100., n, w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sweet spot is defined by maximising # of examples, while minimising\n",
    "#   probability of mis-identification (i.e. low coverage is good)\n",
    "# Perhaps, though, need to avoid overlapping masks (if possible?)\n",
    "\n",
    "# Candidates (?) : love, one, never, how, marriage, give, me, right, greatest, getting\n",
    "# Except : bachelor's and bachelors, \n",
    "# Except : man(~woman), woman's, man's\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mel_sym_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now create histogram of symbol frequencies corresponding to mask\n",
    "def histogram_freqs(mask, remove_silence=True):\n",
    "    print(np.sum(mask) / mask.shape[0])\n",
    "    #inside = bincount(mel_sym, weights=mask)\n",
    "    n_sym = len(mel_sym_chars)\n",
    "    \n",
    "    inside_bins  = np.bincount(mel_sym, weights=mask)\n",
    "    outside_bins = np.bincount(mel_sym, weights=1-mask)\n",
    "    \n",
    "    if remove_silence:\n",
    "        inside_bins[mel_sym_silence]=0\n",
    "        outside_bins[mel_sym_silence]=0\n",
    "    \n",
    "    plt.figure(figsize=(12,4))\n",
    "    rects1 = plt.bar(np.arange(0, n_sym)-.2, width=0.4, color='r',\n",
    "                     height=inside_bins/np.sum(inside_bins))\n",
    "    rects2 = plt.bar(np.arange(0, n_sym)+.2, width=0.4, color='b',\n",
    "                     height=outside_bins/np.sum(outside_bins))\n",
    "\n",
    "    plt.xlabel('Symbol#')\n",
    "    plt.ylabel('Freq')\n",
    "    plt.xticks(np.arange(0, n_sym, 1.0))\n",
    "    plt.grid(True)\n",
    "    plt.show()\n",
    "    \n",
    "histogram_freqs(word_range_mask('greatest'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's find the ranges of the actual sounds in the \n",
    "# speech audio - ignoring short silences\n",
    "\n",
    "#silence_is_short_len = 8  # 100ms\n",
    "silence_is_short_len = 40  # 500ms\n",
    "\n",
    "audio_spans = []\n",
    "def add_span(span_start_index, s_i, s_n):\n",
    "    if len(s_i)>0:\n",
    "        span=[]\n",
    "        for sym, count in zip(s_i, s_n):\n",
    "            span.extend( [sym]*count )\n",
    "        audio_spans.append(dict(\n",
    "            t_start=span_start_index,\n",
    "            t_end  =span_start_index+len(span),\n",
    "            syms=s_i,\n",
    "            count=s_n,\n",
    "            span=span,\n",
    "        ))\n",
    "        #print(span_start_index, \n",
    "        #      mel_sym_str[span_start_index:span_start_index+len(span)])\n",
    "\n",
    "span_start, span_i, span_n = -1, [], []  # Indices and counts\n",
    "for idx, c in enumerate(mel_sym_cc):\n",
    "    if span_start<0: \n",
    "        span_start = mel_sym_ct[idx]\n",
    "    ci, cn = mel_sym_ci[idx], mel_sym_cn[idx]\n",
    "    \n",
    "    if ci==mel_sym_silence and cn>silence_is_short_len:\n",
    "        #print(cn)\n",
    "        add_span(span_start, span_i, span_n)\n",
    "        span_start, span_i, span_n = -1, [], []\n",
    "        continue \n",
    "        \n",
    "    span_i.append(ci)\n",
    "    span_n.append(cn)\n",
    "add_span(span_start, span_i, span_n)\n",
    "\n",
    "len(audio_spans), #audio_spans[92]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Local alignment (looking within word-error-bar ranges only)\n",
    "\n",
    "# Idea : Mostly textual alignment\n",
    "# Have a look symbols in appropriate ranges for several word examples\n",
    "# See whether a simple optimisation can align multiple segments\n",
    "# Would reduce error bars massively\n",
    "# Possibly : \n",
    "#   http://mlpy.sourceforge.net/docs/3.4/lcs.html#standard-lcs  (GPL3, though)\n",
    "#   https://github.com/Samnsparky/py_common_subseq (MIT)\n",
    "#   https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Longest_common_substring#Python_3 \n",
    "#   https://docs.python.org/3/library/difflib.html (not quite...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  !pip install py_common_subseq\n",
    "#import py_common_subseq as subseq  # But doesn't output alignment\n",
    "\n",
    "# Homebrew version : does what we actually want (return array of index correspondences)\n",
    "#  See : https://en.wikipedia.org/wiki/Longest_common_subsequence_problem\n",
    "def lcs(a_arr, b_arr, just_length=True):  \n",
    "    m = np.zeros( (len(a_arr)+1, len(b_arr)+1) )\n",
    "    # offset all the i and j by 1, since we need blank first row+col\n",
    "    for i, a in enumerate(a_arr):\n",
    "        for j, b in enumerate(b_arr):\n",
    "            if a == b:\n",
    "                m[i+1, j+1] = m[i, j] + 1\n",
    "            else:\n",
    "                m[i+1, j+1] = max(m[i+1, j], m[i, j+1])\n",
    "    #print(m)\n",
    "    if just_length:\n",
    "        return m[-1, -1]\n",
    "                \n",
    "    #?  a_i=np.zeros( (lengths[-1,-1], ))\n",
    "    a_i, b_i = [], []\n",
    "    i, j = len(a_arr), len(b_arr)\n",
    "    while i>0 and j>0:\n",
    "        if m[i, j] == m[i-1, j]:\n",
    "            i -= 1\n",
    "        elif m[i, j] == m[i, j-1]:\n",
    "            j -= 1\n",
    "        else: # a_arr[i-1] == b_arr[j-1]\n",
    "            a_i.append(i-1)\n",
    "            b_i.append(j-1)\n",
    "            i -= 1\n",
    "            j -= 1\n",
    "    \n",
    "    return a_i[::-1], b_i[::-1]\n",
    "\n",
    "lcs([17, 9,99,7,4,8,3,7,5,2,4,1,2,1,2,4,5,6],\n",
    "    [1, 17,11,7,4,8,3,7,0,2,4,0,2,  2,  5,6], just_length=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# So now let's pick a word, and find its shortest range\n",
    "# Then go through 1 second (=80 position) chunks within that range,\n",
    "# and see how that matches each of the ~1000 spans \n",
    "# (possibly by searching over overlapping 2 second chunks) \n",
    "\n",
    "word_probe = 'greatest'\n",
    "#word_probe = 'getting'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find all the audio_spans that overlap the mask for this word\n",
    "mask_probe = word_range_mask(word_probe)\n",
    "spans_probe = [ s for s in audio_spans \n",
    "                if mask_probe[s['t_start']]>0 \n",
    "                or mask_probe[s['t_end']]>0 ]\n",
    "len(spans_probe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Now go through these spans in segments of 1 second, in 0.5 second increments...\n",
    "probe_win = int(1./fft_step)\n",
    "probe_step = probe_win//2\n",
    "\n",
    "probe_results=None\n",
    "def get_probe_results(spans_probe):\n",
    "    probe_res=[]  # Will be a list of probe score arrays\n",
    "    for sp in spans_probe:\n",
    "        #for i in range(sp['t_start'], sp['t_end'], probe_step):\n",
    "        for i in range(0, len(sp['span']), probe_step):\n",
    "            segment_probe = sp['span'][i:i+probe_win]\n",
    "            #print( i, sp['span'] )\n",
    "            #print( segment_probe )\n",
    "            #break\n",
    "            # And scan the segment_probe across all the spans in the audio,\n",
    "            #   With similar windowing idea\n",
    "\n",
    "            audio_res = []\n",
    "            for sa in audio_spans:\n",
    "                #for j in range(sa['t_start'], sa['t_end'], probe_step):\n",
    "                for j in range(0, len(sa['span']), probe_step):\n",
    "                    segment_audio = sa['span'][j:j+probe_win]\n",
    "\n",
    "                    probe_minlen = min(len(segment_probe), len(segment_audio))\n",
    "                    audio_res.append( lcs(segment_audio, segment_probe)/(probe_minlen+1) )\n",
    "\n",
    "            probe_res.append( audio_res )\n",
    "        #break\n",
    "\n",
    "    return np.array( probe_res )\n",
    "\n",
    "#probe_results = get_probe_results(spans_probe)\n",
    "#probe_results.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#[5,6,7,6,8][2:13]\n",
    "if probe_results is not None:\n",
    "    probe_results.sum() \n",
    "    \n",
    "    plt.figure(figsize = (20,2))\n",
    "    plt.imshow(probe_results, cmap='Purples')  #, interpolation='nearest'\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random, string\n",
    "\n",
    "# pip install soundfile\n",
    "import soundfile\n",
    "import librosa\n",
    "\n",
    "from IPython.display import Audio as audio_playback_widget\n",
    "\n",
    "os.makedirs('./data/tmp', exist_ok=True)\n",
    "\n",
    "audio_filename_test = audio_filenames[audio_filename_test_idx]\n",
    "audio_samples, _sample_rate = librosa.core.load(audio_filename_test, sr=None)\n",
    "audio_samples = audio_samples/np.max(audio_samples)\n",
    "\n",
    "def play_audio_span(s, autoplay=False):\n",
    "    hsh = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))\n",
    "    f = './data/tmp/%s.wav' % (hsh,)\n",
    "    def ts(t_mel, end=False):  # t_mel to samples\n",
    "        return int( (t_mel*fft_step + (fft_window if end else 0.))*_sample_rate )\n",
    "    audio_span = audio_samples[ts(s['t_start']):ts(s['t_end'], end=True)]\n",
    "    soundfile.write(f, audio_span, samplerate=_sample_rate)\n",
    "    \n",
    "    plt.figure(figsize=(12,2))\n",
    "    plt.plot(audio_span)\n",
    "    #plt.plot(np.arange(s['t_start'],s['t_end'],), audio_span)\n",
    "    #plt.xticks( np.arange(s['t_start'], s['t_end'], 20.), rotation=90 )\n",
    "    plt.grid(True)\n",
    "\n",
    "    plt.show()\n",
    "    \n",
    "    return audio_playback_widget(f, autoplay=autoplay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "play_audio_span(audio_spans[13])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bachelors  (near beginning)\n",
    "print_one_sec_per_line(mel_sym_str, 0./80, 800./80)\n",
    "#play_audio_span(dict(t_start=0, t_end=800), autoplay=True)\n",
    "print()\n",
    "print_one_sec_per_line(mel_sym_str, 640./80, 700./80)\n",
    "play_audio_span(dict(t_start=640, t_end=690), autoplay=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an embedding for all the symbols\n",
    "sym_embed = np.random.normal( size=(len(mel_sym_chars), embedding_dim) )\n",
    "sym_embed = sym_embed / np.linalg.norm(sym_embed, axis=1)[:, np.newaxis]\n",
    "sym_embed[mel_sym_silence, :] = 0.\n",
    "\n",
    "sym_embed[3,:], np.linalg.norm(sym_embed[3,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create combined embedding for each audio_span\n",
    "\n",
    "#        audio_spans.append(dict(\n",
    "#            t_start=span_start_index,\n",
    "#            t_end  =span_start_index+len(span),\n",
    "#            syms=s_i,\n",
    "#            count=s_n,\n",
    "#            span=span,\n",
    "#        ))\n",
    "\n",
    "for s in audio_spans:\n",
    "    #np.bincount( np.array([1,2,6,5,3,3,2,5,1,1,0,1]), minlength=10)\n",
    "    span_bins = np.bincount(mel_sym[ s['t_start']:s['t_end'] ], \n",
    "                            minlength=len(mel_sym_chars))\n",
    "    span_emb = np.dot(span_bins, sym_embed)\n",
    "    span_emb /= np.linalg.norm(span_emb)\n",
    "    s['span_emb'] = span_emb\n",
    "\n",
    "print(audio_spans[10]['span_emb'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For every word (n_occurrences>0) create embedding via masks\n",
    "#   NB: masks depend on current txt_starts, txt_err\n",
    "\n",
    "def create_word_embeddings():\n",
    "    word_embed = dict()\n",
    "    \n",
    "    overall_bins= np.bincount(mel_sym)\n",
    "    overall_emb = np.dot(overall_bins, sym_embed)\n",
    "    #overall_emb /= np.linalg.norm(overall_emb)\n",
    "    \n",
    "    #for w in words_freq_ordered:\n",
    "    for w in word_to_idx.keys():\n",
    "        n = len(word_to_idx[w])\n",
    "        #if n<2: continue # Not enough for anything...\n",
    "        word_mask=word_range_mask(w)\n",
    "        \n",
    "        inside_bins = np.bincount(mel_sym, weights=word_mask)\n",
    "        inside_emb  = np.dot( inside_bins, sym_embed)\n",
    "\n",
    "        outside_emb = overall_emb - inside_emb\n",
    "        word_emb = inside_emb - outside_emb\n",
    "        \n",
    "        norm = np.linalg.norm(word_emb)\n",
    "        if norm>0.:\n",
    "            word_emb /= norm\n",
    "        #else == nonsense\n",
    "\n",
    "        word_embed[w]=word_emb\n",
    "    return word_embed\n",
    "\n",
    "word_embedding = create_word_embeddings()\n",
    "len(word_embedding), word_embedding['love'], word_embedding['bachelor']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create combined embedding for each sentence\n",
    "#   Uses the word_embeddings above (will change if txt_starts, txt_err changes)\n",
    "#txt_arr[:6], txt_arr[-6:]\n",
    "\n",
    "#        sentence_spans.append(dict(\n",
    "#            t_start= span_start_index,\n",
    "#            t_end  = span_start_index+len(span),\n",
    "#            span   = span,\n",
    "#        ))\n",
    "def create_sentence_embedding(word_embedding):\n",
    "    for s in sentence_spans:\n",
    "        #word_idx = [ txt_arr[i] for i in range(s['t_start'], s['t_end']+1) ]\n",
    "        #span_bins = np.bincount(mel_sym[  ], \n",
    "        #                        minlength=len(mel_sym_chars))\n",
    "        #span_emb = np.dot(span_bins, sym_embed)\n",
    "        span_emb = np.zeros( (embedding_dim,) )\n",
    "        for w in s['span']:\n",
    "            span_emb += word_embedding[w]\n",
    "        norm = np.linalg.norm(span_emb)\n",
    "        if norm>0.:\n",
    "            span_emb /= norm\n",
    "        s['span_emb'] = span_emb\n",
    "\n",
    "sentence_embedding = create_sentence_embedding(word_embedding)        \n",
    "print(sentence_spans[10]['span_emb'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do a vector DTW across spans and sentences\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   Graph it out\n",
    "#   Output new 'known_starts' dictionary\n",
    "#   Update txt_starts, txt_err\n",
    "# Function to output sentence and audio in that span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://colinraffel.com/publications/thesis.pdf\n",
    "\n",<commit_msg>DTW first pass - looks SAD<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fft_step   = 12.5/1000. # 12.5ms\n",
    "fft_window = 50.0/1000.  # 50ms\n",
    "\n",
    "audio_filenames = [ './librivox/guidetomen_%02d_rowland_64kb.mp3' % (i,) for i in [1,2,3]]\n",
    "audio_filenames\n",
    "\n",
    "mel_filenames = [ f.replace('.mp3', '.melspectra.hkl') for f in audio_filenames ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_filename_test_idx = 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mel_filename_test = mel_filenames[audio_filename_test_idx]\n",
    "\n",
    "#with open(mel_filename_test.replace('.hkl', '.16_k2.sym'), 'rt') as f:\n",
    "with open(mel_filename_test.replace('.hkl', '.64_k2.sym'), 'rt') as f:\n",
    "    mel_sym_str = f.read()\n",
    "    \n",
    "mel_sym_chars = set(mel_sym_str)\n",
    "mel_sym_dict  = { c:i for i,c in enumerate(sorted(list(mel_sym_chars))) }\n",
    "mel_sym = np.array( [mel_sym_dict[c] for c in mel_sym_str] )\n",
    "\n",
    "mel_sym_silence = mel_sym_dict[' ']\n",
    "\n",
    "mel_sym_str[100:115], mel_sym[100:115], mel_sym.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_one_sec_per_line(s, t_min=0., t_max=None):\n",
    "    #each_line = int(1/fft_step)\n",
    "    if t_max is None: t_max = len(s)*fft_step\n",
    "    for t in np.arange(t_min, t_max, 1.):\n",
    "        i_min = int(t/fft_step)\n",
    "        i_max = int((t+1.)/fft_step)\n",
    "        if i_max>t_max/fft_step: \n",
    "            i_max = int(t_max/fft_step)\n",
    "        print(\"%6.2f %s\" % (t, s[i_min:i_max]) )\n",
    "        \n",
    "print_one_sec_per_line(mel_sym_str, 0.0, 10.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compress the sym data, by counting the duplicates, \n",
    "# and storing 'initial char', 'char count' and 'initial char idx'\n",
    "mel_sym_ct, mel_sym_cc, mel_sym_cn = [], [], []\n",
    "prev_c, prev_n = '', 0\n",
    "for t, c in enumerate(mel_sym_str):\n",
    "    if c==prev_c: \n",
    "        prev_n+=1 # Add one to count\n",
    "    else:\n",
    "        mel_sym_cn.append(prev_n)  # Store count of previous char\n",
    "        mel_sym_ct.append(t)       # Start on new char's index\n",
    "        mel_sym_cc.append(c)       # Start on new char's value\n",
    "        prev_c, prev_n = c, 1\n",
    "mel_sym_cn.append(prev_n)  # Store last count value\n",
    "\n",
    "mel_sym_ct = np.array( mel_sym_ct )\n",
    "mel_sym_ci = np.array( [mel_sym_dict[c] for c in mel_sym_cc] )\n",
    "mel_sym_cn = np.array( mel_sym_cn[1:])   # Kill the first value, and convert to numpy\n",
    "\n",
    "(mel_sym_str[66:95],   # Original String \n",
    " mel_sym_cc[0:10],     # Distinct characters\n",
    " mel_sym_ci[0:10],     # Distinct symbol indicies\n",
    " mel_sym_cn[0:10],     #   x number of repetitions\n",
    " mel_sym_ct[0:10],     # mel time\n",
    " mel_sym_ci.shape[0],) # total number of mels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in text as words\n",
    "\n",
    "# Create initial array of word starts\n",
    "#   with initial guess of maximum error bars\n",
    "# Create map of word -> word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set #FILE: = #FILE:  guidetomen_02_rowland_64kb.mp3\n",
    "#Set #OFFSET_START: = #OFFSET_START: 7.0\n",
    "#Set #OFFSET_END: = #OFFSET_END: 613.0\n",
    "offset_start, offset_end = 7.0, 613.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(mel_filename_test.replace('.melspectra.hkl', '.txt'), 'rt') as f:\n",
    "    mel_txt = f.read()\n",
    "\n",
    "txt_arr = mel_txt.replace('\\n', ' ').split(' ')\n",
    "txt_arr.insert(0, '#EOS') # Extra one at start\n",
    "txt_arr.insert(0, '') # Helps start process\n",
    "len(txt_arr), ','.join( txt_arr[0:10] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_spans = []\n",
    "def add_sentence_span(span_start_index, span):\n",
    "    if len(span)>0:\n",
    "        sentence_spans.append(dict(\n",
    "            t_start= span_start_index,\n",
    "            t_end  = span_start_index+len(span),\n",
    "            span   = span,\n",
    "        ))\n",
    "\n",
    "span_start, span_words = -1, []  # Indices and text\n",
    "for idx, w in enumerate(txt_arr):\n",
    "    if span_start<0: \n",
    "        span_start = idx\n",
    "    if w=='#EOS':\n",
    "        add_sentence_span(span_start, span_words)\n",
    "        span_start, span_words = -1, []\n",
    "        continue \n",
    "    span_words.append(w)\n",
    "add_sentence_span(span_start, span_words)\n",
    "\n",
    "len(sentence_spans), sentence_spans[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up some matrices to fill in - these are timings in seconds \n",
    "txt_length_est = np.array( [ len(s) for s in txt_arr ] )\n",
    "#txt_err = np.zeros_like( txt_starts )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Total up the lengths of the words (in characters) \n",
    "#  - the initial timing guess is going to be proportional \n",
    "txt_length_est[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_err_min = 5. # Seconds\n",
    "txt_err_pct = 0.10 # i.e. plus or minus this amount of 'unknown length'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need function to fix starts and errs for every word in txt_arr\n",
    "#   given a dict of word_index to known starts+errs\n",
    "known_starts = {\n",
    "    0 : (offset_start, txt_err_min),\n",
    "    (txt_length_est.shape[0]-1) : (offset_end, txt_err_min),\n",
    "}\n",
    "known_starts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_starts_and_errs(known_starts):\n",
    "    txt_starts  = np.zeros( txt_length_est.shape )\n",
    "    txt_err_fwd = np.zeros_like( txt_starts )\n",
    "    txt_err_bwd = np.zeros_like( txt_starts )\n",
    "    \n",
    "    known_start_i = sorted( known_starts.keys() )\n",
    "    for i_start, i_end in  zip( known_start_i[:-1], known_start_i[1:]):\n",
    "        v_start, v_end = known_starts[i_start], known_starts[i_end]\n",
    "        \n",
    "        # Create a span including updated duration estimates\n",
    "        actual_duration = (v_end[0]-v_start[0])\n",
    "        length_est = txt_length_est[i_start:i_end] # Copy range\n",
    "        length_adj = length_est/length_est.sum()*actual_duration\n",
    "        \n",
    "        # Put the span into the txt_starts array\n",
    "        cs_fwd = np.cumsum(length_adj)\n",
    "        cs_bwd = np.flip(np.flip(length_adj, 0).cumsum(), 0)\n",
    "        \n",
    "        txt_starts[i_start] = v_start[0]\n",
    "        txt_starts[i_start+1:i_end+1] = v_start[0] + cs_fwd\n",
    "\n",
    "        txt_err_fwd[i_start] = v_start[1]\n",
    "        txt_err_fwd[i_start+1:i_end+1] = v_start[1] + cs_fwd*txt_err_pct\n",
    "\n",
    "        txt_err_bwd[i_end] = v_end[1]\n",
    "        txt_err_bwd[i_start:i_end] = v_end[1] + cs_bwd*txt_err_pct\n",
    "    \n",
    "    return txt_starts, np.minimum(txt_err_fwd, txt_err_bwd)\n",
    "\n",
    "#np.cumsum( np.array( [66,3,72,12,42] ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_starts, txt_err = create_starts_and_errs(known_starts)\n",
    "\n",
    "txt_length_est[0:10], txt_starts[0:10], txt_starts[-10:]\n",
    "#txt_err[0:5], txt_err[ txt_starts.shape[0]//2-5:txt_starts.shape[0]//2 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Old version\n",
    "#txt_starts = txt_length_est.cumsum()\n",
    "#txt_starts = offset_start + txt_starts*(offset_end-offset_start)/txt_starts[-1]\n",
    "#txt_starts[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Old version\n",
    "#txt_err = np.array( [ (i-txt_starts.shape[0]) for i, txt in enumerate(txt_arr) ] )\n",
    "#txt_err = np.square(txt_err)\n",
    "#txt_err = txt_err.max() - txt_err # Now a parabola peaking in the middle\n",
    "#\n",
    "#txt_err_scale = (txt_err_max - txt_err_min) / txt_err[ txt_starts.shape[0]//2 ]\n",
    "#\n",
    "#txt_err = txt_err*txt_err_scale + txt_err_min\n",
    "#\n",
    "#txt_err[0:5], txt_err[ txt_starts.shape[0]//2:txt_starts.shape[0]//2+5 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Global alignment\n",
    "\n",
    "# Idea : Train up a word embedding based on range (within error bars)\n",
    "# Function to map word to set of ranges\n",
    "# Function to convert ranges into a % of whole\n",
    "# Table of words with % coverage (to check whether that's a doable idea)\n",
    "# Find word-range average vector (vs. not-in-word-range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_to_idx={}\n",
    "for i, w in enumerate(txt_arr):\n",
    "    if w not in word_to_idx: word_to_idx[w]=[]\n",
    "    word_to_idx[w].append( i )\n",
    "word_to_idx['heart'], len(txt_arr), len(word_to_idx)  #bachelor mere"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_range_mask(w):\n",
    "    mask = np.zeros_like(mel_sym)\n",
    "    for i in word_to_idx.get(w, []):\n",
    "        i_min = (txt_starts[i]-txt_err[i])/fft_step\n",
    "        if i_min<0: i_min=0/fft_step\n",
    "            \n",
    "        i_next = i+1 if i<txt_starts.shape[0]-1 else i\n",
    "        i_max = (txt_starts[i_next]+txt_err[i])/fft_step\n",
    "        if i_max>mask.shape[0]: i_max=mask.shape[0]/fft_step\n",
    "            \n",
    "        #print(\"(i_min, i_max) = \", i_min, i_max)\n",
    "        mask[ int(i_min):int(i_max) ] = 1.\n",
    "    return mask\n",
    "\n",
    "word_mask=word_range_mask('love')  # bachelor mere woman man the\n",
    "np.sum(word_mask) / word_mask.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's look at word frequencies, and mask coverage\n",
    "words_freq_ordered = sorted(word_to_idx.keys(), key=lambda k: -len(word_to_idx[k]))\n",
    "len(words_freq_ordered), words_freq_ordered[0], words_freq_ordered[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for w in words_freq_ordered:\n",
    "    n = len(word_to_idx[w])\n",
    "    if n<2: continue # Not enough for anything...\n",
    "    word_mask=word_range_mask(w)\n",
    "    coverage=np.sum(word_mask) / word_mask.shape[0]\n",
    "    print(\"%6.2f%%, %4d, %s\" % (coverage*100., n, w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sweet spot is defined by maximising # of examples, while minimising\n",
    "#   probability of mis-identification (i.e. low coverage is good)\n",
    "# Perhaps, though, need to avoid overlapping masks (if possible?)\n",
    "\n",
    "# Candidates (?) : love, one, never, how, marriage, give, me, right, greatest, getting\n",
    "# Except : bachelor's and bachelors, \n",
    "# Except : man(~woman), woman's, man's\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mel_sym_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now create histogram of symbol frequencies corresponding to mask\n",
    "def histogram_freqs(mask, remove_silence=True):\n",
    "    print(np.sum(mask) / mask.shape[0])\n",
    "    #inside = bincount(mel_sym, weights=mask)\n",
    "    n_sym = len(mel_sym_chars)\n",
    "    \n",
    "    inside_bins  = np.bincount(mel_sym, weights=mask)\n",
    "    outside_bins = np.bincount(mel_sym, weights=1-mask)\n",
    "    \n",
    "    if remove_silence:\n",
    "        inside_bins[mel_sym_silence]=0\n",
    "        outside_bins[mel_sym_silence]=0\n",
    "    \n",
    "    plt.figure(figsize=(12,4))\n",
    "    rects1 = plt.bar(np.arange(0, n_sym)-.2, width=0.4, color='r',\n",
    "                     height=inside_bins/np.sum(inside_bins))\n",
    "    rects2 = plt.bar(np.arange(0, n_sym)+.2, width=0.4, color='b',\n",
    "                     height=outside_bins/np.sum(outside_bins))\n",
    "\n",
    "    plt.xlabel('Symbol#')\n",
    "    plt.ylabel('Freq')\n",
    "    plt.xticks(np.arange(0, n_sym, 1.0))\n",
    "    plt.grid(True)\n",
    "    plt.show()\n",
    "    \n",
    "histogram_freqs(word_range_mask('greatest'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's find the ranges of the actual sounds in the \n",
    "# speech audio - ignoring short silences\n",
    "\n",
    "#silence_is_short_len = 8  # 100ms\n",
    "silence_is_short_len = 40  # 500ms\n",
    "\n",
    "audio_spans = []\n",
    "def add_span(span_start_index, s_i, s_n):\n",
    "    if len(s_i)>0:\n",
    "        span=[]\n",
    "        for sym, count in zip(s_i, s_n):\n",
    "            span.extend( [sym]*count )\n",
    "        audio_spans.append(dict(\n",
    "            t_start=span_start_index,\n",
    "            t_end  =span_start_index+len(span),\n",
    "            syms=s_i,\n",
    "            count=s_n,\n",
    "            span=span,\n",
    "        ))\n",
    "        #print(span_start_index, \n",
    "        #      mel_sym_str[span_start_index:span_start_index+len(span)])\n",
    "\n",
    "span_start, span_i, span_n = -1, [], []  # Indices and counts\n",
    "for idx, c in enumerate(mel_sym_cc):\n",
    "    if span_start<0: \n",
    "        span_start = mel_sym_ct[idx]\n",
    "    ci, cn = mel_sym_ci[idx], mel_sym_cn[idx]\n",
    "    \n",
    "    if ci==mel_sym_silence and cn>silence_is_short_len:\n",
    "        #print(cn)\n",
    "        add_span(span_start, span_i, span_n)\n",
    "        span_start, span_i, span_n = -1, [], []\n",
    "        continue \n",
    "        \n",
    "    span_i.append(ci)\n",
    "    span_n.append(cn)\n",
    "add_span(span_start, span_i, span_n)\n",
    "\n",
    "len(audio_spans), #audio_spans[92]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Local alignment (looking within word-error-bar ranges only)\n",
    "\n",
    "# Idea : Mostly textual alignment\n",
    "# Have a look symbols in appropriate ranges for several word examples\n",
    "# See whether a simple optimisation can align multiple segments\n",
    "# Would reduce error bars massively\n",
    "# Possibly : \n",
    "#   http://mlpy.sourceforge.net/docs/3.4/lcs.html#standard-lcs  (GPL3, though)\n",
    "#   https://github.com/Samnsparky/py_common_subseq (MIT)\n",
    "#   https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Longest_common_substring#Python_3 \n",
    "#   https://docs.python.org/3/library/difflib.html (not quite...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  !pip install py_common_subseq\n",
    "#import py_common_subseq as subseq  # But doesn't output alignment\n",
    "\n",
    "# Homebrew version : does what we actually want (return array of index correspondences)\n",
    "#  See : https://en.wikipedia.org/wiki/Longest_common_subsequence_problem\n",
    "def lcs(a_arr, b_arr, just_length=True):  \n",
    "    m = np.zeros( (len(a_arr)+1, len(b_arr)+1) )\n",
    "    # offset all the i and j by 1, since we need blank first row+col\n",
    "    for i, a in enumerate(a_arr):\n",
    "        for j, b in enumerate(b_arr):\n",
    "            if a == b:\n",
    "                m[i+1, j+1] = m[i, j] + 1\n",
    "            else:\n",
    "                m[i+1, j+1] = max(m[i+1, j], m[i, j+1])\n",
    "    #print(m)\n",
    "    if just_length:\n",
    "        return m[-1, -1]\n",
    "                \n",
    "    #?  a_i=np.zeros( (lengths[-1,-1], ))\n",
    "    a_i, b_i = [], []\n",
    "    i, j = len(a_arr), len(b_arr)\n",
    "    while i>0 and j>0:\n",
    "        if m[i, j] == m[i-1, j]:\n",
    "            i -= 1\n",
    "        elif m[i, j] == m[i, j-1]:\n",
    "            j -= 1\n",
    "        else: # a_arr[i-1] == b_arr[j-1]\n",
    "            a_i.append(i-1)\n",
    "            b_i.append(j-1)\n",
    "            i -= 1\n",
    "            j -= 1\n",
    "    \n",
    "    return a_i[::-1], b_i[::-1]\n",
    "\n",
    "lcs([17, 9,99,7,4,8,3,7,5,2,4,1,2,1,2,4,5,6],\n",
    "    [1, 17,11,7,4,8,3,7,0,2,4,0,2,  2,  5,6], just_length=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# So now let's pick a word, and find its shortest range\n",
    "# Then go through 1 second (=80 position) chunks within that range,\n",
    "# and see how that matches each of the ~1000 spans \n",
    "# (possibly by searching over overlapping 2 second chunks) \n",
    "\n",
    "word_probe = 'greatest'\n",
    "#word_probe = 'getting'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find all the audio_spans that overlap the mask for this word\n",
    "mask_probe = word_range_mask(word_probe)\n",
    "spans_probe = [ s for s in audio_spans \n",
    "                if mask_probe[s['t_start']]>0 \n",
    "                or mask_probe[s['t_end']]>0 ]\n",
    "len(spans_probe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Now go through these spans in segments of 1 second, in 0.5 second increments...\n",
    "probe_win = int(1./fft_step)\n",
    "probe_step = probe_win//2\n",
    "\n",
    "probe_results=None\n",
    "def get_probe_results(spans_probe):\n",
    "    probe_res=[]  # Will be a list of probe score arrays\n",
    "    for sp in spans_probe:\n",
    "        #for i in range(sp['t_start'], sp['t_end'], probe_step):\n",
    "        for i in range(0, len(sp['span']), probe_step):\n",
    "            segment_probe = sp['span'][i:i+probe_win]\n",
    "            #print( i, sp['span'] )\n",
    "            #print( segment_probe )\n",
    "            #break\n",
    "            # And scan the segment_probe across all the spans in the audio,\n",
    "            #   With similar windowing idea\n",
    "\n",
    "            audio_res = []\n",
    "            for sa in audio_spans:\n",
    "                #for j in range(sa['t_start'], sa['t_end'], probe_step):\n",
    "                for j in range(0, len(sa['span']), probe_step):\n",
    "                    segment_audio = sa['span'][j:j+probe_win]\n",
    "\n",
    "                    probe_minlen = min(len(segment_probe), len(segment_audio))\n",
    "                    audio_res.append( lcs(segment_audio, segment_probe)/(probe_minlen+1) )\n",
    "\n",
    "            probe_res.append( audio_res )\n",
    "        #break\n",
    "\n",
    "    return np.array( probe_res )\n",
    "\n",
    "#probe_results = get_probe_results(spans_probe)\n",
    "#probe_results.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#[5,6,7,6,8][2:13]\n",
    "if probe_results is not None:\n",
    "    probe_results.sum() \n",
    "    \n",
    "    plt.figure(figsize = (20,2))\n",
    "    plt.imshow(probe_results, cmap='Purples')  #, interpolation='nearest'\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random, string\n",
    "\n",
    "# pip install soundfile\n",
    "import soundfile\n",
    "import librosa\n",
    "\n",
    "from IPython.display import Audio as audio_playback_widget\n",
    "\n",
    "os.makedirs('./data/tmp', exist_ok=True)\n",
    "\n",
    "audio_filename_test = audio_filenames[audio_filename_test_idx]\n",
    "audio_samples, _sample_rate = librosa.core.load(audio_filename_test, sr=None)\n",
    "audio_samples = audio_samples/np.max(audio_samples)\n",
    "\n",
    "def play_audio_span(s, autoplay=False):\n",
    "    hsh = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))\n",
    "    f = './data/tmp/%s.wav' % (hsh,)\n",
    "    def ts(t_mel, end=False):  # t_mel to samples\n",
    "        return int( (t_mel*fft_step + (fft_window if end else 0.))*_sample_rate )\n",
    "    audio_span = audio_samples[ts(s['t_start']):ts(s['t_end'], end=True)]\n",
    "    soundfile.write(f, audio_span, samplerate=_sample_rate)\n",
    "    \n",
    "    plt.figure(figsize=(12,2))\n",
    "    plt.plot(audio_span)\n",
    "    #plt.plot(np.arange(s['t_start'],s['t_end'],), audio_span)\n",
    "    #plt.xticks( np.arange(s['t_start'], s['t_end'], 20.), rotation=90 )\n",
    "    plt.grid(True)\n",
    "\n",
    "    plt.show()\n",
    "    \n",
    "    return audio_playback_widget(f, autoplay=autoplay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "play_audio_span(audio_spans[13])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bachelors  (near beginning)\n",
    "print_one_sec_per_line(mel_sym_str, 0./80, 800./80)\n",
    "#play_audio_span(dict(t_start=0, t_end=800), autoplay=True)\n",
    "print()\n",
    "print_one_sec_per_line(mel_sym_str, 640./80, 700./80)\n",
    "play_audio_span(dict(t_start=640, t_end=690), autoplay=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an embedding for all the symbols\n",
    "sym_embed = np.random.normal( size=(len(mel_sym_chars), embedding_dim) )\n",
    "sym_embed = sym_embed / np.linalg.norm(sym_embed, axis=1)[:, np.newaxis]\n",
    "sym_embed[mel_sym_silence, :] = 0.\n",
    "\n",
    "sym_embed[3,:], np.linalg.norm(sym_embed[3,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create combined embedding for each audio_span\n",
    "\n",
    "#        audio_spans.append(dict(\n",
    "#            t_start=span_start_index,\n",
    "#            t_end  =span_start_index+len(span),\n",
    "#            syms=s_i,\n",
    "#            count=s_n,\n",
    "#            span=span,\n",
    "#        ))\n",
    "\n",
    "audio_spans_embedding = np.zeros( (len(audio_spans), embedding_dim))\n",
    "for i, s in enumerate(audio_spans):\n",
    "    #np.bincount( np.array([1,2,6,5,3,3,2,5,1,1,0,1]), minlength=10)\n",
    "    span_bins = np.bincount(mel_sym[ s['t_start']:s['t_end'] ], \n",
    "                            minlength=len(mel_sym_chars))\n",
    "    span_emb = np.dot(span_bins, sym_embed)\n",
    "    span_emb /= np.linalg.norm(span_emb)\n",
    "    audio_spans_embedding[i,:] = span_emb\n",
    "\n",
    "print(audio_spans_embedding[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For every word (n_occurrences>0) create embedding via masks\n",
    "#   NB: masks depend on current txt_starts, txt_err\n",
    "\n",
    "def create_word_embeddings():\n",
    "    word_embed = dict()\n",
    "    \n",
    "    overall_bins= np.bincount(mel_sym)\n",
    "    overall_emb = np.dot(overall_bins, sym_embed)\n",
    "    #overall_emb /= np.linalg.norm(overall_emb)\n",
    "    \n",
    "    #for w in words_freq_ordered:\n",
    "    for w in word_to_idx.keys():\n",
    "        n = len(word_to_idx[w])\n",
    "        #if n<2: continue # Not enough for anything...\n",
    "        word_mask=word_range_mask(w)\n",
    "        \n",
    "        inside_bins = np.bincount(mel_sym, weights=word_mask)\n",
    "        inside_emb  = np.dot( inside_bins, sym_embed)\n",
    "\n",
    "        outside_emb = overall_emb - inside_emb\n",
    "        word_emb = inside_emb - outside_emb\n",
    "        \n",
    "        norm = np.linalg.norm(word_emb)\n",
    "        if norm>0.:\n",
    "            word_emb /= norm\n",
    "        #else == nonsense\n",
    "\n",
    "        word_embed[w]=word_emb\n",
    "    return word_embed\n",
    "\n",
    "word_embedding = create_word_embeddings()\n",
    "len(word_embedding), word_embedding['love'], word_embedding['bachelor']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create combined embedding for each sentence\n",
    "#   Uses the word_embeddings above (will change if txt_starts, txt_err changes)\n",
    "#txt_arr[:6], txt_arr[-6:]\n",
    "\n",
    "#        sentence_spans.append(dict(\n",
    "#            t_start= span_start_index,\n",
    "#            t_end  = span_start_index+len(span),\n",
    "#            span   = span,\n",
    "#        ))\n",
    "\n",
    "def create_sentence_embedding(word_embedding):\n",
    "    ss_embedding = np.zeros( (len(sentence_spans), embedding_dim))\n",
    "    for i, s in enumerate(sentence_spans):\n",
    "        #word_idx = [ txt_arr[i] for i in range(s['t_start'], s['t_end']+1) ]\n",
    "        #span_bins = np.bincount(mel_sym[  ], \n",
    "        #                        minlength=len(mel_sym_chars))\n",
    "        #span_emb = np.dot(span_bins, sym_embed)\n",
    "        span_emb = np.zeros( (embedding_dim,) )\n",
    "        for w in s['span']:\n",
    "            span_emb += word_embedding[w]\n",
    "        norm = np.linalg.norm(span_emb)\n",
    "        if norm>0.:\n",
    "            span_emb /= norm\n",
    "        ss_embedding[i, :] = span_emb\n",
    "    return ss_embedding\n",
    "\n",
    "sentence_spans_embedding = create_sentence_embedding(word_embedding)        \n",
    "print(sentence_spans_embedding[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do a vector DTW across spans and sentences\n",
    "sentence_spans_embedding.shape, audio_spans_embedding.shape\n",
    "#((73, 8), (117, 8))\n",
    " \n",
    "D, wp = librosa.dtw(audio_spans_embedding.T, \n",
    "                    sentence_spans_embedding.T, \n",
    "                    metric='cosine', subseq=False, )\n",
    "D.shape, wp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   Graph out the DTW path\n",
    "#     See : https://musicinformationretrieval.com/dtw_example.html\n",
    "plt.imshow(D.T, aspect='auto', origin='lower', interpolation='nearest', cmap='gray')\n",
    "plt.plot(wp[:,0], wp[:,1], 'r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=32; sentence_spans_embedding[i,:], ' '.join(sentence_spans[i]['span'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   Output new 'known_starts' dictionary\n",
    "#   Update txt_starts, txt_err\n",
    "# Function to output sentence and audio in that span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://colinraffel.com/publications/thesis.pdf\n",
    "\n",
{'fraction_non_alphanumeric': 0.20094659243747554, 'fraction_numerical': 0.013603010027750821, 'mean_word_length': 2.3866466789667897, 'pattern_counts': {'":': 514, '<': 15, '<?xml version=': 0, '>': 29, 'https://': 9, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2271686', 'n_tokens_mistral': 23808, 'n_tokens_neox': 23276, 'n_words': 4091}