<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.random import RandomState\n",
    "\n",
    "from fgn import fbm\n",
    "\n",
    "import os, re\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "N = 2**19 + 1\n",
    "generator1 = fbm( N = N, H = .80 )\n",
    "generator1.set_rnd( RandomState( ) )\n",
    "T1, X1 = generator1( )\n",
    "\n",
    "generator2 = fbm( N = N, H = .85 )\n",
    "generator2.set_rnd( RandomState( ) )\n",
    "T2, X2 = generator2( )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Paste the sample paths\n",
    "X = np.concatenate( ( X1[:-1], X2 + X1[ -1 ] ) )\n",
    "T = np.concatenate( ( T1[:-1], T2 + T1[ -1 ] ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print np.subtract( *np.percentile( np.diff( X ), [ 75, 25 ] ) )\n",
    "print np.std( np.diff( X ) )\n",
    "print np.median( np.abs( np.diff( X ) ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# T = np.sort( np.random.uniform( size = 2**10 + 1 ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from crossing_tree import xtree_build, xtree_build_old\n",
    "window = 0.05\n",
    "## Prepare the sample times for running hurst estimation. Allow at\n",
    "##  most \"window\" overlap between data used for the estimates.\n",
    "## Do not make assumptions about the spacing of the times in (T, X).\n",
    "##  Except that it is sorted in ascending order.\n",
    "bins = np.arange( T[ 0 ], T[-1], window )\n",
    "binx = np.searchsorted( T, bins, side = 'left' )\n",
    "hurst = np.empty( len( binx ) - 25, dtype = np.float )\n",
    "for i,j,k in zip( binx[:-25], binx[25:], range( hurst.shape[ 0 ] ) ) :\n",
    "    S, Y = T[i:j], X[i:j]\n",
    "    delta = ( 2**5 ) * np.median( np.abs( np.diff( Y ) ) )\n",
    "    Tnk, Xnk, Znk, Vnk, Wnk = xtree_build( S, Y, delta = delta, max_height = 1 )\n",
    "    hurst[ k ] = np.log( 2 ) / np.log( np.average( np.concatenate( Znk[1:] ) ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A process $X(t)$ is self-similar with hurst exponent $H$ is \n",
    "$$ X(a t) \\overset{\\mathcal{D}}{=} a^H X(t)$$ for all $t\\in [0,1]$ and $a>0$, where equality is understood as equality of all finite vector distributions.\n",
    "Since $a$ is arbitrary positive real number the self-similarity definition is equivalent to\n",
    "$$ X(\\frac{t}{a}) \\overset{\\mathcal{D}}{=} a^{-H} X(t)$$\n",
    "which reduces to\n",
    "$$ X(s) \\overset{\\mathcal{D}}{=} a^{-H} X(a s)$$\n",
    "\n",
    "For the crossing tree\n",
    "$$ X(s) \\overset{\\mathcal{D}}{=} 2^{-k} X( \\mu^k s) = (\\mu^k)^{- \\frac{\\log 2}{\\log \\mu}} X( \\mu^k s)$$\n",
    "whence the followign Hurst exponent estimator could be constructed:\n",
    "$$\\hat{H} = \\frac{\\log 2}{\\log \\mu} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Znk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.plot( hurst )\n",
    "# plt."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot( T, X )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def running_hurst( T, X, window = .05 ) :\n",
    "index = np.nonzero( ( T > window ) & ( T < 1 - window ) )[ 0 ]\n",
    "for i in index :\n",
    "## Select the \n",
    "    mask = ( np.abs( T - T[ i ] ) <= window )\n",
    "    delta = ( 2**3 ) * np.median( np.abs( np.diff( X[ mask ] ) ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Tnk, Xnk, Znk, Vnk, Wnk = xtree_build( T[ mask ], X[ mask ], delta = delta, max_height = 6 - 3 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.log( 2 ) / np.log( np.average( np.concatenate( Znk[1:] ) ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Cjn = np.reshape( np.sum( Djnk, axis = ( 2, ), dtype = np.float ), Djnk.shape[ :-1 ] + ( 1, ) )\n",
    "Zk = 2 * np.reshape( 1 + np.arange( Djnk.shape[ -1 ] ), ( 1, 1, Djnk.shape[ -1 ] ) )\n",
    "Pjnk = np.where( Cjn > 0, Djnk / Cjn, 0 )\n",
    "Mjn = np.reshape( np.sum( Pjnk * Zk, axis = ( 2, ) ), Cjn.shape )\n",
    "Hjn = np.log( 2 ) / np.log( Mjn )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "delta = np.subtract( *np.percentile( np.diff( X ), [ 75, 25 ] ) )\n",
    "# delta = np.std( np.diff( X ) )\n",
    "# delta = np.median( np.abs( np.diff( X ) ) )\n",
    "Tnk, Xnk, Znk, Vnk, Wnk = xtree_build( T, X, delta = delta )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
<commit_msg>Further investigations into Geoffrey\'s proposal<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.random import RandomState\n",
    "\n",
    "from fgn import fbm\n",
    "\n",
    "import os, re\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "N = 2**15 + 1\n",
    "generator1 = fbm( N = N, H = .80 )\n",
    "generator1.set_rnd( RandomState( ) )\n",
    "T1, X1 = generator1( )\n",
    "\n",
    "generator2 = fbm( N = N, H = .95 )\n",
    "generator2.set_rnd( RandomState( ) )\n",
    "T2, X2 = generator2( )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Paste the sample paths\n",
    "X = np.concatenate( ( X1[:-1], X2 + X1[ -1 ] ) )\n",
    "T = np.concatenate( ( T1[:-1], T2 + T1[ -1 ] ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print np.subtract( *np.percentile( np.diff( X ), [ 75, 25 ] ) )\n",
    "print np.std( np.diff( X ) )\n",
    "print np.median( np.abs( np.diff( X ) ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# T = np.sort( np.random.uniform( size = 2**10 + 1 ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from crossing_tree import xtree_build, xtree_build_old\n",
    "window = 0.005\n",
    "L = int( T[ -1 ] / ( 2 * window ) )\n",
    "## Prepare the sample times for running hurst estimation. Allow at\n",
    "##  most \"window\" overlap between data used for the estimates.\n",
    "## Do not make assumptions about the spacing of the times in (T, X).\n",
    "##  Except that it is sorted in ascending order.\n",
    "bins = np.concatenate( ( np.arange( T[ 0 ], T[ -1 ], window ), [ np.inf ] ) )\n",
    "binx = np.searchsorted( T, bins, side = 'left' )\n",
    "hurst = np.empty( len( binx ) - L, dtype = np.float )\n",
    "waiting = hurst.copy( )\n",
    "for i,j,k in zip( binx[:-L], binx[L:], range( hurst.shape[ 0 ] ) ) :\n",
    "    S, Y = T[ i : j ], X[ i : j ]\n",
    "    delta = ( 2**5 ) * np.median( np.abs( np.diff( Y ) ) )\n",
    "    Tnk, Xnk, Znk, Vnk, Wnk = xtree_build( S, Y, delta = delta, max_height = 3 )\n",
    "    hurst[ k ] = np.log( 2 ) / np.log( np.average( np.concatenate( Znk[1:] ) ) )\n",
    "    waiting[k] = np.average( np.concatenate( Wnk[1:] ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print np.log( np.average( Wnk[1] ) ) + np.log( np.average( np.concatenate( Znk[1:] ) ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A process $X(t)$ is self-similar with hurst exponent $H$ is \n",
    "$$ X(a t) \\overset{\\mathcal{D}}{=} a^H X(t)$$ for all $t\\in [0,1]$ and $a>0$, where equality is understood as equality of all finite vector distributions.\n",
    "Since $a$ is arbitrary positive real number the self-similarity definition is equivalent to\n",
    "$$ X(\\frac{t}{a}) \\overset{\\mathcal{D}}{=} a^{-H} X(t)$$\n",
    "which reduces to\n",
    "$$ X(s) \\overset{\\mathcal{D}}{=} a^{-H} X(a s)$$\n",
    "\n",
    "For the crossing tree\n",
    "$$ X(s) \\overset{\\mathcal{D}}{=} 2^{-k} X( \\mu^k s) = (\\mu^k)^{- \\frac{\\log 2}{\\log \\mu}} X( \\mu^k s)$$\n",
    "whence the following Hurst exponent estimator could be constructed:\n",
    "$$\\hat{H} = \\frac{\\log 2}{\\log \\mu} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Znk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure( figsize = (16,6) )\n",
    "ax = plt.subplot(121)\n",
    "ax.plot( hurst )\n",
    "\n",
    "\n",
    "ax = plt.subplot(122)\n",
    "ax.plot( np.log( waiting ) )\n",
    "\n",
    "# plt.plot( waiting )\n",
    "# plt."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot( T, X )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# def running_hurst( T, X, window = .05 ) :\n",
    "index = np.nonzero( ( T > window ) & ( T < 1 - window ) )[ 0 ]\n",
    "for i in index :\n",
    "## Select the \n",
    "    mask = ( np.abs( T - T[ i ] ) <= window )\n",
    "    delta = ( 2**3 ) * np.median( np.abs( np.diff( X[ mask ] ) ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Tnk, Xnk, Znk, Vnk, Wnk = xtree_build( T[ mask ], X[ mask ], delta = delta, max_height = 6 - 3 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.log( 2 ) / np.log( np.average( np.concatenate( Znk[1:] ) ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Cjn = np.reshape( np.sum( Djnk, axis = ( 2, ), dtype = np.float ), Djnk.shape[ :-1 ] + ( 1, ) )\n",
    "Zk = 2 * np.reshape( 1 + np.arange( Djnk.shape[ -1 ] ), ( 1, 1, Djnk.shape[ -1 ] ) )\n",
    "Pjnk = np.where( Cjn > 0, Djnk / Cjn, 0 )\n",
    "Mjn = np.reshape( np.sum( Pjnk * Zk, axis = ( 2, ) ), Cjn.shape )\n",
    "Hjn = np.log( 2 ) / np.log( Mjn )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "delta = np.subtract( *np.percentile( np.diff( X ), [ 75, 25 ] ) )\n",
    "# delta = np.std( np.diff( X ) )\n",
    "# delta = np.median( np.abs( np.diff( X ) ) )\n",
    "Tnk, Xnk, Znk, Vnk, Wnk = xtree_build( T, X, delta = delta )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

{'fraction_non_alphanumeric': 0.23385539583780304, 'fraction_numerical': 0.014231566902667525, 'mean_word_length': 2.3583093179634966, 'pattern_counts': {'":': 242, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '532648', 'n_tokens_mistral': 5996, 'n_tokens_neox': 5653, 'n_words': 1312}