<jupyter_start><jupyter_text>the Monte Carlo experiment<jupyter_code>import numpy as np

%matplotlib inline
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>A handy routines to store and recover python objects, in particular, the experiment resutls dictionaires.<jupyter_code>import time, gzip
import os, pickle

def save(obj, path, prefix=None):
    prefix_ = "" if prefix is None else "%s_"%(prefix,)
    filename_ = os.path.join(path, "%s%s.gz"%(prefix_, time.strftime("%Y%m%d-%H%M%S"),))
    with gzip.open(filename_, "wb+", 9) as fout_:
        pickle.dump(obj, fout_)
    return filename_

def load(filename):
    with gzip.open(filename, "rb") as f:
        return pickle.load(f)<jupyter_output><empty_output><jupyter_text>The path analyzer<jupyter_code>from crossing_tree import structural_statistics<jupyter_output><empty_output><jupyter_text>Collect a list of results returned by path_analyze into aligned data tensors.<jupyter_code>from crossing_tree import collect_structural_statistics<jupyter_output><empty_output><jupyter_text>A function implementing various delta choices.<jupyter_code>import warnings

def get_delta_method(delta=1.0):
    if isinstance(delta, str):
        if delta == "std":
            # the standard deviation of increments
            delta_ = lambda X: np.diff(X).std()
        elif delta == "med":
            # Use the median absolute difference [<NAME>; 2009] p. 11 (arxiv:0911.5204v2)
            delta_ = lambda X: np.median(np.abs(np.diff(X)))
        elif delta == "mean":
            # Use the mean absolute difference
            delta_ = lambda X: np.mean(np.abs(np.diff(X)))
        elif delta == "iqr":
            # Interquartile range
            delta_ = lambda X: np.subtract(*np.percentile(np.diff(X), [75, 25]))
        elif delta == "rng":
            # Use the range estimate as suggested by Geoffrey on 2015-05-28
            warnings.warn("""Use of `range`-based grid resolution """
                          """is discouraged since it may cause misaligned """
                          """crossing trees.""", RuntimeWarning)
            delta_ = lambda X: (X.max() - X.min()) / (2**12)
        else:
            raise ValueError("""Invalid `delta` setting. Accepted values """
                             """are: [`iqr`, `std`, `med`, `rng`, `mean`].""")
    elif isinstance(delta, float) and delta > 0:
        delta_ = lambda X: delta
    else:
        raise TypeError("""`delta` must be either a float, or a method """
                        """identifier.""")
    return delta_<jupyter_output><empty_output><jupyter_text>An MC experiment kernel.<jupyter_code>from sklearn.base import clone

def experiment(experiment_id, n_replications, methods, generator):
    generator = clone(generator)
    generator.start()

    deltas = [get_delta_method(method_) for method_ in methods]

    results = {method_: list() for method_ in methods}
    for j in xrange(n_replications):
        T, X = generator.draw()

        # Apply all methods to the same sample path.
        for delta, method in zip(deltas, methods):
            result_ = structural_statistics(X, T, scale=delta(X), origin=X[0])
            results[method].append(result_)

    generator.finish()

    return experiment_id, results<jupyter_output><empty_output><jupyter_text>Experiments<jupyter_code>from joblib import Parallel, delayed<jupyter_output><empty_output><jupyter_text>A couple of random seeds from [here](https://www.random.org/bytes/).<jupyter_code># Extra random seeds should be prepended to the array.
master_seeds = [0xD5F60A17, 0x26F9935C, 0x0E4C1E75, 0xDA7C4291, 0x7ABE722E,
                0x126F3E10, 0x045300B1, 0xB0A9AD11, 0xEED05353, 0x824736C7,
                0x7AA17C9C, 0xB695D6B1, 0x7E214411, 0x538CDEEF, 0xFD55FF46,
                0xE14E1801, 0x872F687C, 0xA58440D9, 0xB8A273FD, 0x0BD1DD28,
                0xAB6A6AE6, 0x7180E905, 0x870E7BAB, 0x846D0C7A, 0xAEF0422D,
                0x16C53C83, 0xE32EA61D, 0xE0AD0A26, 0xCC90CA9A, 0x7D4020D2,]<jupyter_output><empty_output><jupyter_text>the Monte Carlo experiemnt is run in parallel batches, with eachinitialized to a randomly picked seed.<jupyter_code>MAX_RAND_SEED = np.iinfo(np.int32).max<jupyter_output><empty_output><jupyter_text>The folder to store the results in<jupyter_code>OUTPUT_PATH = "../results/"<jupyter_output><empty_output><jupyter_text>fBM experiment<jupyter_code>from crossing_tree.processes import FractionalBrownianMotion

seed = master_seeds.pop()
print("Using seed %X"%(seed,))
random_state = np.random.RandomState(seed)

skip = False<jupyter_output>Using seed 7D4020D2<jupyter_text>Setup<jupyter_code>n_samples, methods = 1 << 23, ["med", "std", "iqr", "mean",]
hurst_exponents = [0.500, 0.550, 0.600, 0.650, 0.700, 0.750, 0.800, 0.850, 0.900,
                   0.910, 0.915, 0.920, 0.925, 0.930, 0.935, 0.940, 0.945, 0.950,
                   0.990,]
n_per_batch, n_batches = 125, 8<jupyter_output><empty_output><jupyter_text>Run the experiment for the Fractional Brownian Motion.<jupyter_code>if not skip:
    par_ = Parallel(n_jobs=-1, verbose=0)
    for hurst_ in hurst_exponents:
        name_ = "FBM-%d-%0.3f-%dx%d"%(n_samples, hurst_, n_per_batch, n_batches)
        print(name_,)

        # Schedule the experiments
        seeds = random_state.randint(MAX_RAND_SEED, size=(n_batches,))
        schedule_ = (delayed(experiment)(seed_, n_per_batch, methods,
                                         FractionalBrownianMotion(N=n_samples,
                                                                  hurst=hurst_,
                                                                  random_state=seed_))
                                    for seed_ in seeds)

        # Run the experiment and collect the results
        tick_ = time.time()
        experiment_ids = list()
        results_ = {method: list() for method in methods}
        for id_, dict_ in par_(schedule_):
            experiment_ids.append(id_)
            for method in methods:
                results_[method].extend(dict_[method])
        results = {key_: collect_structural_statistics(list_)
                   for key_, list_ in results_.iteritems()}
        tock_ = time.time()

        # Save the results and log
        filename_ = save((tick_, tock_, experiment_ids, results), OUTPUT_PATH, name_)
        print("%0.3fsec."%(tock_ - tick_,), filename_)<jupyter_output>FBM-8388608-0.500-125x8 1056.620sec. ../results/FBM-8388608-0.500-125x8_20161108-162012.gz
FBM-8388608-0.550-125x8 1133.918sec. ../results/FBM-8388608-0.550-125x8_20161108-164004.gz
FBM-8388608-0.600-125x8 1186.027sec. ../results/FBM-8388608-0.600-125x8_20161108-170050.gz
FBM-8388608-0.650-125x8 1231.505sec. ../results/FBM-8388608-0.650-125x8_20161108-172209.gz
FBM-8388608-0.700-125x8 1288.416sec. ../results/FBM-8388608-0.700-125x8_20161108-174427.gz
FBM-8388608-0.750-125x8 1351.370sec. ../results/FBM-8388608-0.750-125x8_20161108-180744.gz
FBM-8388608-0.800-125x8 1420.173sec. ../results/FBM-8388608-0.800-125x8_20161108-183207.gz
FBM-8388608-0.850-125x8 1500.556sec. ../results/FBM-8388608-0.850-125x8_20161108-185753.gz
FBM-8388608-0.900-125x8 1604.555sec. ../results/FBM-8388608-0.900-125x8_20161108-192519.gz
FBM-8388608-0.910-125x8 1635.019sec. ../results/FBM-8388608-0.910-125x8_20161108-195316.gz
FBM-8388608-0.915-125x8 1634.690sec. ../results/FBM-8388608-0.915-125x8_20161108-202113.gz[...]<jupyter_text>Hermite process experiment<jupyter_code>from crossing_tree.processes import HermiteProcess

seed = master_seeds.pop()
print("Using seed %X"%(seed,))
random_state = np.random.RandomState(seed)

skip = False<jupyter_output>Using seed CC90CA9A<jupyter_text>Setup: use no downsampling.<jupyter_code>n_samples, n_downsample = 1 << 23, 1
degrees, methods = [2, 3, 4], ["med", "std", "iqr", "mean",]
hurst_exponents = [       0.550, 0.600, 0.650, 0.700, 0.750, 0.800, 0.850, 0.900,
                   0.910, 0.915, 0.920, 0.925, 0.930, 0.935, 0.940, 0.945, 0.950,
                   0.990,]
n_per_batch, n_batches = 125, 8<jupyter_output><empty_output><jupyter_text>Run the experiment for the Hermite process.<jupyter_code>if not skip:
    par_ = Parallel(n_jobs=-1, verbose=0)
    for degree_ in degrees:
        for hurst_ in hurst_exponents:
            name_ = "HRP%d_%d-%d-%0.3f-%dx%d"%(degree_, n_downsample, n_samples, hurst_, n_per_batch, n_batches)
            print(name_,)

            # Schedule the experiments
            seeds = random_state.randint(MAX_RAND_SEED, size=(n_batches,))
            schedule_ = (delayed(experiment)(seed_, n_per_batch, methods,
                                             HermiteProcess(N=n_samples,
                                                            degree=degree_,
                                                            n_downsample=n_downsample,
                                                            hurst=hurst_,
                                                            random_state=seed_))
                                        for seed_ in seeds)

            # Run the experiment and collect the results
            tick_ = time.time()
            experiment_ids = list()
            results_ = {method: list() for method in methods}
            for id_, dict_ in par_(schedule_):
                experiment_ids.append(id_)
                for method in methods:
                    results_[method].extend(dict_[method])
            results = {key_: collect_structural_statistics(list_)
                       for key_, list_ in results_.iteritems()}
            tock_ = time.time()

            # Save the results and log
            filename_ = save((tick_, tock_, experiment_ids, results), OUTPUT_PATH, name_)
            print("%0.3fsec."%(tock_ - tick_,), filename_)<jupyter_output>HRP2_1-8388608-0.550-125x8 1313.578sec. ../results/HRP2_1-8388608-0.550-125x8_20161109-004331.gz
HRP2_1-8388608-0.600-125x8 1335.730sec. ../results/HRP2_1-8388608-0.600-125x8_20161109-010637.gz
HRP2_1-8388608-0.650-125x8 1373.954sec. ../results/HRP2_1-8388608-0.650-125x8_20161109-013022.gz
HRP2_1-8388608-0.700-125x8 1412.096sec. ../results/HRP2_1-8388608-0.700-125x8_20161109-015439.gz
HRP2_1-8388608-0.750-125x8 1457.555sec. ../results/HRP2_1-8388608-0.750-125x8_20161109-021942.gz
HRP2_1-8388608-0.800-125x8 1519.344sec. ../results/HRP2_1-8388608-0.800-125x8_20161109-024547.gz
HRP2_1-8388608-0.850-125x8 1585.849sec. ../results/HRP2_1-8388608-0.850-125x8_20161109-031254.gz
HRP2_1-8388608-0.900-125x8 1681.800sec. ../results/HRP2_1-8388608-0.900-125x8_20161109-034137.gz
HRP2_1-8388608-0.910-125x8 1697.750sec. ../results/HRP2_1-8388608-0.910-125x8_20161109-041035.gz
HRP2_1-8388608-0.915-125x8 1705.061sec. ../results/HRP2_1-8388608-0.915-125x8_20161109-043942.gz
HRP2_1-8388608-0.920-125x8 171[...]<jupyter_text>Weierstrass experiment -- $\lambda_0 = 1.2$<jupyter_code>from crossing_tree.processes import WeierstrassFunction

seed = master_seeds.pop()
print("Using seed %X"%(seed,))
random_state = np.random.RandomState(seed)

skip = False<jupyter_output>Using seed E0AD0A26<jupyter_text>Setup<jupyter_code>n_samples, lambda_0 = 1 << 23, 1.2
methods = ["med", "std", "iqr", "mean",]
holder_exponents = [0.500, 0.550, 0.600, 0.650, 0.700, 0.750, 0.800, 0.850, 0.900,
                    0.910, 0.915, 0.920, 0.925, 0.930, 0.935, 0.940, 0.945, 0.950,
                    0.990,]
n_per_batch, n_batches = 125, 8<jupyter_output><empty_output><jupyter_text>Run the experimnet for the random Weierstrass function $[0, 1]\mapsto \mathbb{R}$:$$ W_H(t) = \sum_{k\geq 0} \lambda_0^{-k H} \bigl(\cos(2 \pi \lambda_0^k t + \phi_k) - \cos \phi_k\bigr)\,, $$with $(\phi_k)_{k\geq0} \sim \mathbb{U}[0, 2\pi]$, and $\lambda_0 > 1$ -- the fundamental harmonic.<jupyter_code>if not skip:
    par_ = Parallel(n_jobs=-1, verbose=0)
    for holder_ in holder_exponents:
        name_ = "WEI_%g-%d-%0.3f-%dx%d"%(lambda_0, n_samples, holder_, n_per_batch, n_batches)
        print(name_,)

        # Schedule the experiments
        seeds = random_state.randint(MAX_RAND_SEED, size=(n_batches,))
        schedule_ = (delayed(experiment)(seed_, n_per_batch, methods,
                                         WeierstrassFunction(N=n_samples,
                                                             lambda_0=lambda_0,
                                                             holder=holder_,
                                                             random_state=seed_,
                                                             one_sided=False))
                                    for seed_ in seeds)

        # Run the experiment and collect the results
        tick_ = time.time()
        experiment_ids = list()
        results_ = {method: list() for method in methods}
        for id_, dict_ in par_(schedule_):
            experiment_ids.append(id_)
            for method in methods:
                results_[method].extend(dict_[method])
        results = {key_: collect_structural_statistics(list_)
                   for key_, list_ in results_.iteritems()}
        tock_ = time.time()

        # Save the results and log
        filename_ = save((tick_, tock_, experiment_ids, results), OUTPUT_PATH, name_)
        print("%0.3fsec."%(tock_ - tick_,), filename_)<jupyter_output>WEI_1.2-8388608-0.500-125x8 13288.305sec. ../results/WEI_1.2-8388608-0.500-125x8_20161110-054850.gz
WEI_1.2-8388608-0.550-125x8 13345.497sec. ../results/WEI_1.2-8388608-0.550-125x8_20161110-093202.gz
WEI_1.2-8388608-0.600-125x8 13390.412sec. ../results/WEI_1.2-8388608-0.600-125x8_20161110-131601.gz
WEI_1.2-8388608-0.650-125x8 13429.911sec. ../results/WEI_1.2-8388608-0.650-125x8_20161110-170034.gz
WEI_1.2-8388608-0.700-125x8 13551.845sec. ../results/WEI_1.2-8388608-0.700-125x8_20161110-204707.gz
WEI_1.2-8388608-0.750-125x8 13604.540sec. ../results/WEI_1.2-8388608-0.750-125x8_20161111-003431.gz
WEI_1.2-8388608-0.800-125x8 13661.736sec. ../results/WEI_1.2-8388608-0.800-125x8_20161111-042252.gz
WEI_1.2-8388608-0.850-125x8 13766.987sec. ../results/WEI_1.2-8388608-0.850-125x8_20161111-081257.gz
WEI_1.2-8388608-0.900-125x8 13917.185sec. ../results/WEI_1.2-8388608-0.900-125x8_20161111-120532.gz
WEI_1.2-8388608-0.910-125x8 13925.786sec. ../results/WEI_1.2-8388608-0.910-125x8_20161111-155816.gz
[...]<jupyter_text>Additional experiments  Hermite process experiment: with downsampling<jupyter_code>from crossing_tree.processes import HermiteProcess

seed = master_seeds.pop()
print("Using seed %X"%(seed,))
random_state = np.random.RandomState(seed)

skip = False<jupyter_output>Using seed E32EA61D<jupyter_text>Setup<jupyter_code>n_samples, n_downsample = 1 << 19, 1 << 4
degrees, methods = [2, 3, 4], ["med", "std", "iqr", "mean",]
hurst_exponents = [       0.550, 0.600, 0.650, 0.700, 0.750, 0.800, 0.850, 0.900,
                   0.910, 0.915, 0.920, 0.925, 0.930, 0.935, 0.940, 0.945, 0.950,
                   0.990,]
n_per_batch, n_batches = 125, 8<jupyter_output><empty_output><jupyter_text>Run the experiment for the Hermite process.<jupyter_code>if not skip:
    par_ = Parallel(n_jobs=-1, verbose=0)
    for degree_ in degrees:
        for hurst_ in hurst_exponents:
            name_ = "HRP%d_%d-%d-%0.3f-%dx%d"%(degree_, n_downsample, n_samples, hurst_, n_per_batch, n_batches)
            print(name_,)

            # Schedule the experiments
            seeds = random_state.randint(MAX_RAND_SEED, size=(n_batches,))
            schedule_ = (delayed(experiment)(seed_, n_per_batch, methods,
                                             HermiteProcess(N=n_samples,
                                                            degree=degree_,
                                                            n_downsample=n_downsample,
                                                            hurst=hurst_,
                                                            random_state=seed_))
                                        for seed_ in seeds)

            # Run the experiment and collect the results
            tick_ = time.time()
            experiment_ids = list()
            results_ = {method: list() for method in methods}
            for id_, dict_ in par_(schedule_):
                experiment_ids.append(id_)
                for method in methods:
                    results_[method].extend(dict_[method])
            results = {key_: collect_structural_statistics(list_)
                       for key_, list_ in results_.iteritems()}
            tock_ = time.time()

            # Save the results and log
            filename_ = save((tick_, tock_, experiment_ids, results), OUTPUT_PATH, name_)
            print("%0.3fsec."%(tock_ - tick_,), filename_)<jupyter_output>HRP2_16-524288-0.550-125x8 408.249sec. ../results/HRP2_16-524288-0.550-125x8_20161113-024553.gz
HRP2_16-524288-0.600-125x8 417.433sec. ../results/HRP2_16-524288-0.600-125x8_20161113-025327.gz
HRP2_16-524288-0.650-125x8 419.657sec. ../results/HRP2_16-524288-0.650-125x8_20161113-030100.gz
HRP2_16-524288-0.700-125x8 415.866sec. ../results/HRP2_16-524288-0.700-125x8_20161113-030827.gz
HRP2_16-524288-0.750-125x8 418.563sec. ../results/HRP2_16-524288-0.750-125x8_20161113-031559.gz
HRP2_16-524288-0.800-125x8 426.054sec. ../results/HRP2_16-524288-0.800-125x8_20161113-032334.gz
HRP2_16-524288-0.850-125x8 430.168sec. ../results/HRP2_16-524288-0.850-125x8_20161113-033113.gz
HRP2_16-524288-0.900-125x8 431.081sec. ../results/HRP2_16-524288-0.900-125x8_20161113-033851.gz
HRP2_16-524288-0.910-125x8 434.201sec. ../results/HRP2_16-524288-0.910-125x8_20161113-034633.gz
HRP2_16-524288-0.915-125x8 437.488sec. ../results/HRP2_16-524288-0.915-125x8_20161113-035418.gz
HRP2_16-524288-0.920-125x8 443.848sec. .[...]<jupyter_text>Weierstrass experiment -- $\lambda_0 = 3$<jupyter_code>from crossing_tree.processes import WeierstrassFunction

seed = master_seeds.pop()
print("Using seed %X"%(seed,))
random_state = np.random.RandomState(seed)

skip = True<jupyter_output>Using seed 16C53C83<jupyter_text>Setup<jupyter_code>n_samples, lambda_0 = 1 << 23, 3.0
methods = ["med", "std", "iqr", "mean",]
holder_exponents = [0.500, 0.550, 0.600, 0.650, 0.700, 0.750, 0.800, 0.850, 0.900,
                    0.910, 0.915, 0.920, 0.925, 0.930, 0.935, 0.940, 0.945, 0.950,
                    0.990,]
n_per_batch, n_batches = 125, 8<jupyter_output><empty_output><jupyter_text>Run the experimnet for the random Weierstrass function $[0, 1]\mapsto \mathbb{R}$:$$ W_H(t) = \sum_{k\geq 0} \lambda_0^{-k H} \bigl(\cos(2 \pi \lambda_0^k t + \phi_k) - \cos \phi_k\bigr)\,, $$with $(\phi_k)_{k\geq0} \sim \mathbb{U}[0, 2\pi]$, and $\lambda_0 > 1$ -- the fundamental harmonic.<jupyter_code>if not skip:
    par_ = Parallel(n_jobs=-1, verbose=0)
    for holder_ in holder_exponents:
        name_ = "WEI_%g-%d-%0.3f-%dx%d"%(lambda_0, n_samples, holder_, n_per_batch, n_batches)
        print(name_,)

        # Schedule the experiments
        seeds = random_state.randint(MAX_RAND_SEED, size=(n_batches,))
        schedule_ = (delayed(experiment)(seed_, n_per_batch, methods,
                                         WeierstrassFunction(N=n_samples,
                                                             lambda_0=lambda_0,
                                                             holder=holder_,
                                                             random_state=seed_,
                                                             one_sided=False))
                                    for seed_ in seeds)

        # Run the experiment and collect the results
        tick_ = time.time()
        experiment_ids = list()
        results_ = {method: list() for method in methods}
        for id_, dict_ in par_(schedule_):
            experiment_ids.append(id_)
            for method in methods:
                results_[method].extend(dict_[method])
        results = {key_: collect_structural_statistics(list_)
                   for key_, list_ in results_.iteritems()}
        tock_ = time.time()

        # Save the results and log
        filename_ = save((tick_, tock_, experiment_ids, results), OUTPUT_PATH, name_)
        print("%0.3fsec."%(tock_ - tick_,), filename_)<jupyter_output><empty_output><jupyter_text>Weierstrass experiment -- $\lambda_0 = 1.7$<jupyter_code>from crossing_tree.processes import WeierstrassFunction

seed = master_seeds.pop()
print("Using seed %X"%(seed,))
random_state = np.random.RandomState(seed)

skip = True<jupyter_output>Using seed AEF0422D<jupyter_text>Setup<jupyter_code>n_samples, lambda_0 = 1 << 23, 1.7
methods = ["med", "std", "iqr", "mean",]
holder_exponents = [0.500, 0.550, 0.600, 0.650, 0.700, 0.750, 0.800, 0.850, 0.900,
                    0.910, 0.915, 0.920, 0.925, 0.930, 0.935, 0.940, 0.945, 0.950,
                    0.990,]
n_per_batch, n_batches = 125, 8<jupyter_output><empty_output><jupyter_text>Run the experimnet for the random Weierstrass function $[0, 1]\mapsto \mathbb{R}$:$$ W_H(t) = \sum_{k\geq 0} \lambda_0^{-k H} \bigl(\cos(2 \pi \lambda_0^k t + \phi_k) - \cos \phi_k\bigr)\,, $$with $(\phi_k)_{k\geq0} \sim \mathbb{U}[0, 2\pi]$, and $\lambda_0 > 1$ -- the fundamental harmonic.<jupyter_code>if not skip:
    par_ = Parallel(n_jobs=-1, verbose=0)
    for holder_ in holder_exponents:
        name_ = "WEI_%g-%d-%0.3f-%dx%d"%(lambda_0, n_samples, holder_, n_per_batch, n_batches)
        print(name_,)

        # Schedule the experiments
        seeds = random_state.randint(MAX_RAND_SEED, size=(n_batches,))
        schedule_ = (delayed(experiment)(seed_, n_per_batch, methods,
                                         WeierstrassFunction(N=n_samples,
                                                             lambda_0=lambda_0,
                                                             holder=holder_,
                                                             random_state=seed_,
                                                             one_sided=False))
                                    for seed_ in seeds)

        # Run the experiment and collect the results
        tick_ = time.time()
        experiment_ids = list()
        results_ = {method: list() for method in methods}
        for id_, dict_ in par_(schedule_):
            experiment_ids.append(id_)
            for method in methods:
                results_[method].extend(dict_[method])
        results = {key_: collect_structural_statistics(list_)
                   for key_, list_ in results_.iteritems()}
        tock_ = time.time()

        # Save the results and log
        filename_ = save((tick_, tock_, experiment_ids, results), OUTPUT_PATH, name_)
        print("%0.3fsec."%(tock_ - tick_,), filename_)<jupyter_output><empty_output><jupyter_text>fBM experiment: super long<jupyter_code>from crossing_tree.processes import FractionalBrownianMotion

seed = master_seeds.pop()
print("Using seed %X"%(seed,))
random_state = np.random.RandomState(seed)

skip = True<jupyter_output>Using seed 846D0C7A<jupyter_text>Setup<jupyter_code>n_samples, methods = 1 << 25, ["med", "std", "iqr", "mean",]
hurst_exponents = [0.500, 0.550, 0.600, 0.650, 0.700, 0.750, 0.800, 0.850, 0.900,
                   0.910, 0.915, 0.920, 0.925, 0.930, 0.935, 0.940, 0.945, 0.950,
                   0.990,]
n_per_batch, n_batches, n_threads = 334, 3, 4<jupyter_output><empty_output><jupyter_text>Run the experiment for the Fractional Brownian Motion.<jupyter_code>if not skip:
    par_ = Parallel(n_jobs=-1, verbose=0)
    for hurst_ in hurst_exponents:
        name_ = "FBM-%d-%0.3f-%dx%d"%(n_samples, hurst_, n_per_batch, n_batches)
        print(name_,)

        # Schedule the experiments
        seeds = random_state.randint(MAX_RAND_SEED, size=(n_batches,))
        schedule_ = (delayed(experiment)(seed_, n_per_batch, methods,
                                         FractionalBrownianMotion(N=n_samples,
                                                                  hurst=hurst_,
                                                                  random_state=seed_,
                                                                  n_threads=n_threads))
                                    for seed_ in seeds)

        # Run the experiment and collect the results
        tick_ = time.time()
        experiment_ids = list()
        results_ = {method: list() for method in methods}
        for id_, dict_ in par_(schedule_):
            experiment_ids.append(id_)
            for method in methods:
                results_[method].extend(dict_[method])
        results = {key_: collect_structural_statistics(list_)
                   for key_, list_ in results_.iteritems()}
        tock_ = time.time()

        # Save the results and log
        filename_ = save((tick_, tock_, experiment_ids, results), OUTPUT_PATH, name_)
        print("%0.3fsec."%(tock_ - tick_,), filename_)<jupyter_output>FBM-33554432-0.500-334x3 7922.633sec. ../results/FBM-33554432-0.500-334x3_20161108-184220.gz
FBM-33554432-0.550-334x3 8994.173sec. ../results/FBM-33554432-0.550-334x3_20161108-211311.gz
FBM-33554432-0.600-334x3 9671.584sec. ../results/FBM-33554432-0.600-334x3_20161108-235516.gz
FBM-33554432-0.650-334x3 9985.724sec. ../results/FBM-33554432-0.650-334x3_20161109-024230.gz
FBM-33554432-0.700-334x3 10591.847sec. ../results/FBM-33554432-0.700-334x3_20161109-053945.gz
FBM-33554432-0.750-334x3 11290.630sec. ../results/FBM-33554432-0.750-334x3_20161109-084839.gz
FBM-33554432-0.800-334x3 11924.460sec. ../results/FBM-33554432-0.800-334x3_20161109-120805.gz
FBM-33554432-0.850-334x3 12146.259sec. ../results/FBM-33554432-0.850-334x3_20161109-153112.gz
FBM-33554432-0.900-334x3 13212.555sec. ../results/FBM-33554432-0.900-334x3_20161109-191203.gz
FBM-33554432-0.910-334x3 13205.301sec. ../results/FBM-33554432-0.910-334x3_20161109-225250.gz
FBM-33554432-0.915-334x3 13467.640sec. ../results/FBM-33554432-0[...]
{'fraction_non_alphanumeric': 0.12574084834398605, 'fraction_numerical': 0.14545806701530117, 'mean_word_length': 2.4234186447420765, 'pattern_counts': {'":': 5, '<': 130, '<?xml version=': 0, '>': 118, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 73, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '31', 'dir': 'jupyter-structured-clean-dedup', 'id': '27737', 'license': 'MIT', 'n_tokens_mistral': 11457, 'n_tokens_neox': 9128, 'n_words': 1618, 'path': 'experiments/full_experiment.ipynb', 'repo_name': 'ivannz/crossing_paper2017', 'size': '25821'}