<jupyter_start><jupyter_text>Optimizing pymultifracs  Toy dataLet's first get some toy data to test out timings<jupyter_code>import mne
raw = mne.io.read_raw_fif('/media/merlin/Media/ScaledTime/MEG/eb180237/eb180237_ScaledTime_RS01_filtered_ica_sss_raw.fif')
data = raw[:]
del raw
data = data[0]<jupyter_output><empty_output><jupyter_text>Wavelet analysis<jupyter_code>from importlib import reload
import pymultifracs.wavelet
reload(pymultifracs.wavelet)
from pymultifracs.wavelet import wavelet_analysis
data.shape
p_exp = 2
wt_name = 'db3'
j1 = 1
j2 = 10
gamint = 1.0
from megfractal.utils import scale2freq
%%prun -T "profile.txt"

wavelet_analysis(data[0], p_exp, wt_name, j1, j2, gamint)
%%timeit

wavelet_analysis(data[0], p_exp, wt_name, j1, j2, gamint)
%%timeit

wavelet_analysis(data[0], p_exp, wt_name, j1, j2, gamint)
import numpy as np
np.show_config()
%%prun
wavelet_analysis(data[0], p_exp, wt_name, j1, j2, gamint)<jupyter_output><jupyter_text>Full multifractal<jupyter_code>from pymultifracs.mf_analysis import mf_analysis_full
import numpy as np
# l = np.logspace(np.log10(0.01), np.log10(5), 10)
# q = np.array([*-l, *l, 0, 1, 2, -1, -2])
# q.sort()
q = np.array([-2, -1, 0, 1, 2])
n_cumul = 2
%%timeit

mf_analysis_full(data[0], j1, j2, gamint=1.0, wt_name=wt_name, p_exp=p_exp, q=q, n_cumul=n_cumul)
%%timeit

mf_analysis_full(data[0], j1, j2, gamint=1.0, wt_name=wt_name, p_exp=p_exp, q=q, n_cumul=n_cumul)
%%timeit

mf_analysis_full(data[0], j1, j2, gamint=1.0, wt_name=wt_name, p_exp=p_exp, q=q, n_cumul=n_cumul)<jupyter_output>275 ms ± 13.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Multifractal<jupyter_code>import pymultifracs.mfspectrum as mfs
reload(mfs)
import pymultifracs.mf_analysis
reload(pymultifracs.mf_analysis)
from pymultifracs.mf_analysis import minimal_mf_analysis
from pymultifracs.mf_analysis import mf_analysis
wt_transform = wavelet_analysis(data[0], p_exp, wt_name, j1, j2, gamint=1.0)
%%timeit

mf_analysis(wt_transform.wt_coefs,
            wt_transform.wt_leaders,
            wt_transform.j2_eff,
            p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)
%%timeit

mf_analysis(wt_transform.wt_coefs,
            wt_transform.wt_leaders,
            wt_transform.j2_eff,
            p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)
%%timeit

mf_analysis(wt_transform.wt_coefs,
            wt_transform.wt_leaders,
            wt_transform.j2_eff,
            p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)
%%timeit

mf_analysis(wt_transform.wt_coefs,
            wt_transform.wt_leaders,
            wt_transform.j2_eff,
            p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)
%%timeit

mf_analysis(wt_transform.wt_coefs,
            wt_transform.wt_leaders,
            wt_transform.j2_eff,
            p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)
%%timeit

mf_analysis(wt_transform.wt_coefs,
            wt_transform.wt_leaders,
            wt_transform.j2_eff,
            p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)
%%timeit

minimal_mf_analysis(wt_transform.wt_coefs,
                    wt_transform.wt_leaders,
                    wt_transform.j2_eff,
                    p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)
%%prun -T "profile_mf.txt"

mf_analysis(wt_transform.wt_coefs,
            wt_transform.wt_leaders,
            wt_transform.j2_eff,
            p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)
%%prun -T "profile_mf.txt"

minimal_mf_analysis(wt_transform.wt_coefs,
                    wt_transform.wt_leaders,
                    wt_transform.j2_eff,
                    p_exp=p_exp, j1=j1, weighted=True, n_cumul=n_cumul, q=q)<jupyter_output>*** Profile printout saved to text file '"profile_mf.txt"'.<jupyter_text>Timing wavelet analysis<jupyter_code>wt_transform = wavelet_analysis(data[0], p_exp, wt_name, j1, j2, gamint=0.0)
%timeit np.power(2, 10)
%timeit 2**10
%timeit np.power(2, 10)
%timeit 2**10<jupyter_output>5.13 ns ± 0.0965 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)<jupyter_text>Timing array ops<jupyter_code>detail = np.random.randn(37200000)<jupyter_output><empty_output><jupyter_text>Squaring<jupyter_code>%timeit np.power(detail, 2)
%timeit detail * detail
%timeit np.square(detail)
%timeit np.power(detail, 2)
%timeit detail * detail
%timeit np.square(detail)<jupyter_output>92.2 ms ± 1.18 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)<jupyter_text>Rooting<jupyter_code>%timeit np.power(np.abs(detail), 1/2)
%timeit np.sqrt(np.abs(detail))
%timeit np.power(np.abs(detail), 1/2)
%timeit np.sqrt(np.abs(detail))<jupyter_output>215 ms ± 31.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Raising to arbitrary power<jupyter_code>q = 3
%timeit detail ** q
%timeit np.power(detail, q)
%%timeit

array_out = np.ones(detail.shape)

for _ in range(q):
    array_out *= detail

array_out
q = 10
%timeit detail ** q
%timeit np.power(detail, q)
%%timeit

array_out = np.ones(detail.shape)

for _ in range(q):
    array_out *= detail

array_out
detail ** q
np.power(detail, q)
array_out = np.ones(detail.shape)

for _ in range(q):
    array_out *= detail

array_out<jupyter_output><empty_output><jupyter_text>Computing mean<jupyter_code>detail = np.random.randn(37200000)
%timeit detail.mean()
%timeit np.mean(detail)<jupyter_output>1.68 ms ± 28.3 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)<jupyter_text>Testing smart power<jupyter_code>import pymultifracs.utils
reload(pymultifracs.utils)
from pymultifracs.utils import fast_power
exponent = 0
%timeit fast_power(detail, exponent)
%timeit detail ** exponent
exponent = 1
%timeit fast_power(detail, exponent)
%timeit detail ** exponent
exponent = 2
%timeit fast_power(detail, exponent)
%timeit detail ** exponent
exponent = -1
%timeit fast_power(detail, exponent)
%timeit detail ** exponent
exponent = -2
%timeit fast_power(detail, exponent)
%timeit detail ** exponent<jupyter_output>718 ms ± 7.4 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Summing<jupyter_code>%timeit detail.sum()
%timeit np.sum(detail)
%timeit detail2.sum()
%timeit np.sum(detail2)<jupyter_output>383 µs ± 23.8 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)<jupyter_text>Filtering<jupyter_code>approx = np.random.randn(20)
from scipy.signal import convolve
import pywt
wavelet = pywt.Wavelet(wt_name)
high_filter = -1*np.array(wavelet.dec_hi)
low_filter = np.array(wavelet.dec_lo)
nj_temp = len(approx)
approx[3] = np.inf
high = convolve(approx, high_filter, mode='full', method='direct')
low = convolve(approx, low_filter, mode='full', method='direct')
high_v = convolve(approx, high_filter, mode='valid', method='direct')
low_v = convolve(approx, low_filter, mode='valid', method='direct')
%timeit high = convolve(approx, high_filter, mode='full', method='direct')
%timeit high = convolve(approx, high_filter, mode='full')
high
high_v
low
# index of first good value
fp = len(high_filter) - 2
# index of last good value
lp = nj_temp - 1
# replace border with Inf
high[0:fp] = np.inf
high[lp+1:] = np.inf
low[0:fp] = np.inf
low[lp+1:] = np.inf
high
# centering and subsampling
detail_idx = np.arange(1, nj_temp + 1, 2)
approx_idx = np.arange(-1, nj_temp - 1, 2) + len(high_filter)
detail = high[detail_idx]
approx = low[approx_idx]
detail
approx
A = np.random.rand(1000, 100000)
B = np.random.rand(100000, 1000)
%%timeit
np.dot(A,B)
%%timeit
np.dot(A,B)
detail2 = np.random.randn(100, 10000)
%timeit detail2 * detail2
%timeit detail2 ** 2
%timeit np.power(detail2, 2)
3 * 2 ** 3
3 * (2 ** 3)
(3 * 2) ** 3<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1283023803295841, 'fraction_numerical': 0.037405179178655504, 'mean_word_length': 4.824067022086824, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '398499', 'license': 'MIT', 'n_tokens_mistral': 3350, 'n_tokens_neox': 3097, 'n_words': 650, 'path': 'notebooks/opti.ipynb', 'repo_name': 'agramfort/pymultifracs', 'size': '7646'}