<commit_before>#- Test loopsine functions with multiprocessing parallelism

from __future__ import division, print_function

import multiprocessing as mp
from loopsine import loopsine_purepy, loopsine_numpy, loopsine_numba
from knltest import timeit

#- Wake up functions in case there is loading overhead
loopsine_purepy(2)
loopsine_numpy(2)
loopsine_numba(2)

n = 10000000
args = [n,] * mp.cpu_count() * 2
ncpu = 1
print("# Testing multiprocessing sum(sin(range({})))".format(n))
print("# Processing {} batches with ncpu processes".format(len(args)))
print("# method ncpu time     rate")
while ncpu <= mp.cpu_count():
    pool = mp.Pool(ncpu)
    t1 = timeit(pool.map, (loopsine_purepy, args))
    t2 = timeit(pool.map, (loopsine_numpy, args))
    t3 = timeit(pool.map, (loopsine_numba, args))
    pool.close()
    
    r1 = len(args) / t1
    r2 = len(args) / t2
    r3 = len(args) / t3
    print('purepy {:3d} {:8.3f} {:8.3f}'.format(ncpu, t1, r1))
    print('numpy  {:3d} {:8.3f} {:8.3f}'.format(ncpu, t2, r2))
    print('numba  {:3d} {:8.3f} {:8.3f}'.format(ncpu, t3, r3))
    
    ncpu *= 2
    
    <commit_msg>adjust max number of batches<commit_after>#- Test loopsine functions with multiprocessing parallelism

from __future__ import division, print_function

import multiprocessing as mp
import numpy as np
from loopsine import loopsine_purepy, loopsine_numpy, loopsine_numba
from knltest import timeit

#- Wake up functions in case there is loading overhead
loopsine_purepy(2)
loopsine_numpy(2)
loopsine_numba(2)

n = 10000000

#- Fill up the CPUs with work to do, but don't go crazy
x = min(6, np.log2(mp.cpu_count() * 2))
nargs = 2**int(x)

args = [n,] * nargs
ncpu = 1
print("# Testing multiprocessing sum(sin(range({})))".format(n))
print("# Processing {} batches with ncpu processes".format(len(args)))
print("# method ncpu time     rate")
while ncpu <= nargs:
    pool = mp.Pool(ncpu)
    t1 = timeit(pool.map, (loopsine_purepy, args))
    t2 = timeit(pool.map, (loopsine_numpy, args))
    t3 = timeit(pool.map, (loopsine_numba, args))
    pool.close()
    
    r1 = len(args) / t1
    r2 = len(args) / t2
    r3 = len(args) / t3
    print('purepy {:3d} {:8.3f} {:8.3f}'.format(ncpu, t1, r1))
    print('numpy  {:3d} {:8.3f} {:8.3f}'.format(ncpu, t2, r2))
    print('numba  {:3d} {:8.3f} {:8.3f}'.format(ncpu, t3, r3))
    
    ncpu *= 2

{'fraction_non_alphanumeric': 0.15061518879932118, 'fraction_numerical': 0.03860840050912177, 'mean_word_length': 3.943396226415094, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '190010', 'n_tokens_mistral': 982, 'n_tokens_neox': 936, 'n_words': 241}