<commit_before>#!/bin/sh

PYOPENCL_CTX=1
PYTHONPATH=../bumps:../periodictable:../sasview/build/lib.macosx-10.5-x86_64-2.7
export PYOPENCL_CTX PYTHONPATH

set -x

for dim in 1d100 2d32; do
    ./compare_many.py all 100 $dim 0 > ${dim}_poly_0.csv
    ./compare_many.py all 100 $dim 1e-5 > ${dim}_poly_1e-5.csv
    ./compare_many.py all 100 $dim 1e-3 > ${dim}_poly_1e-3.csv
    ./compare_many.py all 10000 $dim mono > ${dim}_mono.csv
done<commit_msg>find sasview path dynamically in compare_all<commit_after>#!/bin/sh

SASVIEW=$(ls -d ../sasview/build/lib.*)
PYOPENCL_CTX=1
PYTHONPATH=../bumps:../periodictable:$SASVIEW
export PYOPENCL_CTX PYTHONPATH

echo PYTHONPATH=$PYTHONPATH
set -x

for dim in 1d100 2d32; do
    ./compare_many.py all 100 $dim 0 > ${dim}_poly_0.csv
    ./compare_many.py all 100 $dim 1e-5 > ${dim}_poly_1e-5.csv
    ./compare_many.py all 100 $dim 1e-3 > ${dim}_poly_1e-3.csv
    ./compare_many.py all 10000 $dim mono > ${dim}_mono.csv
done
{'fraction_non_alphanumeric': 0.15031315240083507, 'fraction_numerical': 0.07620041753653445, 'mean_word_length': 3.588516746411483, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6090220', 'n_tokens_mistral': 460, 'n_tokens_neox': 405, 'n_words': 87}