<commit_before># Required python packages. You can install them all via something like:
#    pip install -r requirements.txt

# Numpy must be fully installed before many of the other packages will install,
# So you may well have to do this one manually first, before using this requirements file.
numpy

scipy
matplotlib
cython>=0.19
quantities
guppy
psutil
xlwt
MarkupSafe # this makes Jinja2 faster
Jinja2 # this is for rendering the HTML output files
cairocffi

# For postprocessing the profiling data
argparse
#gprof2dot
git+https://github.com/jrfonseca/gprof2dot.git@bf25a2abb09b # (or newer)
pyparsing==1.5.7 # default pyparsing only works with Python3, 1.5.7 works on Python2.7. Pydot needs it.
pydot

# For running the unit tests
nose
coverage

# for generating the documentation
Sphinx


<commit_msg>add SCOOP to requirements.txt<commit_after># Required python packages. You can install them all via something like:
#    pip install -r requirements.txt

# Numpy must be fully installed before many of the other packages will install,
# So you may well have to do this one manually first, before using this requirements file.
numpy

scipy
matplotlib
cython>=0.19
quantities
guppy
psutil
xlwt
MarkupSafe # this makes Jinja2 faster
Jinja2 # this is for rendering the HTML output files
cairocffi

# For postprocessing the profiling data
argparse
#gprof2dot
git+https://github.com/jrfonseca/gprof2dot.git@bf25a2abb09b # (or newer)
pyparsing==1.5.7 # default pyparsing only works with Python3, 1.5.7 works on Python2.7. Pydot needs it.
pydot

# For running the unit tests
nose
coverage

# for generating the documentation
Sphinx

# SCOOP for parallel computing
git+https://github.com/soravux/scoop.git@0.7.0


{'fraction_non_alphanumeric': 0.06133177570093458, 'fraction_numerical': 0.026285046728971962, 'mean_word_length': 4.616393442622951, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1152223', 'n_tokens_mistral': 563, 'n_tokens_neox': 544, 'n_words': 218}