<commit_before>#!/bin/bash
#
# Fetch current Numpy sources and extract docstrings from them.
#

if test "$1" = ""; then
    echo "Usage: pull-numpy.sh OUTPUTFILE"
    exit 1
fi

set -e

# 1. Fetch/update sources from SVN

#  a) numpy

if test ! -d numpy; then
    svn co http://scipy.org/svn/numpy/trunk numpy
fi

pushd numpy

svn up
svn revert -R .

# 2. Build and install the module

rm -rf dist
python2.5 setup.py install --prefix=$PWD/dist
# NB: it's important to give an absolute path to --prefix so that source file
#     information also contains absolute paths.

popd

#  b) numpy-docs

if test ! -d numpy-docs; then
    svn co http://svn.scipy.org/svn/numpy/numpy-docs/trunk numpy-docs
fi

pushd numpy-docs

svn up
svn revert -R .

popd


# 3. Extract docstrings

export SITEPATH=$PWD/numpy/dist/lib/python2.5/site-packages

python2.5 $PYDOCTOOL collect -s $SITEPATH \
    numpy numpy.doc numpy.core.records \
| $PYDOCTOOL prune -i - \
| $PYDOCTOOL numpy-docs -i - -s $SITEPATH \
    -f numpy/numpy/add_newdocs.py \
    -f numpy/numpy/core/code_generators/docstrings.py \
| $PYDOCTOOL pyrex-docs -i - -s $SITEPATH \
    -f numpy/numpy/random/mtrand/mtrand.pyx:numpy.random.mtrand \
| $PYDOCTOOL sphinx-docs -i - -n numpy-docs -e .rst \
    numpy-docs/source \
> "$1"

# NB: you can consider adding "$PYDOCTOOL mangle -i -" after prune in the pipe
#     chain to mangle object names so that they appear to originate from
#     the topmost module they were imported into.

<commit_msg>Update modules/pull-numpy.sh example script<commit_after>#!/bin/bash
#
# Fetch current Numpy sources and extract docstrings from them.
#

if test "$1" = ""; then
    echo "Usage: pull-numpy.sh OUTPUTFILE"
    exit 1
fi

set -e

# 1. Fetch/update sources from SVN

#  a) numpy

if test ! -d numpy; then
    svn co http://scipy.org/svn/numpy/trunk numpy
fi

pushd numpy

svn up
svn revert -R .

# 2. Build and install the module

rm -rf dist
python2.5 setup.py install --prefix=$PWD/dist
# NB: it's important to give an absolute path to --prefix so that source file
#     information also contains absolute paths.

popd

# 3. Extract docstrings

export SITEPATH=$PWD/numpy/dist/lib/python2.5/site-packages

python2.5 $PYDOCTOOL collect -s $SITEPATH \
    numpy numpy.doc numpy.core.records \
| $PYDOCTOOL prune -i - \
| $PYDOCTOOL numpy-docs -i - -s $SITEPATH \
    -f numpy/numpy/add_newdocs.py \
    -f numpy/numpy/core/code_generators/docstrings.py \
| $PYDOCTOOL pyrex-docs -i - -s $SITEPATH \
    -f numpy/numpy/random/mtrand/mtrand.pyx:numpy.random.mtrand \
| $PYDOCTOOL sphinx-docs -i - -n numpy-docs -e .rst \
    numpy/doc/source \
> "$1"

# NB: you can consider adding "$PYDOCTOOL mangle -i -" after prune in the pipe
#     chain to mangle object names so that they appear to originate from
#     the topmost module they were imported into.


{'fraction_non_alphanumeric': 0.11259676284306826, 'fraction_numerical': 0.00844475721323012, 'mean_word_length': 2.8680272108843536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7518814', 'n_tokens_mistral': 1097, 'n_tokens_neox': 1069, 'n_words': 329}