<commit_before>#!/bin/bash
###############################################################
# Create the rst directly from the ipynb files
#  ./make_notebooks_rst.sh 2>&1 | tee ./make_notebooks_rst.log
###############################################################
DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

date
echo "--------------------------------------"
echo "convert notebooks to rst"
echo "--------------------------------------"

# notebook directory
NBDIR=$DIR/../examples/notebooks/core
# widget notebook directory
WIDGETDIR=$DIR/../examples/notebooks/widgets
# output directory
NBOUTDIR=$DIR/_notebooks/core

# clean output dir
rm -rf ${NBOUTDIR}
mkdir -p ${NBOUTDIR}
cd ${NBOUTDIR}

# convert the notebooks to rst after running headlessly
# if errors should abort, remove the --allow-errors option
# jupyter nbconvert --to=rst --allow-errors --execute $NBDIR/*.ipynb
# In the process the notebooks are completely executed
jupyter nbconvert --to=rst --allow-errors --output-dir=${NBOUTDIR} --execute $NBDIR/*.ipynb
#jupyter nbconvert --to=rst --allow-errors --output-dir=${NBOUTDIR} --execute $WIDGETDIR/*.ipynb
echo "DONE converting notebooks"

echo "--------------------------------------"
echo "postprocessing rst"
echo "--------------------------------------"
# remove the following lines from the documentation
sed -i '/%matplotlib inline/d' ./*.rst
sed -i '/Back to the main `Index <..\/index.ipynb>`__/d' ./*.rst
sed -i '/from __future__ import print_function/d' ./*.rst
sed -i '/te.setDefaultPlottingEngine("matplotlib")/d' ./*.rst
<commit_msg>use separate log files for notebooks<commit_after>#!/bin/bash
###############################################################
# Create the rst directly from the ipynb files
#  ./make_notebooks_rst.sh 2>&1 | tee ./make_notebooks_rst.log
###############################################################
DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

date
echo "--------------------------------------"
echo "convert notebooks to rst"
echo "--------------------------------------"

# notebook directory
NBDIR=$DIR/../examples/notebooks/core
# widget notebook directory
WIDGETDIR=$DIR/../examples/notebooks/widgets
# output directory
NBOUTDIR=$DIR/_notebooks/core

# clean output dir
rm -rf ${NBOUTDIR}
mkdir -p ${NBOUTDIR}
cd ${NBOUTDIR}

# convert the notebooks to rst after running headlessly
# if errors should abort, remove the --allow-errors option
# jupyter nbconvert --to=rst --allow-errors --execute $NBDIR/*.ipynb
# In the process the notebooks are completely executed
nsucceeded=0
nfailed=0
for f in $NBDIR/*.ipynb; do
    b=$(basename $f)
    l=$b.log
    logfile=/tmp/$l
    echo "Converting $b -> log $logfile"
    jupyter nbconvert --to=rst --allow-errors --output-dir=${NBOUTDIR} --execute $f >$logfile 2>&1
    if [ ! $? -eq 0 ]; then
        echo "  Failed: $l"
        nfailed=$((nfailed+=1))
    else
        nsucceeded=$((nsucceeded+=1))
    fi
done
#jupyter nbconvert --to=rst --allow-errors --output-dir=${NBOUTDIR} --execute $WIDGETDIR/*.ipynb
echo "DONE converting notebooks"
echo -e "  Succeeded: $nsucceeded, Failed: $nfailed\n\n"

echo "--------------------------------------"
echo "postprocessing rst"
echo "--------------------------------------"
# remove the following lines from the documentation
sed -i '/%matplotlib inline/d' ./*.rst
sed -i '/Back to the main `Index <..\/index.ipynb>`__/d' ./*.rst
sed -i '/from __future__ import print_function/d' ./*.rst
sed -i '/te.setDefaultPlottingEngine("matplotlib")/d' ./*.rst

{'fraction_non_alphanumeric': 0.2864022662889518, 'fraction_numerical': 0.0036827195467422098, 'mean_word_length': 2.8422198041349294, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5687838', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1074, 'n_words': 305}