<commit_before>#!/usr/bin/env python
""" sphinx_notebooks

Run IPython example notebooks in [source_dir] and copy them to [output_dir].

Usage: sphinxify_notebooks [source_dir] [output_dir]

Default [source_dir] is 'doc/notebooks'.
Default [ouptut_dir] is'doc/source/user/notebooks'.

Much of this was cribbe from @hadim
(https://gistgithub.com/hadim/16e29b5848672e2e497c).
"""

from __future__ import division, print_function, absolute_import

import os
import sys
import shutil

from IPython.nbconvert import RSTExporter
from IPython.nbconvert.writers import FilesWriter
from IPython import nbformat

DEFAULT_SOURCE = 'doc/notebooks'
DEFAULT_OUTPUT = 'doc/source/user'
DEFAULT_GEN_DIR = '_notebooks'

# WindowsError is not defined on unix systems
try:
    WindowsError
except NameError:
    WindowsError = None


def _prepare_execute(file, _nbformat=None):
    if _nbformat is None:
        _nbformat = nbformat.current
    return _nbformat.reads_json(open(file).read())


def _execute_runipy(file):
    notebook = _prepare_execute(file)

    from runipy.notebook_runner import NotebookRunner

    runner = NotebookRunner(notebook)
    runner.run_notebook(skip_exceptions=True)

    return runner.nb, {}


def _execute_ipython(file, resources=None):
    notebook = _prepare_execute(file)

    from IPython.nbconvert.preprocessors import ExecutePreprocessor
    if resources is None:
        resources = {}
    notebook, resources = ExecutePreprocessor().preprocess(notebook, resources)

    return notebook, resources


def convert_notebook(notebook, tofile, resources=None):
    todir = os.path.dirname(tofile)

    exporter = RSTExporter()
    if resources is None:
        resources = {}
    output, resources = exporter.from_notebook_node(notebook,
                                                    resources=resources)

    writer = FilesWriter()
    writer.build_directory = todir
    writer.write(output, resources, notebook_name=tofile)


def find_process_files(source_dir, output_dir):
    # Figure out what processors we have available
    try:
        import runipy
        execute_runipy = True
        execute_notebook = _execute_runipy
    except ImportError:
        execute_runipy = False
    try:<commit_msg>FIX: notebook images now have unique filenames<commit_after>#!/usr/bin/env python
""" sphinx_notebooks

Run IPython example notebooks in [source_dir] and copy them to [output_dir].

Usage: sphinxify_notebooks [source_dir] [output_dir]

Default [source_dir] is 'doc/notebooks'.
Default [ouptut_dir] is'doc/source/user/notebooks'.

Much of this was cribbe from @hadim
(https://gistgithub.com/hadim/16e29b5848672e2e497c).
"""

from __future__ import division, print_function, absolute_import

import os
import sys
import shutil

from IPython.nbconvert import RSTExporter
from IPython.nbconvert.writers import FilesWriter
from IPython import nbformat

DEFAULT_SOURCE = 'doc/notebooks'
DEFAULT_OUTPUT = 'doc/source/user'
DEFAULT_GEN_DIR = '_notebooks'

# WindowsError is not defined on unix systems
try:
    WindowsError
except NameError:
    WindowsError = None


def _prepare_execute(file, _nbformat=None):
    if _nbformat is None:
        _nbformat = nbformat.current
    return _nbformat.reads_json(open(file).read())


def _execute_runipy(file):
    notebook = _prepare_execute(file)

    from runipy.notebook_runner import NotebookRunner

    runner = NotebookRunner(notebook)
    runner.run_notebook(skip_exceptions=True)

    return runner.nb, {}


def _execute_ipython(file, resources=None):
    notebook = _prepare_execute(file)

    from IPython.nbconvert.preprocessors import ExecutePreprocessor
    if resources is None:
        resources = {}
    notebook, resources = ExecutePreprocessor().preprocess(notebook, resources)

    return notebook, resources


def convert_notebook(notebook, tofile, resources=None):
    todir = os.path.dirname(tofile)

    exporter = RSTExporter()
    if resources is None:
        resources = {}
    resources.setdefault('unique_key', os.path.basename(tofile).replace('.', '_'))
    output, resources = exporter.from_notebook_node(notebook,
                                                    resources=resources)

    writer = FilesWriter()
    writer.build_directory = todir
    writer.write(output, resources, notebook_name=tofile)


def find_process_files(source_dir, output_dir):
    # Figure out what processors we have available
    try:
        import runipy
        execute_runipy = True
        execute_notebook = _execute_runipy
    except ImportError:
        execute_runipy = False
    try:
{'fraction_non_alphanumeric': 0.080017490161784, 'fraction_numerical': 0.006558810668998688, 'mean_word_length': 3.851537645811241, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947950', 'n_tokens_mistral': 1440, 'n_tokens_neox': 1393, 'n_words': 341}