<commit_before>import os
from invoke import task
from invoke import run as _run
from copy import deepcopy
from textwrap import dedent

from IPython.nbformat import read, write
from IPython.nbconvert.preprocessors import ClearOutputPreprocessor

def run(*args, **kwargs):
    if 'pty' not in kwargs:
        kwargs['pty'] = True
    if 'echo' not in kwargs:
        kwargs['echo'] = True
    return _run(*args, **kwargs)

def echo(msg):
    print("\033[1;37m{0}\033[0m".format(msg))

def _check_if_directory_in_path(pth, target):
    while pth != '':
        pth, dirname = os.path.split(pth)
        if dirname == target:
            return True
    return False

@task
def check_docs_input(root='.'):
    """Check that docs have output cleared."""
    echo("Checking that all docs have cleared outputs...")
    bad = []
    for dirpath, dirnames, filenames in os.walk(root):
        # skip submitted directory -- those files are allowed to have outputs
        if _check_if_directory_in_path(dirpath, 'submitted'):
            continue

        for filename in filenames:
            if os.path.splitext(filename)[1] == '.ipynb':
                # read in the notebook
                pth = os.path.join(dirpath, filename)
                with open(pth, 'r') as fh:
                    nb = read(fh, 4)

                # check notebook metadata
                if len(nb.metadata) != 0:
                    bad.append(pth)
                    continue

                # check outputs of all the cells
                for cell in nb.cells:
                    if cell.cell_type != 'code':
                        continue
                    if len(cell.outputs) != 0 or cell.execution_count is not None:
                        bad.append(pth)
                        break

    if len(bad) > 0:
        raise RuntimeError(dedent(
            """

            The following notebooks have not been properly cleared:

            {}

            Please run 'invoke clear_docs' from the root of the repository
            in order to clear the outputs of these notebooks.
            """.format(bad)
        ))

@task
def check_docs_output(root='.'):
    """Check that none of the cells in the documentation has errors."""
    echo("Checking that all docs were successfully executed...")
    for dirpath, dirnames, filenames in os.walk(root):
        # skip example directory -- those files are allowed to have errors
        if _check_if_directory_in_path(dirpath, 'release_example'):
            continue
        if _check_if_directory_in_path(dirpath, 'grade_example'):
            continue

        for filename in filenames:
            if os.path.splitext(filename)[1] == '.ipynb':
                # read in the notebook
                pth = os.path.join(dirpath, filename)
                with open(pth, 'r') as fh:
                    nb = read(fh, 4)

                # check outputs of all the cells
                for cell in nb.cells:
                    if cell.cell_type != 'code':
                        continue
                    for output in cell.outputs:
                        if output.output_type == 'error':
                            raise RuntimeError(
                                "Notebook '{}' was not successfully executed".format(pth))

@task
def docs(root='docs'):
    """Build documentation."""
    echo("Building documentation from '{}'...".format(os.path.abspath(root)))

    cwd = os.getcwd()
    os.chdir(root)

    # cleanup ignored files
    run('git clean -fdX docs')

    # make sure all the docs have been cleared
    check_docs_input(root='.')

    # build the docs
    run(
        'ipython nbconvert '<commit_msg>Give a more informative error if docs don't build<commit_after>import os
from invoke import task
from invoke import run as _run
from copy import deepcopy
from textwrap import dedent

from IPython.nbformat import read, write
from IPython.nbconvert.preprocessors import ClearOutputPreprocessor

def run(*args, **kwargs):
    if 'pty' not in kwargs:
        kwargs['pty'] = True
    if 'echo' not in kwargs:
        kwargs['echo'] = True
    return _run(*args, **kwargs)

def echo(msg):
    print("\033[1;37m{0}\033[0m".format(msg))

def _check_if_directory_in_path(pth, target):
    while pth != '':
        pth, dirname = os.path.split(pth)
        if dirname == target:
            return True
    return False

@task
def check_docs_input(root='.'):
    """Check that docs have output cleared."""
    echo("Checking that all docs have cleared outputs...")
    bad = []
    for dirpath, dirnames, filenames in os.walk(root):
        # skip submitted directory -- those files are allowed to have outputs
        if _check_if_directory_in_path(dirpath, 'submitted'):
            continue

        for filename in filenames:
            if os.path.splitext(filename)[1] == '.ipynb':
                # read in the notebook
                pth = os.path.join(dirpath, filename)
                with open(pth, 'r') as fh:
                    nb = read(fh, 4)

                # check notebook metadata
                if len(nb.metadata) != 0:
                    bad.append(pth)
                    continue

                # check outputs of all the cells
                for cell in nb.cells:
                    if cell.cell_type != 'code':
                        continue
                    if len(cell.outputs) != 0 or cell.execution_count is not None:
                        bad.append(pth)
                        break

    if len(bad) > 0:
        raise RuntimeError(dedent(
            """

            The following notebooks have not been properly cleared:

            {}

            Please run 'invoke clear_docs' from the root of the repository
            in order to clear the outputs of these notebooks.
            """.format(bad)
        ))

@task
def check_docs_output(root='.'):
    """Check that none of the cells in the documentation has errors."""

    error_msg = dedent(
        """

        Notebook '{}' was not successfully executed. The cell that failed was:

        ```
        {}
        ```

        And the error was:

        {}
        """
    )

    echo("Checking that all docs were successfully executed...")
    for dirpath, dirnames, filenames in os.walk(root):
        # skip example directory -- those files are allowed to have errors
        if _check_if_directory_in_path(dirpath, 'release_example'):
            continue
        if _check_if_directory_in_path(dirpath, 'grade_example'):
            continue

        for filename in filenames:
            if os.path.splitext(filename)[1] == '.ipynb':
                # read in the notebook
                pth = os.path.join(dirpath, filename)
                with open(pth, 'r') as fh:
                    nb = read(fh, 4)

                # check outputs of all the cells
                for cell in nb.cells:
                    if cell.cell_type != 'code':
                        continue
                    for output in cell.outputs:
                        if output.output_type == 'error':
                            raise RuntimeError(error_msg.format(
                                pth, cell.source, "\n".join(output.traceback)))

@task
def docs(root='docs'):
    """Build documentation."""
    echo("Building documentation from '{}'...".format(os.path.abspath(root)))

    cwd = os.getcwd()
    os.chdir(root)

    # cleanup ignored files
    run('git clean -fdX docs')

    # make sure all the docs have been cleared
    check_docs_input(root='.')

    # build the docs
    run(
        'ipython nbconvert '
{'fraction_non_alphanumeric': 0.09631093544137022, 'fraction_numerical': 0.0047430830039525695, 'mean_word_length': 1.7563543936092956, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3965862', 'n_tokens_mistral': 2159, 'n_tokens_neox': 2028, 'n_words': 662}