<commit_before>import os
import time
import glob
import pytest
import logging
import nbformat
import subprocess
from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError


logging.basicConfig(level=logging.INFO)

EXECUTE_NOTEBOOKS = []
AVOID_NOTEBOOKS = [
    'docs/examples/data_management/change_carto_table_privacy.ipynb',
    'docs/examples/publish_and_share/publish_visualization_layout.ipynb',
    'docs/examples/publish_and_share/publish_visualization_private_table.ipynb',
    'docs/examples/publish_and_share/publish_visualization_public_table.ipynb',
    'docs/examples/_debug/testing_polygons_features.ipynb',
    'docs/examples/_debug/enrichment_big_polygons.ipynb',
]

USERNAME = os.environ.get('USERNAME')
API_KEY = os.environ.get('APIKEY')
OVERWRITE = os.environ.get('OVERWRITE', 'false').lower() == 'true'
TIMEOUT = int(os.environ.get('TIMEOUT', 600))
KERNEL = os.environ.get('KERNEL', 'python3')

CREDS_FILE = f'''
{{
    "username": "{USERNAME}",
    "api_key": "{API_KEY}"
}}
'''


def find_notebooks():
    return list(set(EXECUTE_NOTEBOOKS) - set(AVOID_NOTEBOOKS)) if EXECUTE_NOTEBOOKS \
        else list(set(glob.glob('docs/examples/**/*.ipynb', recursive=True) +
                      glob.glob('docs/guides/**/*.ipynb', recursive=True)) - set(AVOID_NOTEBOOKS)) 


class TestNotebooks:
    def teardown(self):
        time.sleep(10)

    def custom_setup(self, path):
        with open(f'{path}/creds.json', 'w') as creds_file:
            creds_file.write(CREDS_FILE)

    def custom_teardown(self, path):
        os.remove(f'{path}/creds.json')

    @pytest.mark.parametrize('notebook_filename', find_notebooks())
    def test_docs(self, notebook_filename):
        try:
            path = '/'.join(notebook_filename.split('/')[:-1])

            self.custom_setup(path)
            self.execute_notebook(notebook_filename, path)
        finally:
            self.custom_teardown(path)

    def execute_notebook(self, notebook_filename, path):
        with open(notebook_filename) as f:
            logging.info('Executing notebook: %s', notebook_filename)

            nb = nbformat.read(f, as_version=4)
            ep = ExecutePreprocessor(timeout=TIMEOUT, kernel_name=KERNEL, allow_errors=OVERWRITE,
                                     store_widget_state=OVERWRITE)
            ep.preprocess(nb, {'metadata': {'path': path}})

            if OVERWRITE:
                logging.info('Overwriting notebook: %s', notebook_filename)
                with open(notebook_filename, 'w') as fwrite:
                    nbformat.write(nb, fwrite)

                logging.info('Trusting notebook: %s', notebook_filename)
                p_jupyter = subprocess.Popen(f'jupyter trust {notebook_filename}', shell=True,
                                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                _, stderr_jupyter = p_jupyter.communicate()

                if len(stderr_jupyter) > 0:
                    raise RuntimeError(f'Error trusting the notebook ({notebook_filename}): {stderr_jupyter}')
<commit_msg>Fixed tests<commit_after>import os
import time
import glob
import pytest
import logging
import nbformat
import subprocess
from nbconvert.preprocessors import ExecutePreprocessor


logging.basicConfig(level=logging.INFO)

EXECUTE_NOTEBOOKS = []
AVOID_NOTEBOOKS = [
    'docs/examples/data_management/change_carto_table_privacy.ipynb',
    'docs/examples/publish_and_share/publish_visualization_layout.ipynb',
    'docs/examples/publish_and_share/publish_visualization_private_table.ipynb',
    'docs/examples/publish_and_share/publish_visualization_public_table.ipynb',
    'docs/examples/_debug/testing_polygons_features.ipynb',
    'docs/examples/_debug/enrichment_big_polygons.ipynb',
]

USERNAME = os.environ.get('USERNAME')
API_KEY = os.environ.get('APIKEY')
OVERWRITE = os.environ.get('OVERWRITE', 'false').lower() == 'true'
TIMEOUT = int(os.environ.get('TIMEOUT', 600))
KERNEL = os.environ.get('KERNEL', 'python3')

CREDS_FILE = f'''
{{
    "username": "{USERNAME}",
    "api_key": "{API_KEY}"
}}
'''


def find_notebooks():
    return list(set(EXECUTE_NOTEBOOKS) - set(AVOID_NOTEBOOKS)) if EXECUTE_NOTEBOOKS \
        else list(set(glob.glob('docs/examples/**/*.ipynb', recursive=True) +
                      glob.glob('docs/guides/**/*.ipynb', recursive=True)) - set(AVOID_NOTEBOOKS))


class TestNotebooks:
    def teardown(self):
        time.sleep(10)

    def custom_setup(self, path):
        with open(f'{path}/creds.json', 'w') as creds_file:
            creds_file.write(CREDS_FILE)

    def custom_teardown(self, path):
        os.remove(f'{path}/creds.json')

    @pytest.mark.parametrize('notebook_filename', find_notebooks())
    def test_docs(self, notebook_filename):
        try:
            path = '/'.join(notebook_filename.split('/')[:-1])

            self.custom_setup(path)
            self.execute_notebook(notebook_filename, path)
        finally:
            self.custom_teardown(path)

    def execute_notebook(self, notebook_filename, path):
        with open(notebook_filename) as f:
            logging.info('Executing notebook: %s', notebook_filename)

            nb = nbformat.read(f, as_version=4)
            ep = ExecutePreprocessor(timeout=TIMEOUT, kernel_name=KERNEL, allow_errors=OVERWRITE,
                                     store_widget_state=OVERWRITE)
            ep.preprocess(nb, {'metadata': {'path': path}})

            if OVERWRITE:
                logging.info('Overwriting notebook: %s', notebook_filename)
                with open(notebook_filename, 'w') as fwrite:
                    nbformat.write(nb, fwrite)

                logging.info('Trusting notebook: %s', notebook_filename)
                p_jupyter = subprocess.Popen(f'jupyter trust {notebook_filename}', shell=True,
                                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                _, stderr_jupyter = p_jupyter.communicate()

                if len(stderr_jupyter) > 0:
                    raise RuntimeError(f'Error trusting the notebook ({notebook_filename}): {stderr_jupyter}')

{'fraction_non_alphanumeric': 0.12155327133300702, 'fraction_numerical': 0.002936857562408223, 'mean_word_length': 3.081225033288948, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3810773', 'n_tokens_mistral': 2033, 'n_tokens_neox': 1943, 'n_words': 304}