<jupyter_start><jupyter_text>RUN001 - Run a notebook DescriptionThis notebook abstracts the mechanics of running and saving the resultsof a single notebook of any kernel. It:1.  Use a valid notebook executor based on Kernel type    (`azdata notebook run` or `Invoke-SqlNotebook`).2.  Validate if a kernel threw an error (not all kernels bubble errors    up to the executor).3.  Save results (metrics and the .ipynb/.html output) to the Big Data    Cluster.    -   [RUN002 - Save result in Big Data        Cluster](../notebook-runner/run002-save-result-in-bdc.ipynb)If `save_results_in_storage_pool` = True, ensure the T/SQL objects havebeen setup, using: - [RUN000 - Setup Master Pool runnerinfrastructure](../notebook-runner/run000-setup-infrastructure.ipynb) ParametersDescription of parameters:-   `notebook_path`: The notebook to run-   `namespace`: If using the SQL / PySpark/ Scala kernels specify the    Big Data Cluster namespace-   `sql_master_pool_username`: If running the SQL kernel, specific    username-   `sql_master_pool_password`: If running the SQL kernel, specific    password-   `knox_username`: If running the PySpark / Scala kernel in secure    (Kerberos) mode specify the Knox username (it is hardcoded to ‘root’    in Basic auth), \ e.g. admin-   `knox_user_domain`: i.e. AZDATA.LOCAL-   `knox_password`: If running the PySpark / Scala kernel specify the    Knox password-   `save_results_in_storage_pool`: Set to “True” to save results in    master pool & storage pool-   `app_name`: If saving results, specify any app_name (i.e. “my_app”)-   `app_version`: If saving results, specify any app_version    (i.e. “v1”)NOTE: All types are strings, due to `azdata notebook run --arguments`only supporting string type.<jupyter_code>import os, getpass, datetime

notebook_path = os.path.join(os.getcwd(), "run500-hello-world.ipynb")

namespace = "" if "AZDATA_NAMESPACE" not in os.environ else os.environ["AZDATA_NAMESPACE"]
sql_master_pool_username = "" if "AZDATA_USERNAME" not in os.environ else os.environ["AZDATA_USERNAME"]
sql_master_pool_password = "" if "AZDATA_PASSWORD" not in os.environ else os.environ["AZDATA_PASSWORD"]

knox_username = "" if "DOMAIN_SERVICE_ACCOUNT_USERNAME" not in os.environ else os.environ["DOMAIN_SERVICE_ACCOUNT_USERNAME"] 
knox_user_domain = "" if "DOMAIN_SERVICE_ACCOUNT_DOMAIN_NAME" not in os.environ else os.environ["DOMAIN_SERVICE_ACCOUNT_DOMAIN_NAME"]
knox_password = "" if "DOMAIN_SERVICE_ACCOUNT_PASSWORD" not in os.environ else os.environ["DOMAIN_SERVICE_ACCOUNT_PASSWORD"]

save_results_in_storage_pool = "True"
app_name = "app-" + getpass.getuser().lower() # set default to be app-<username>
app_version = "v1"

session_start = str(datetime.datetime.utcnow())

NOTEBOOK_CELL_TIMEOUT = 600 # Per cell timeout in seconds (10 minutes)<jupyter_output><empty_output><jupyter_text>Common functionsDefine helper functions used in this notebook.<jupyter_code># Define `run` function for transient fault handling, suggestions on error, and scrolling updates on Windows
import sys
import os
import re
import platform
import shlex
import shutil
import datetime

from subprocess import Popen, PIPE
from IPython.display import Markdown

retry_hints = {} # Output in stderr known to be transient, therefore automatically retry
error_hints = {} # Output in stderr where a known SOP/TSG exists which will be HINTed for further help
install_hint = {} # The SOP to help install the executable if it cannot be found

def run(cmd, return_output=False, no_output=False, retry_count=0, base64_decode=False, return_as_json=False):
    """Run shell command, stream stdout, print stderr and optionally return output

    NOTES:

    1.  Commands that need this kind of ' quoting on Windows e.g.:

            kubectl get nodes -o jsonpath={.items[?(@.metadata.annotations.pv-candidate=='data-pool')].metadata.name}

        Need to actually pass in as '"':

            kubectl get nodes -o jsonpath={.items[?(@.metadata.annotations.pv-candidate=='"'data-pool'"')].metadata.name}

        The ' quote approach, although correct when pasting into Windows cmd, will hang at the line:
        
            `iter(p.stdout.readline, b'')`

        The shlex.split call does the right thing for each platform, just use the '"' pattern for a '
    """
    MAX_RETRIES = 5
    output = ""
    retry = False

    # When running `azdata sql query` on Windows, replace any \n in """ strings, with " ", otherwise we see:
    #
    #    ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)')
    #
    if platform.system() == "Windows" and cmd.startswith("azdata sql query"):
        cmd = cmd.replace("\n", " ")

    # shlex.split is required on bash and for Windows paths with spaces
    #
    cmd_actual = shlex.split(cmd)

    # Store this (i.e. kubectl, python etc.) to support binary context aware error_hints and retries
    #
    user_provided_exe_name = cmd_actual[0].lower()

    # When running python, use the python in the ADS sandbox ({sys.executable})
    #
    if cmd.startswith("python "):
        cmd_actual[0] = cmd_actual[0].replace("python", sys.executable)

        # On Mac, when ADS is not launched from terminal, LC_ALL may not be set, which causes pip installs to fail
        # with:
        #
        #    UnicodeDecodeError: 'ascii' codec can't decode byte 0xc5 in position 4969: ordinal not in range(128)
        #
        # Setting it to a default value of "en_US.UTF-8" enables pip install to complete
        #
        if platform.system() == "Darwin" and "LC_ALL" not in os.environ:
            os.environ["LC_ALL"] = "en_US.UTF-8"

    # When running `kubectl`, if AZDATA_OPENSHIFT is set, use `oc`
    #
    if cmd.startswith("kubectl ") and "AZDATA_OPENSHIFT" in os.environ:
        cmd_actual[0] = cmd_actual[0].replace("kubectl", "oc")

    # To aid supportability, determine which binary file will actually be executed on the machine
    #
    which_binary = None

    # Special case for CURL on Windows.  The version of CURL in Windows System32 does not work to
    # get JWT tokens, it returns "(56) Failure when receiving data from the peer".  If another instance
    # of CURL exists on the machine use that one.  (Unfortunately the curl.exe in System32 is almost
    # always the first curl.exe in the path, and it can't be uninstalled from System32, so here we
    # look for the 2nd installation of CURL in the path)
    if platform.system() == "Windows" and cmd.startswith("curl "):
        path = os.getenv('PATH')
        for p in path.split(os.path.pathsep):
            p = os.path.join(p, "curl.exe")
            if os.path.exists(p) and os.access(p, os.X_OK):
                if p.lower().find("system32") == -1:
                    cmd_actual[0] = p
                    which_binary = p
                    break

    # Find the path based location (shutil.which) of the executable that will be run (and display it to aid supportability), this
    # seems to be required for .msi installs of azdata.cmd/az.cmd.  (otherwise Popen returns FileNotFound) 
    #
    # NOTE: Bash needs cmd to be the list of the space separated values hence shlex.split.
    #
    if which_binary == None:
        which_binary = shutil.which(cmd_actual[0])

    # Display an install HINT, so the user can click on a SOP to install the missing binary
    #
    if which_binary == None:
        print(f"The path used to search for '{cmd_actual[0]}' was:")
        print(sys.path)

        if user_provided_exe_name in install_hint and install_hint[user_provided_exe_name] is not None:
            display(Markdown(f'HINT: Use [{install_hint[user_provided_exe_name][0]}]({install_hint[user_provided_exe_name][1]}) to resolve this issue.'))

        raise FileNotFoundError(f"Executable '{cmd_actual[0]}' not found in path (where/which)")
    else:   
        cmd_actual[0] = which_binary

    start_time = datetime.datetime.now().replace(microsecond=0)

    print(f"START: {cmd} @ {start_time} ({datetime.datetime.utcnow().replace(microsecond=0)} UTC)")
    print(f"       using: {which_binary} ({platform.system()} {platform.release()} on {platform.machine()})")
    print(f"       cwd: {os.getcwd()}")

    # Command-line tools such as CURL and AZDATA HDFS commands output
    # scrolling progress bars, which causes Jupyter to hang forever, to
    # workaround this, use no_output=True
    #

    # Work around a infinite hang when a notebook generates a non-zero return code, break out, and do not wait
    #
    wait = True 

    try:
        if no_output:
            p = Popen(cmd_actual)
        else:
            p = Popen(cmd_actual, stdout=PIPE, stderr=PIPE, bufsize=1)
            with p.stdout:
                for line in iter(p.stdout.readline, b''):
                    line = line.decode()
                    if return_output:
                        output = output + line
                    else:
                        if cmd.startswith("azdata notebook run"): # Hyperlink the .ipynb file
                            regex = re.compile('  "(.*)"\: "(.*)"') 
                            match = regex.match(line)
                            if match:
                                if match.group(1).find("HTML") != -1:
                                    display(Markdown(f' - "{match.group(1)}": "{match.group(2)}"'))
                                else:
                                    display(Markdown(f' - "{match.group(1)}": "[{match.group(2)}]({match.group(2)})"'))

                                    wait = False
                                    break # otherwise infinite hang, have not worked out why yet.
                        else:
                            print(line, end='')

        if wait:
            p.wait()
    except FileNotFoundError as e:
        if install_hint is not None:
            display(Markdown(f'HINT: Use {install_hint} to resolve this issue.'))

        raise FileNotFoundError(f"Executable '{cmd_actual[0]}' not found in path (where/which)") from e

    exit_code_workaround = 0 # WORKAROUND: azdata hangs on exception from notebook on p.wait()

    if not no_output:
        for line in iter(p.stderr.readline, b''):
            try:
                line_decoded = line.decode()
            except UnicodeDecodeError:
                # NOTE: Sometimes we get characters back that cannot be decoded(), e.g.
                #
                #   \xa0
                #
                # For example see this in the response from `az group create`:
                #
                # ERROR: Get Token request returned http error: 400 and server 
                # response: {"error":"invalid_grant",# "error_description":"AADSTS700082: 
                # The refresh token has expired due to inactivity.\xa0The token was 
                # issued on 2018-10-25T23:35:11.9832872Z
                #
                # which generates the exception:
                #
                # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa0 in position 179: invalid start byte
                #
                print("WARNING: Unable to decode stderr line, printing raw bytes:")
                print(line)
                line_decoded = ""
                pass
            else:

                # azdata emits a single empty line to stderr when doing an hdfs cp, don't
                # print this empty "ERR:" as it confuses.
                #
                if line_decoded == "":
                    continue
                
                print(f"STDERR: {line_decoded}", end='')

                if line_decoded.startswith("An exception has occurred") or line_decoded.startswith("ERROR: An error occurred while executing the following cell"):
                    exit_code_workaround = 1

                # inject HINTs to next TSG/SOP based on output in stderr
                #
                if user_provided_exe_name in error_hints:
                    for error_hint in error_hints[user_provided_exe_name]:
                        if line_decoded.find(error_hint[0]) != -1:
                            display(Markdown(f'HINT: Use [{error_hint[1]}]({error_hint[2]}) to resolve this issue.'))

                # Verify if a transient error, if so automatically retry (recursive)
                #
                if user_provided_exe_name in retry_hints:
                    for retry_hint in retry_hints[user_provided_exe_name]:
                        if line_decoded.find(retry_hint) != -1:
                            if retry_count < MAX_RETRIES:
                                print(f"RETRY: {retry_count} (due to: {retry_hint})")
                                retry_count = retry_count + 1
                                output = run(cmd, return_output=return_output, retry_count=retry_count)

                                if return_output:
                                    if base64_decode:
                                        import base64
                                        return base64.b64decode(output).decode('utf-8')
                                    else:
                                        return output

    elapsed = datetime.datetime.now().replace(microsecond=0) - start_time

    # WORKAROUND: We avoid infinite hang above in the `azdata notebook run` failure case, by inferring success (from stdout output), so
    # don't wait here, if success known above
    #
    if wait: 
        if p.returncode != 0:
            raise SystemExit(f'Shell command:\n\n\t{cmd} ({elapsed}s elapsed)\n\nreturned non-zero exit code: {str(p.returncode)}.\n')
    else:
        if exit_code_workaround !=0 :
            raise SystemExit(f'Shell command:\n\n\t{cmd} ({elapsed}s elapsed)\n\nreturned non-zero exit code: {str(exit_code_workaround)}.\n')

    print(f'\nSUCCESS: {elapsed}s elapsed.\n')

    if return_output:
        if base64_decode:
            import base64
            return base64.b64decode(output).decode('utf-8')
        else:
            return output



# Hints for tool retry (on transient fault), known errors and install guide
#
retry_hints = {'azdata': ['Endpoint sql-server-master does not exist', 'Endpoint livy does not exist', 'Failed to get state for cluster', 'Endpoint webhdfs does not exist', 'Adaptive Server is unavailable or does not exist', 'Error: Address already in use', 'Login timeout expired (0) (SQLDriverConnect)', 'SSPI Provider: No Kerberos credentials available',  ], 'kubectl': ['A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond',  ], 'python': [ ], }
error_hints = {'azdata': [['Please run \'azdata login\' to first authenticate', 'SOP028 - azdata login', '../common/sop028-azdata-login.ipynb'], ['The token is expired', 'SOP028 - azdata login', '../common/sop028-azdata-login.ipynb'], ['Reason: Unauthorized', 'SOP028 - azdata login', '../common/sop028-azdata-login.ipynb'], ['Max retries exceeded with url: /api/v1/bdc/endpoints', 'SOP028 - azdata login', '../common/sop028-azdata-login.ipynb'], ['Look at the controller logs for more details', 'TSG027 - Observe cluster deployment', '../diagnose/tsg027-observe-bdc-create.ipynb'], ['provided port is already allocated', 'TSG062 - Get tail of all previous container logs for pods in BDC namespace', '../log-files/tsg062-tail-bdc-previous-container-logs.ipynb'], ['Create cluster failed since the existing namespace', 'SOP061 - Delete a big data cluster', '../install/sop061-delete-bdc.ipynb'], ['Failed to complete kube config setup', 'TSG067 - Failed to complete kube config setup', '../repair/tsg067-failed-to-complete-kube-config-setup.ipynb'], ['Data source name not found and no default driver specified', 'SOP069 - Install ODBC for SQL Server', '../install/sop069-install-odbc-driver-for-sql-server.ipynb'], ['Can\'t open lib \'ODBC Driver 17 for SQL Server', 'SOP069 - Install ODBC for SQL Server', '../install/sop069-install-odbc-driver-for-sql-server.ipynb'], ['Control plane upgrade failed. Failed to upgrade controller.', 'TSG108 - View the controller upgrade config map', '../diagnose/tsg108-controller-failed-to-upgrade.ipynb'], ['NameError: name \'azdata_login_secret_name\' is not defined', 'SOP013 - Create secret for azdata login (inside cluster)', '../common/sop013-create-secret-for-azdata-login.ipynb'], ['ERROR: No credentials were supplied, or the credentials were unavailable or inaccessible.', 'TSG124 - \'No credentials were supplied\' error from azdata login', '../repair/tsg124-no-credentials-were-supplied.ipynb'], ['Please accept the license terms to use this product through', 'TSG126 - azdata fails with \'accept the license terms to use this product\'', '../repair/tsg126-accept-license-terms.ipynb'],  ], 'kubectl': [['no such host', 'TSG010 - Get configuration contexts', '../monitor-k8s/tsg010-get-kubernetes-contexts.ipynb'], ['No connection could be made because the target machine actively refused it', 'TSG056 - Kubectl fails with No connection could be made because the target machine actively refused it', '../repair/tsg056-kubectl-no-connection-could-be-made.ipynb'],  ], 'python': [['Library not loaded: /usr/local/opt/unixodbc', 'SOP012 - Install unixodbc for Mac', '../install/sop012-brew-install-odbc-for-sql-server.ipynb'], ['WARNING: You are using pip version', 'SOP040 - Upgrade pip in ADS Python sandbox', '../install/sop040-upgrade-pip.ipynb'],  ], }
install_hint = {'azdata': [ 'SOP063 - Install azdata CLI (using package manager)', '../install/sop063-packman-install-azdata.ipynb' ],  'kubectl': [ 'SOP036 - Install kubectl command line interface', '../install/sop036-install-kubectl.ipynb' ],  }


print('Common functions defined successfully.')<jupyter_output><empty_output><jupyter_text>Is notebook being run inside a Kubernetes clusterWhen this is notebook is running inside a Kubernetes cluster, such aswhen running inside an App-Deploy pod, there is no KUBECONFIG present,therefore azdata login needs to use the -e (endpoint) approach to login.<jupyter_code>import os

if "KUBERNETES_SERVICE_PORT" in os.environ and "KUBERNETES_SERVICE_HOST" in os.environ:
    inside_kubernetes_cluster = True
    print("This notebook is running inside a Kubernetes cluster")
else:
    inside_kubernetes_cluster = False
    print("This notebook is not running inside a Kubernetes cluster")<jupyter_output><empty_output><jupyter_text>Get the Kubernetes namespace for the big data clusterGet the namespace of the Big Data Cluster use the kubectl command lineinterface .**NOTE:**If there is more than one Big Data Cluster in the target Kubernetescluster, then either:-   set \[0\] to the correct value for the big data cluster.-   set the environment variable AZDATA_NAMESPACE, before starting Azure    Data Studio.<jupyter_code># Place Kubernetes namespace name for BDC into 'namespace' variable

if "AZDATA_NAMESPACE" in os.environ:
    namespace = os.environ["AZDATA_NAMESPACE"]
else:
    try:
        namespace = run(f'kubectl get namespace --selector=MSSQL_CLUSTER -o jsonpath={{.items[0].metadata.name}}', return_output=True)
    except:
        from IPython.display import Markdown
        print(f"ERROR: Unable to find a Kubernetes namespace with label 'MSSQL_CLUSTER'.  SQL Server Big Data Cluster Kubernetes namespaces contain the label 'MSSQL_CLUSTER'.")
        display(Markdown(f'HINT: Use [TSG081 - Get namespaces (Kubernetes)](../monitor-k8s/tsg081-get-kubernetes-namespaces.ipynb) to resolve this issue.'))
        display(Markdown(f'HINT: Use [TSG010 - Get configuration contexts](../monitor-k8s/tsg010-get-kubernetes-contexts.ipynb) to resolve this issue.'))
        display(Markdown(f'HINT: Use [SOP011 - Set kubernetes configuration context](../common/sop011-set-kubernetes-context.ipynb) to resolve this issue.'))
        raise

print(f'The SQL Server Big Data Cluster Kubernetes namespace is: {namespace}')<jupyter_output><empty_output><jupyter_text>Establish if cluster is Active Directory enabledAn Active Directory enabled cluster will have a `dns` pod. Non ActiveDirectory enabled clusters do not have a `dns` pod.<jupyter_code>dns_pod = run(f'kubectl get pods -n {namespace} -o name -l app=dns', return_output=True)

if len(dns_pod) > 0:
    is_ad_enabled_cluster = True
    print(f"Cluster {namespace} is an Active Directory enabled cluster")
else:
    is_ad_enabled_cluster = False
    print(f"Cluster {namespace} is not an Active Directory enabled cluster")<jupyter_output><empty_output><jupyter_text>Notebook filename managementThe notebook to be run has many filenames over the life of it being run.Here a class is defined to abstract the mechanics of these filenames.<jupyter_code>import os

class NotebookPath:

    _full_path = None

    _azdata_logging_path = None

    _output_filename = None
    _output_dir = None
    _output_full_path = None

    _step = None

    def __init__(self, full_path):
        self._full_path = full_path
        self._output_filename = f"output-{os.path.basename(full_path)}"
        self._output_dir = os.getcwd()
        self._output_full_path = os.path.join(self._output_dir, self._output_filename)
        self._azdata_logging_path = os.path.join(self._output_dir, "tmp", "azdata_log-" + os.path.basename(full_path)[:-6])

    @property
    def full_path(self):
        return self._full_path.replace('\\', '\\\\')

    @property
    def azdata_logging_path(self):
        """To allow for concurrent execution of notebooks, ensure each notebook
           has it's own azdata.log flie
        """
        return self._azdata_logging_path.replace('\\', '\\\\')

    @property
    def output_dir(self):
        return self._output_dir.replace("\\", "\\\\")

    @property
    def output_full_path(self):
        """The output filename, that is saved/uploaded to the Big Data Cluster
        """ 
        return self._output_full_path.replace('\\', '\\\\')

    @property
    def output_full_path_html(self):
        return self._output_full_path.replace(".ipynb", ".html").replace('\\', '\\\\')

    @property
    def output_full_path_to_pass_as_azdata_arg(self):
        """When passing --arguments to azdata, an extra level of \ escaping is needed
        """
        return self.output_full_path.replace('\\', '\\\\')

input_notebook = NotebookPath(notebook_path)

print(f"full_path: {input_notebook.full_path}")
print(f"output_dir: {input_notebook.output_dir}")
print(f"output_full_path: {input_notebook.output_full_path}")
print(f"output_full_path_to_pass_as_azdata_arg: {input_notebook.output_full_path_to_pass_as_azdata_arg}")<jupyter_output><empty_output><jupyter_text>Set the `azdata` logging directoryTo support running multiple creates at the same time, place theazdata.log separately. This code is placed here, so it runs after‘injected parameters’ (which may change the app_name/app_version)<jupyter_code>os.environ["AZDATA_LOGGING_LOG_DIR"] = input_notebook.azdata_logging_path

print("Set AZDATA_LOGGING_LOG_DIR: " + input_notebook.azdata_logging_path)<jupyter_output><empty_output><jupyter_text>Notebook JSON managementTo run the notebook, the metadata of the notebook is inspected to makedecisions on what kernel type is required, what internal parametervalues are needed at exectiuon time.Here a class is defined to abtract the mechanics of inspecting theNotebook JSON.<jupyter_code>import json

class NotebookJson:

    _json = None

    def __init__(self, full_path):
        self._json = NotebookJson._load_json(full_path)

    @property
    def kernel_name(self):
        return self._json["metadata"]["kernelspec"]["name"].lower()

    @property
    def timeout(self):
        if "azdata" in self._json["metadata"]:
            if "timeout" in self._json["metadata"]["azdata"]:
                return int(self._json["metadata"]["azdata"]["timeout"])
            else:
                return NOTEBOOK_CELL_TIMEOUT
        else:
            return NOTEBOOK_CELL_TIMEOUT

    @property
    def internal_parameters(self):
        cmdline_args = ""

        if "azdata" in self._json["metadata"]:
            if "internal" in self._json["metadata"]["azdata"]:
                if "parameters" in self._json["metadata"]["azdata"]["internal"]:
                    parameters = self._json["metadata"]["azdata"]["internal"]["parameters"]

                    cmdline_args = str(parameters).replace("'", '\\"') # Windows cmd line, requires ", not '

        if cmdline_args != "":
            cmdline_args = '--arguments "' + cmdline_args + '"'

        return cmdline_args

    @property
    def json(self):
        return self._json

    def save_as(self, filename):
        NotebookJson._save_json(filename, self._json)

    @staticmethod
    def _load_json(filename):
        with open(filename, encoding="utf8") as json_file:
            return json.load(json_file)

    @staticmethod
    def _save_json(filename, contents):
        with open(filename, 'w', encoding="utf8") as outfile:
            json.dump(contents, outfile, indent=4)

input_json = NotebookJson(input_notebook.full_path)

print(f"Kernel type: {input_json.kernel_name}")<jupyter_output><empty_output><jupyter_text>Run a SQL Kernel notebookNotebooks that use the SQL kernel cannot be run using`azdata notebook run`, therefore the Powershell methodInvoke-SqlNotebook is used.Here the command line to execute a SQL Kernel notebook is built up:<jupyter_code>def get_sql_kernel_cmd_line(input_file, output_file):
    exit_code = 0

    if inside_kubernetes_cluster:
        sql_server_master_endpoint = "master-p-svc,1433"
    else:
        endpoint = run('azdata bdc endpoint list --endpoint="sql-server-master"', return_output=True)
        endpoint = json.loads(endpoint)
        sql_server_master_endpoint = endpoint['endpoint']

    print (f"The sql-server-master endpoint: {sql_server_master_endpoint}")

    if platform.system() == "Windows":
        powershell_cmd = "powershell"
    else:
        powershell_cmd = "pwsh" # on Linux powershell is called 'pwsh'!

    return f"""{powershell_cmd} -ExecutionPolicy Bypass -Command "Invoke-SqlNotebook -InputFile {input_file} -ServerInstance \\\"{sql_server_master_endpoint}\\\" -Username {sql_master_pool_username} -Password {sql_master_pool_password} -Force -OutputFile {output_file}" """

print("Function `get_sql_kernel_cmd_line` defined")<jupyter_output><empty_output><jupyter_text>Check a SQL Kernel notebook for errorsThe SQL Kernel does not return a non-zero exit code on cell error,therefore the notebook output will be inspected to look for an`output_type` of `error` and will print the `evalue` and will return anon-zero exit_code on error.<jupyter_code>def check_sql_kernel_for_error(j):
    exit_code = 0

    for cell in j["cells"]:
        if cell["cell_type"] == "code":
            if "outputs" in cell:
                for output in cell["outputs"]:
                    if output["output_type"] == "error":
                        print(output["evalue"])
                        exit_code = 1
                        break

    return exit_code

print("Function `check_sql_kernel_for_error` defined")<jupyter_output><empty_output><jupyter_text>Check Spark (Scala/PySpark) kernel output for errorThe Spark (Scala) Kernel does not return a non-zero exit code on cellerror, therefore look for an `output_type` of `stream` with a `stderr`and print the `text` and return a non-zero exit_code.<jupyter_code>def check_for_error(j):
    exit_code = 0

    for cell in j["cells"]:
        if cell["cell_type"] == "code":
            if "outputs" in cell:
                for output in cell["outputs"]:
                    if output["output_type"] == "stream" and output["name"] == "stderr":
                        exit_code = 1
                        break

    return exit_code

print("Function `check_for_error` defined")<jupyter_output><empty_output><jupyter_text>Run the notebookRun the notebook, and raise an exception if a non zero exit code isreturned. The caller should catch the exception (SystemExit) andpreserve this notebook output for offline inspection.NOTE: Different ‘kernel’ types need different execution environments(i.e. sql kernel runs via `Invoke-SqlNotebook` (powershell), most othersrun in `azdata notebook run`)<jupyter_code>from shutil import move, rmtree

exit_code = 0

if input_json.kernel_name == "sql":
    cmd_line = get_sql_kernel_cmd_line(input_notebook.full_path, input_notebook.output_full_path)
else:
    cmd_line = "azdata notebook run --path {0} --output-path {1} --timeout {2} {3}".format(
            input_notebook.full_path, 
            input_notebook.output_dir, 
            input_json.timeout, 
            input_json.internal_parameters)

start = datetime.datetime.utcnow()

try:
    run(cmd_line)
except SystemExit as ex:
    print(ex)
    exit_code = 1

end = datetime.datetime.utcnow()

# Some kernels don't check for errors, so do that here and return non-zero exit code
#
if input_json.kernel_name == "sql" and exit_code == 0:
    output_json = NotebookJson(input_notebook.output_full_path)
    exit_code = check_sql_kernel_for_error(output_json.json)

if input_json.kernel_name in ["pyspark3kernel", "pysparkkernel", "sparkkernel"] and exit_code == 0:
    output_json = NotebookJson(input_notebook.output_full_path)
    exit_code = check_for_error(output_json.json)

output_json = None<jupyter_output><empty_output><jupyter_text>Record the resultsSave the notebook .ipynb/html output files to the Storage Pool (HDFS),and record the metrics in the Master Pool (‘runner’ database)<jupyter_code>from shutil import copyfile

if save_results_in_storage_pool == "True":
    print("save_results_in_storage_pool: True")

    args = { 
        "session_start": str(session_start), 
        "notebook_path": input_notebook.output_full_path_to_pass_as_azdata_arg, 
        "app_name": app_name,
        "app_version": app_version,
        "exit_code": str(exit_code),
        "start": str(start), 
        "end": str(end)
    }

    args = str(args).replace("'", '\\"') # Windows cmd line, requires ", not '

    # Create a copy of run002, so the results for each notebook are seperated
    #
    run002_copy_full_path = "run002-save-result-in-bdc-" + os.path.basename(notebook_path)

    # In the app-deploy app folder, the .ipynbs are flattened into one folder, in the book, they are in folders.
    #
    if os.path.exists("run002-save-result-in-bdc.ipynb"):
        copyfile("run002-save-result-in-bdc.ipynb", run002_copy_full_path)
    else:
        copyfile(os.path.join("..", "notebook-runner", "run002-save-result-in-bdc.ipynb"), run002_copy_full_path)
    
    run('azdata notebook run --path {0} --output-path {1} --timeout {2} --arguments "{3}"'.format(
        run002_copy_full_path,
        input_notebook.output_dir,
        input_json.timeout,
        args))

    os.remove(run002_copy_full_path)

else:
    print("save_results_in_storage_pool: False")<jupyter_output><empty_output><jupyter_text>Raise exception if notebook return a non-zero exit code<jupyter_code>if exit_code != 0:
    raise SystemExit(f'{cmd_line}: returned non-zero exit code: {str(exit_code)}.')
print("Notebook execution is complete.")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10421271824443376, 'fraction_numerical': 0.011468845106519302, 'mean_word_length': 3.0656420942953893, 'pattern_counts': {'":': 20, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '322071', 'license': 'MIT', 'n_tokens_mistral': 9615, 'n_tokens_neox': 9095, 'n_words': 2760, 'path': 'Big-Data-Clusters/CU9/public/content/notebook-runner/run001-run-notebook.ipynb', 'repo_name': 'WilliamAntonRohm/tigertoolbox', 'size': '31215'}