<jupyter_start><jupyter_text>AutoML on remote AML Compute (<NAME> Safe Driving Prediction)This notebook is refactored (from the original AutoML local training notebook) to use AutoML on remote AML compute, in a cluster.It also uses AML Datasets for training instead of Pandas Dataframes.  Utilitity methodsSome methods to keep track of the job, download logs etc.<jupyter_code># utility methods

# Currently, there's no SDK v2 equivalent of v1's 'show_output' or 'wait_for_completion' functionality, 
# that prints the AutoML iteration info

def show_output(client, job) -> None:    
    # This doesn't appear to stream anything at the moment
    client.jobs.stream(created_job.name)


def wait_for_completion(client, job, poll_duration: int = 30) -> None:    
    """Poll for job status every `poll_duration` seconds, until it is terminated"""
    import time
    from azure.ml._operations.run_history_constants import RunHistoryConstants

    cur_status = client.jobs.get(job.name).status
    print("Current job status: ", cur_status)
    while cur_status not in RunHistoryConstants.TERMINAL_STATUSES:
        time.sleep(poll_duration)
        cur_status = client.jobs.get(job.name).status
        print("Current job status: ", cur_status)


def download_outputs(client, job) -> None:
    # This does not download any logs (no models as well, since this is at the parent run level)
    client.jobs.download(job.name, download_path="./outputs")

    # For the child run level, currently this throws an exception saying it's not supported for the job type
    try:
        first_child_run = "{}_0".format(job.name)
        client.jobs.download(first_child_run, download_path="./outputs/")
    except Exception as e:
        import traceback

        print(str(e))
        traceback.print_exc()


def print_studio_url(job, open_in_new_tab: bool = False) -> None:
    # TODO: Any easier way to get the URL?
    
    print("Studio URL: ", job.interaction_endpoints['Studio'].endpoint)
    if open_in_new_tab:
        import webbrowser
        webbrowser.open(job.interaction_endpoints['Studio'].endpoint)
        

def download_outputs_via_mlflow_client(mlflow_client, run_id, path) -> str:
    """Download the `path` (file or dir) from the run artifacts, returns the local path download"""
    local_path = os.path.join("/tmp/artifact_downloads/{}".format(run_id), path)
    if os.path.exists(local_path):
        print("Directory {} already exists. Skipping download.".format(os.path.join(local_dir, path)))
    else:
        # download outputs
        if not os.path.exists(local_dir):
            os.mkdir(local_dir)

        local_path = mlflow_client.download_artifacts(run_id, path, local_dir)
        print("Artifacts downloaded to: {}".format(local_path))
        print("Artifacts: {}".format(os.listdir(local_path)))
    return local_path<jupyter_output><empty_output><jupyter_text>Import Needed PackagesImport the general packages needed for this notebook. These are SDK V2 packages needed to create compute, upload dataset, submit jobs etc.<jupyter_code># Global imports
from azure.ml import MLClient
from azure.core.exceptions import ResourceExistsError

from azure.ml.entities.workspace.workspace import Workspace
from azure.ml.entities.compute.compute import Compute
from azure.ml.entities.assets import Data<jupyter_output><empty_output><jupyter_text>Check Azure ML SDK version<jupyter_code>import azure.ml
print("You are currently using version", azure.ml, "of the Azure ML SDK")<jupyter_output>You are currently using version <module 'azure.ml' from '/home/schrodinger/automl/sdk-cli-v2/src/azure-ml/azure/ml/__init__.py'> of the Azure ML SDK<jupyter_text>Initiliaze MLClientThe MLClient is used to interface with AzureML services, to submit job, create compute, upload data etc.The resource group must be existing at this point.<jupyter_code>subscription_id = '381b38e9-9840-4719-a5a0-61d9585e1e91'
resource_group_name = 'gasi_rg_neu'

client = MLClient(subscription_id, resource_group_name) # default_workspace_name=workspace)<jupyter_output><empty_output><jupyter_text>Initialize WorkspaceAlso set this as the default workspace for submitting ML Jobs<jupyter_code>workspace_name = 'gasi_ws_neu'
workspace = Workspace(name=workspace_name)

try:
    client.workspaces.create(workspace)
except ResourceExistsError as re:
    print(re)
    
client.default_workspace_name = workspace_name<jupyter_output>Workspace with name gasi_ws_neu already exists.<jupyter_text>Initialize MLFlow<jupyter_code># Set the tracking URI to AzureML, and changing the active experiment

import mlflow

##### NOTE: This is SDK v1 API #####
# TODO: How do we get this from MLClient? Tracking URI can't be obtained from v2 Workspace object
from azureml.core import Workspace as WorkspaceV1
ws = WorkspaceV1(workspace_name=workspace_name, resource_group=resource_group_name, subscription_id=subscription_id)
####################################

mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())

# Set the active experiment, creating one if it doesn't exist
# mlflow.set_experiment(experiment_name)

# Get Experiment Details
# experiment = mlflow.get_experiment_by_name(experiment_name)
# print("Experiment_id: {}".format(experiment.experiment_id))
# print("Artifact Location: {}".format(experiment.artifact_location))
# print("Tags: {}".format(experiment.tags))
# print("Lifecycle_stage: {}".format(experiment.lifecycle_stage))

print("\nRegistry URI:         {}".format(mlflow.get_registry_uri()))
print("\nCurrent tracking uri: {}".format(mlflow.get_tracking_uri()))<jupyter_output>Failure while loading azureml_run_type_providers. Failed to load entrypoint hyperdrive = azureml.train.hyperdrive:HyperDriveRun._from_run_dto with exception (azureml-train-restclients-hyperdrive 0.1.0.0 (/home/schrodinger/automl/AzureMlCli/src/azureml-train-restclients-hyperdrive), Requirement.parse('azureml-train-restclients-hyperdrive~=1.27.0')).
Failure while loading azureml_run_type_providers. Failed to load entrypoint automl = azureml.train.automl.run:AutoMLRun._from_run_dto with exception (azureml-dataset-runtime 0.1.0.0 (/home/schrodinger/automl/AzureMlCli/src/azureml-dataset-runtime), Requirement.parse('azureml-dataset-runtime~=1.27.0')).
Warning: Falling back to use azure cli login credentials.
If you run your code in unattended mode, i.e., where you can't give a user input, then we recommend to use ServicePrincipalAuthentication or MsiAuthentication.
Please refer to aka.ms/aml-notebook-auth for different authentication mechanisms in azureml-sdk.<jupyter_text>(Optional) Submit dataset file into DataStore (Azure Blob under the covers)If there's a CSV file locally, upload it to the datastore.Note that this is currently going to upload the dataset as a File Dataset, which is incompatible with AutoML. As such, ensure that a TabularDataset is registered with this name outside of this notebook.<jupyter_code># Upload dataset

dataset_name = "porto_seguro_safe_driver_prediction_trimmed"
dataset_version = 1

training_data = Data(name=dataset_name, version=dataset_version, local_path="./porto_data")

try:
    data = client.data.create_or_update(training_data)
    print("Uploaded to path  : ", data.path)
    print("Datastore location: ", data.datastore)
except Exception as e:
    print("Could not create dataset. ", str(e))

training_data
assert "{}:{}".format(dataset_name, dataset_version) == "{}:{}".format(training_data.name, training_data.version)<jupyter_output><empty_output><jupyter_text>Load data into Azure ML Dataset and Register into WorkspaceTabular Datasets are currently not supported with SDK v2<jupyter_code># Try to load the dataset from the Workspace

dataset = client.data.get(name=dataset_name, version=dataset_version)


# found = False
# aml_dataset_name = "porto_seguro_safe_driver_prediction_train"

# if aml_dataset_name in ws.datasets.keys(): 
#        found = True
#        aml_dataset = ws.datasets[aml_dataset_name] 
#        print("Dataset loaded from the Workspace")
       
# if not found:
#         # Create AML Dataset and register it into Workspace
#         print("Dataset does not exist in the current Workspace. It will be imported and registered.")
        
#         # Option A: Create AML Dataset from file in AML DataStore
#         # datastore = ws.get_default_datastore()
#         # aml_dataset = Dataset.Tabular.from_delimited_files(path=datastore.path('Datasets/porto_seguro_safe_driver_prediction/porto_seguro_safe_driver_prediction_train.csv'))
#         # data_origin_type = 'AMLDataStore'
        
#         # Option B: Create AML Dataset from file in HTTP URL
#         data_url = 'https://azmlworkshopdata.blob.core.windows.net/safedriverdata/porto_seguro_safe_driver_prediction_train.csv'
#         aml_dataset = Dataset.Tabular.from_delimited_files(data_url)  
#         data_origin_type = 'HttpUrl'
        
#         print(aml_dataset)
                
#         #Register Dataset in Workspace
#         registration_method = 'SDK'  # or 'UI'
#         aml_dataset = aml_dataset.register(workspace=ws,
#                                            name=aml_dataset_name,
#                                            description='Porto Seguro Safe Driver Prediction Train dataset file',
#                                            tags={'Registration-Method': registration_method, 'Data-Origin-Type': data_origin_type},
#                                            create_new_version=True)
        
#         print("Dataset created from file and registered in the Workspace")

dataset
# # Use Pandas DataFrame just to sneak peak some data and schema
# data_df = aml_dataset.to_pandas_dataframe()
# print(data_df.shape)
# print(data_df.describe())
# data_df.head(5)<jupyter_output><empty_output><jupyter_text>Split Data into Train and Test AML Tabular DatasetsRemote AML Training you need to use AML Datasets, you cannot submit Pandas Dataframes to remote runs of AutoMLConfig.Note that AutoMLConfig below is not using the Test dataset (you only provide a single dataset that will internally be split in validation/train datasets or use cross-validation depending on the size of the dataset. The boundary for that is 20k rows, using cross-validation if less than 20k. This can also be decided by the user.). The Test dataset will be used at the end of the notebook to manually calculate the quality metrics with a dataset not seen by AutoML training.<jupyter_code># # Split in train/test datasets (Test=10%, Train=90%)

# train_dataset, test_dataset = aml_dataset.random_split(0.9, seed=0)

# # Use Pandas DF only to check the data
# train_df = train_dataset.to_pandas_dataframe()
# test_df = test_dataset.to_pandas_dataframe()
# print(train_df.shape)
# print(test_df.shape)

# train_df.describe()
# train_df.head(5)<jupyter_output><empty_output><jupyter_text>Connect to Remote AML Compute (Existing AML cluster)Note that this step currently fails due to some deserialization error from SDK. Ensure that a compute cluster exists by creating it outside of this notebook.<jupyter_code># Set or create compute

cpu_cluster_name = "cpucluster"
compute = Compute("amlcompute",
                  name=cpu_cluster_name, size="STANDARD_D13_V2",
                  min_instances=0, max_instances=3,
                  idle_time_before_scale_down=120)

# Load directly from YAML file
# compute = Compute.load("./compute.yaml")

try:
    # TODO: This currently results in an exception in Azure ML, please create compute manually.
    client.compute.create(compute)
except ResourceExistsError as re:
    print(re)
except Exception as e:
    import traceback
    
    print("Could not create compute.", str(e))
    traceback.print_exc()
# For additional details of current AmlCompute status:
# aml_remote_compute.get_status()<jupyter_output><empty_output><jupyter_text>Train with Azure AutoML automatically searching for the 'best model' (Best algorithms and best hyper-parameters)  List and select primary metric to drive the AutoML classification problem<jupyter_code># from azureml.train import automl

# # List of possible primary metrics is here:
# # https://docs.microsoft.com/en-us/azure/machine-learning/how-to-configure-auto-train#primary-metric
    
# # Get a list of valid metrics for your given task
# automl.utilities.get_primary_metrics('classification')<jupyter_output><empty_output><jupyter_text>Define AutoML Experiment settings<jupyter_code>from azure.ml._restclient._2020_09_01_preview.machinelearningservices.models import GeneralSettings, LimitSettings, DataSettings, TrainingDataSettings, ValidationDataSettings, TrainingSettings
from azure.ml._restclient._2020_09_01_preview.machinelearningservices.models._azure_machine_learning_workspaces_enums import TaskType, OptimizationMetric
from azure.ml._schema.compute_binding import InternalComputeConfiguration
from azure.ml.entities import AutoMLJob
from azure.ml.entities.job.automl.featurization import FeaturizationSettings

# TODO: Add the following
# blocked_models = ['LogisticRegression', 'ExtremeRandomTrees', 'RandomForest'], 
# allowed_models = ['LightGBM'],
# enable_voting_ensemble = True,
# enable_stack_ensemble = False,
# enable_early_stopping= True,
# experiment_timeout_hours=3,                           
# debug_log='automated_ml_errors.log',
# verbosity= logging.DEBUG,
# enable_onnx_compatible_models=True,

compute = InternalComputeConfiguration(target=cpu_cluster_name)

general_settings = GeneralSettings(task_type=TaskType.CLASSIFICATION,
                                   primary_metric= OptimizationMetric.AUC_WEIGHTED,
                                   enable_model_explainability=True)

# TODO: Seems like a bug here, max_trials=3 + max_concurrent_trials=4 seems to only trigger one child run
limit_settings = LimitSettings(job_timeout=60,
                               max_trials=4,
                               max_concurrent_trials=4,
                               enable_early_termination=False)

training_data_settings = TrainingDataSettings(dataset_arm_id="{}:{}".format(training_data.name, training_data.version),
                                              target_column_name="target")
validation_data_settings = ValidationDataSettings(validation_size=0.1)
data_settings = DataSettings(training_data=training_data_settings, validation_data=validation_data_settings)

featurization_settings = FeaturizationSettings(featurization_config="auto")

training_settings = TrainingSettings(enable_dnn_training=False)

extra_automl_settings = {"save_mlflow": True}

automl_job = AutoMLJob(
#     name=job_name,
    compute=compute,
    general_settings=general_settings,
    limit_settings=limit_settings,
    data_settings=data_settings,
    training_settings=training_settings,
    featurization_settings=featurization_settings,
    properties=extra_automl_settings,
)

automl_job<jupyter_output><empty_output><jupyter_text>Run Experiment (on remote AML Compute) with multiple child runs under the covers<jupyter_code># Submit job
# TODO: There appears to be a bug here (repro: try executing this cell twice)
created_job = client.jobs.create_or_update(automl_job)
created_job

# Dump _all_ info we have about the job 
# created_job._dump_yaml()<jupyter_output><empty_output><jupyter_text>Explore results with Widget> Note: This doesn't have any equivalent SDK v2 API right now.<jupyter_code># # Explore the results of automatic training with a Jupyter widget: https://docs.microsoft.com/en-us/python/api/azureml-widgets/azureml.widgets?view=azure-ml-py
# from azureml.widgets import RunDetails
# RunDetails(parent_run).show()
# Wait for the remote parent run to complete

# Get Studio URL, open in new tab
print_studio_url(created_job)

# Wait until the job is finished
wait_for_completion(client, created_job)

# Download logs + outputs locally
download_outputs(client, created_job)<jupyter_output>Studio URL:  https://ml.azure.com/runs/05bb01e9-a479-48ba-a0e3-3bcd868dbcd7?wsid=/subscriptions/381b38e9-9840-4719-a5a0-61d9585e1e91/resourcegroups/gasi_rg_neu/workspaces/gasi_ws_neu&tid=72f988bf-86f1-41af-91ab-2d7cd011db47
Current job status:  NotStarted
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Running
Current job status:  Completed<jupyter_text>Measure Parent Run Time needed for the whole AutoML process > Note: Todo -  Start time /  End time is available on run.info[start_time, end_time]<jupyter_code># import time
# from datetime import datetime

# run_details = parent_run.get_details()

# # Like: 2020-01-12T23:11:56.292703Z
# end_time_utc_str = run_details['endTimeUtc'].split(".")[0]
# start_time_utc_str = run_details['startTimeUtc'].split(".")[0]
# timestamp_end = time.mktime(datetime.strptime(end_time_utc_str, "%Y-%m-%dT%H:%M:%S").timetuple())
# timestamp_start = time.mktime(datetime.strptime(start_time_utc_str, "%Y-%m-%dT%H:%M:%S").timetuple())

# parent_run_time = timestamp_end - timestamp_start
# print('Run Timing: --- %s minutes needed for running the whole Remote AutoML Experiment ---' % (parent_run_time/60))<jupyter_output><empty_output><jupyter_text>Creating ModelProxy for submitting prediction runs to the training environment.We will create a ModelProxy for the best child run, which will allow us to submit a run that does the prediction in the training environment. Unlike the local client, which can have different versions of some libraries, the training environment will have all the compatible libraries for the model already.> **Note:**  This is currently not possible on SDK V2, the ModelProxy object expects a SDK v1 Run object. **Gaps**:  1. Make ModelProxy v1 concept independent (e.g. Run, Environment etc.), accepting individual properties that are needed for a Model Proxy job. OR, create ModelProxyJob? Or...?  2. Child runs are currently not submitted via. MFE, hence there's no way today to load the underlying child runs as v2 a 'Job', and access related information (like get it's environment, outputs etc.)<jupyter_code># ######################### Prepare Data ############################
# y_test = test_dataset.keep_columns('target')
# test_data_no_label = test_dataset.drop_columns('target')

# test_data_no_label_df = test_data_no_label.to_pandas_dataframe()
# print(test_data_no_label_df.shape)
# ####################################################################


# ################## Model Proxy Run ########################
# from azureml.train.automl.model_proxy import ModelProxy
# best_run = parent_run.get_best_child()
# # best_run = parent_run.get_best_child(metric = "accuracy")

# best_model_proxy = ModelProxy(best_run, aml_remote_compute)
# y_pred_test = best_model_proxy.predict(test_data_no_label)

# y_pred_test
# ############################################################<jupyter_output><empty_output><jupyter_text>Show hyperparametersShow the model pipeline used for the best run with its hyperparameters.> **Note:** This isn't possible with SDK V2, much of the Run DTO stuff gets lost in the v2 equivalent (Job).  **Gaps:**  1. Child runs are currently not submitted via. MFE, hence there's no way today to load the underlying child runs as v2 a 'Job', and access related information (like get it's environment, outputs etc.)  2. Decide if these methods need to be on the Run object, or should be ported to the model object, and be accessed from there (e.g. `best_automl_job.load_model().print_pipeline()`)  Retrieve the 'Best' ModelUsing MLFlowClient to get the best child run<jupyter_code>from mlflow.tracking import MlflowClient

# TODO: Use this run, as it has MLFlow model stored on the run - AutoML_3454b06e-2e3e-4e3e-a8e0-f52f50f9f358

mlflow_client = MlflowClient()
mlflow_parent_run = mlflow_client.get_run(created_job.name)

best_child_run_id = mlflow_parent_run.data.tags["automl_best_child_run_id"]
print("Found best child run id: ", best_child_run_id)

mlflow_best_child_run = mlflow_client.get_run(best_child_run_id)
mlflow_best_child_run<jupyter_output>Found best child run id:  05bb01e9-a479-48ba-a0e3-3bcd868dbcd7_1<jupyter_text>Show hyperparametersShow the model pipeline used for the best run with its hyperparameters.<jupyter_code>!pip install xgboost==0.90
# Local predictions using python function flavor

import mlflow.pyfunc

try:
    fitted_model = mlflow.pyfunc.load_model("runs:/{}/outputs".format(mlflow_best_child_run.info.run_id))
except Exception as e:
    # TODO: This is probably due to a bug, where MLFlow models are not being generated despite 'save_mlflow'
    print(str(e))
    print("Failed to load MLFlow model, downloading artifacts manually and loading the model.")
    
    import os, pickle

    local_dir = "/tmp/artifact_downloads/{}".format(mlflow_best_child_run.info.run_id)
    if not os.path.exists(local_dir):
        os.mkdir(local_dir)

    local_path = mlflow_client.download_artifacts(mlflow_best_child_run.info.run_id, "outputs", local_dir)
    print("Artifacts downloaded in: {}".format(local_path))
    print("Artifacts: {}".format(os.listdir(local_path)))
    
    pickled_model_path = "{}/model.pkl".format(local_path)
    with open(pickled_model_path, "rb") as model_file:
        fitted_model = pickle.load(model_file)

print(fitted_model)
# Print more information about the winning model
print("Preprocessor: \n{}\n".format(fitted_model.steps[1]))
print("Estimator: \n", fitted_model.steps[2])<jupyter_output>Preprocessor: 
('MaxAbsScaler', MaxAbsScaler(copy=True))

Estimator: 
 ('XGBoostClassifier', XGBoostClassifier(
    random_state=0,
    n_jobs=-1,
    problem_info=None
))<jupyter_text>Retrieve METRICS for All Child RunsYou can also use SDK methods to fetch all the child runs and see individual metrics that we log.<jupyter_code>import pandas as pd

all_iter_metrics = dict()

# max_trials is = 4, so gather metrics for the 4 child iterations
for i in range(4):
    # Construct child run id
    child_run_id = mlflow_parent_run.info.run_id + "_{}".format(i)
    
    # parse metrics for this iteration
    metrics = mlflow_client.get_run(child_run_id).data.metrics
    
    # index by iteration
    all_iter_metrics[i] = metrics

rundata = pd.DataFrame(all_iter_metrics).sort_index(1)
rundata<jupyter_output><empty_output><jupyter_text>Retrieve the Best Model's explanationRetrieve the explanation from the best_run which includes explanations for engineered features and raw features. Make sure that the run for generating explanations for the best model is completed.<jupyter_code>model_explain_run_id = mlflow_best_child_run.data.tags["model_explain_run_id"]

# Wait for the best model explanation run to complete
status = ""
while True:
    mlflow_model_explain_run = mlflow_client.get_run(model_explain_run_id)
    status = mlflow_model_explain_run.info.status
    print("Current Status: ", status)
    if status == "FINISHED":
        break
    time.sleep(10)

mlflow_model_explain_run<jupyter_output>Current Status:  FINISHED<jupyter_text>Download and Print engineered feature importance from artifact storeYou can use ExplanationClient to download the engineered feature explanations from the artifact store of the best_run.> **Note**: \  This wouldn't work. It would likely require changes in ExplanationClient to accept non-v1 structures in its initialization, similar to ModelProxy<jupyter_code># from azureml.interpret import ExplanationClient

# client = ExplanationClient.from_run(best_run)
# engineered_explanations = client.download_model_explanation(raw=False)
# exp_data = engineered_explanations.get_feature_importance_dict()
# exp_data<jupyter_output><empty_output><jupyter_text>Download raw feature importance from artifact storeYou can use ExplanationClient to download the raw feature explanations from the artifact store of the best_run.<jupyter_code># client = ExplanationClient.from_run(best_run)
# engineered_explanations = client.download_model_explanation(raw=True)
# exp_data = engineered_explanations.get_feature_importance_dict()
# exp_data<jupyter_output><empty_output><jupyter_text>Register Model in Workspace model registry<jupyter_code># Ensure that the model exists locally
output_path = download_outputs_via_mlflow_client(mlflow_client, mlflow_best_child_run.info.run_id, "outputs")

# Create (register?) the model
azure_model = Model(name="porto-seg-automl-remote-compute", version=1, local_path=os.path.join(output_path, "model.pkl"))
azure_model = client.models.create_or_update(azure_model)


registered_model = parent_run.register_model(model_name='porto-seg-automl-remote-compute', 
                                           description='Porto Seguro Model from plain AutoML in remote AML compute')

print(parent_run.model_id)
registered_model<jupyter_output><empty_output><jupyter_text>Deploy Model<jupyter_code>from azure.ml.entities import Endpoint, ManagedOnlineEndpoint, Environment, \
CodeConfiguration, ManagedOnlineDeployment, ManualScaleSettings, Code

inference_script_file_name = os.path.join(output_path, "scoring_file_v_1_0_0.py")
conda_environment_yaml = os.path.join(output_path, "conda.yaml")

print("Inference File: ", inference_script_file_name)
print("Conda Environment File: ", conda_environment_yaml)

assert os.path.exists(inference_script_file_name)
assert os.path.exists(conda_environment_yaml)


# Prepare the deployment configuration
environment = Environment(
    name="environment-{}".format(mlflow_best_child_run.info.run_id[:8]),
    version=1,
    path=".",
    conda_file=conda_environment_yaml,
    docker_image="mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20210301.v1",
)

code = Code(
    name="environment-{}".format(mlflow_best_child_run.info.run_id[:8]),
    version=1,
    local_path=inference_script_file_name,
)
code_configuration = CodeConfiguration(
    code=code,
    scoring_script=inference_script_file_name
)

scale_settings = ManualScaleSettings(
    scale_type="Manual",
    min_instances=1,
    max_instances=2,
    instance_count=1
)
deployment = ManagedOnlineDeployment(
    name="deployment-{}".format(mlflow_best_child_run.info.run_id[:8]),
    model=registered_model,
    environment=environment,
    code_configuration=code_configuration,
    instance_type="Standard_F2s_v2",
    scale_settings=scale_settings,
                                    )
online_endpoint = ManagedOnlineEndpoint(
    name="endpoint-{}".format(mlflow_best_child_run.info.run_id[:8]),
    deployments=[deployment],
    description="Demo model deployment",
    tags={"deployed_using": "sdkv2"}
)
##### Loading from YAML
# endpoint = Endpoint.load("/home/schrodinger/automl/Easy-AutoML-MLOps/notebooks/3-automl-remote-compute-run/endpoint.yml")

try:
    client.endpoints.create(online_endpoint)
except Exception as e:
    print("Deployment failed: ", str(e))
    traceback.print_exc()<jupyter_output>Inference File:  /tmp/artifact_downloads/05bb01e9-a479-48ba-a0e3-3bcd868dbcd7_1/outputs/scoring_file_v_1_0_0.py
Conda Environment File:  /tmp/artifact_downloads/05bb01e9-a479-48ba-a0e3-3bcd868dbcd7_1/outputs/conda.yaml<jupyter_text>See files associated with the 'Best run'<jupyter_code>artifacts = mlflow_client.list_artifacts(mlflow_best_child_run.info.run_id)

directories = []
files = []
for artifact in artifacts:
    if artifact.is_dir:
        directories.append(artifact)
    else:
        # File
        files.append(artifact)

print("Directories: ", set([d.path for d in directories]))
print("Files: ", set([f.path for f in files]))<jupyter_output>Directories:  {'azureml-logs', 'logs', 'explanation', 'outputs'}
Files:  {'confusion_matrix', 'automl_driver.py', 'accuracy_table'}<jupyter_text>Make Predictions and calculate metrics  Prep Test Data: Generating data in-line (can also load a local test file as pandas dataframe). Tabular Datasets are missing, so can't capture a pandas dataframe object for local predictions, nor there is test run support on SDKv2<jupyter_code>import numpy as np
import json

# Duplicating the same row twice
raw_data = json.dumps({
     'data': [
         [20,2,1,3,1,0,0,1,0,0,0,0,0,0,0,8,1,0,0,0.6,0.1,0.61745445,6,1,-1,0,1,11,1,1,0,1,99,2,0.31622777,0.6396829,0.36878178,3.16227766,0.2,0.6,0.5,2,2,8,1,8,3,10,3,0,0,10,0,1,0,0,1,0, 0],
         [20,2,1,3,1,0,0,1,0,0,0,0,0,0,0,8,1,0,0,0.6,0.1,0.61745445,6,1,-1,0,1,11,1,1,0,1,99,2,0.31622777,0.6396829,0.36878178,3.16227766,0.2,0.6,0.5,2,2,8,1,8,3,10,3,0,0,10,0,1,0,0,1,0, 1]
     ],
     'method': 'predict'  # If you have a classification model, you can get probabilities by changing this to 'predict_proba'.
 })

numpy_data = np.array(json.loads(raw_data)['data'])

df_data = pd.DataFrame(data=numpy_data, columns=['id', 'ps_ind_01', 'ps_ind_02_cat', 'ps_ind_03', 'ps_ind_04_cat',
                                               'ps_ind_05_cat', 'ps_ind_06_bin', 'ps_ind_07_bin', 'ps_ind_08_bin',
                                               'ps_ind_09_bin', 'ps_ind_10_bin', 'ps_ind_11_bin', 'ps_ind_12_bin',
                                               'ps_ind_13_bin', 'ps_ind_14', 'ps_ind_15', 'ps_ind_16_bin',
                                               'ps_ind_17_bin', 'ps_ind_18_bin', 'ps_reg_01', 'ps_reg_02', 'ps_reg_03',
                                               'ps_car_01_cat', 'ps_car_02_cat', 'ps_car_03_cat', 'ps_car_04_cat',
                                               'ps_car_05_cat', 'ps_car_06_cat', 'ps_car_07_cat', 'ps_car_08_cat',
                                               'ps_car_09_cat', 'ps_car_10_cat', 'ps_car_11_cat', 'ps_car_11',
                                               'ps_car_12', 'ps_car_13', 'ps_car_14', 'ps_car_15', 'ps_calc_01',
                                               'ps_calc_02', 'ps_calc_03', 'ps_calc_04', 'ps_calc_05', 'ps_calc_06',
                                               'ps_calc_07', 'ps_calc_08', 'ps_calc_09', 'ps_calc_10', 'ps_calc_11',
                                               'ps_calc_12', 'ps_calc_13', 'ps_calc_14', 'ps_calc_15_bin',
                                               'ps_calc_16_bin', 'ps_calc_17_bin', 'ps_calc_18_bin', 'ps_calc_19_bin',
                                               'ps_calc_20_bin', 'target'])
y_test = df_data.pop('target')
print("y_test shape: ", y_test.shape)
df_data<jupyter_output>y_test shape:  (2,)<jupyter_text>Make predictions in bulk<jupyter_code># Try the best model making predictions with the test dataset
y_predictions = fitted_model.predict(df_data)

print(y_predictions)<jupyter_output>[0 0]<jupyter_text>Get all the predictions' probabilities needed to calculate ROC AUC<jupyter_code>class_probabilities = fitted_model.predict_proba(df_data)

print('Some class probabilities...: ')
print(class_probabilities)<jupyter_output>Some class probabilities...: 
[[0.9700505  0.02994949]
 [0.9700505  0.02994949]]<jupyter_text>Evaluate ModelEvaluating performance is an essential task in machine learning. In this case, because this is a classification problem, the data scientist elected to use an AUC - ROC Curve. When we need to check or visualize the performance of the multi - class classification problem, we use AUC (Area Under The Curve) ROC (Receiver Operating Characteristics) curve. It is one of the most important evaluation metrics for checking any classification model’s performance.<img src="https://www.researchgate.net/profile/Oxana_Trifonova/publication/276079439/figure/fig2/AS:614187332034565@1523445079168/An-example-of-ROC-curves-with-good-AUC-09-and-satisfactory-AUC-065-parameters.png"     alt="Markdown Monster icon"     style="float: left; margin-right: 12px; width: 320px; height: 239px;" />  Calculate the ROC AUC with probabilities vs. the Test Dataset<jupyter_code># Not enough test data, but no changes here

from sklearn.metrics import roc_auc_score

#roc_auc_score(y, clf.predict_proba(X)[:, 1])
#roc_auc_score(y, clf.decision_function(X))

print('ROC AUC *method 1*:')
print(roc_auc_score(y_test, class_probabilities[:,1]))

print('ROC AUC Weighted:')
print(roc_auc_score(y_test, class_probabilities[:,1], average='weighted'))
# AUC with plain LightGBM was: 0.6374553321494826<jupyter_output>ROC AUC *method 1*:
0.5
ROC AUC Weighted:
0.5<jupyter_text>Calculate the Accuracy with predictions vs. the Test Dataset<jupyter_code>print(df_data.shape)
print(y_predictions.shape)
from sklearn.metrics import accuracy_score

print('Accuracy:')
print(accuracy_score(y_test, y_predictions))<jupyter_output>Accuracy:
0.5<jupyter_text>Load model in memory  (Option A: Load from model .pkl file)This is demonstrated above.  (Option B: Load from model registry in Workspace) Using SDK v2<jupyter_code>from azure.ml.entities.assets import Model

# Retrieve the registered model by name.
registered_model = client.models.get(azure_model.name, azure_model.version)
registered_model<jupyter_output>Directory /tmp/artifact_downloads/05bb01e9-a479-48ba-a0e3-3bcd868dbcd7_1/outputs already exists. Skipping download.<jupyter_text>Using MLFlow<jupyter_code># Retrieve the registered model by name using MLFlow client
registered_model = next(filter(lambda model: model.name == model_name, mlflow_client.list_registered_models()))
# print("name={}; run_id={}; version={}".format(registered_model.name, registered_model.run_id, registered_model.version))
registered_model

# Load model from model registry in Workspace
from azureml.core.model import Model

model_path = Model.get_model_path('porto-seg-automl-remote-compute', _workspace=ws)
print(model_path)
# fitted_model = joblib.load(model_path)
# print(fitted_model)<jupyter_output>azureml-models/porto-seg-automl-remote-compute/1/model.pkl<jupyter_text>Try model inference with hardcoded input data for the model to predictThis is demonstrated above.  Retrieve the Best ONNX ModelBelow we select the best pipeline from our iterations. The get_output method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing. Overloads on get_output allow you to retrieve the best run and fitted model for any logged metric or for a particular iteration.Set the parameter return_onnx_model=True to retrieve the best ONNX model, instead of the Python model.<jupyter_code># TODO: This should be similar to above, although not sure why ONNX models are not being generated at the moment.
#       (Same class of bug as MLflow models not being saved?)

best_run, onnx_mdl = parent_run.get_output(return_onnx_model=True)<jupyter_output><empty_output><jupyter_text>Save the best ONNX model to local path Predict with the ONNX model, using onnxruntime package<jupyter_code>def get_onnx_res(onnx_resource_json_path):
    with open(onnx_resource_json_path) as f:
        onnx_res = json.load(f)
    return onnx_res


output_path = download_outputs_via_mlflow_client(mlflow_client, mlflow_best_child_run.info.run_id, "outputs")

onnx_model_path = os.path.join(local_dir, "outputs", "model.onnx")

# Loading an ONNX model with MLFlow can be done via.the following, however, we currently don't save the flavor
# information in the MLModel file.
# mlflow.onnx.load_model(onnx_model_path)

# Loading via. OnnxConverter
from azureml.automl.runtime.onnx_convert import OnnxConverter
from azureml.automl.runtime.onnx_convert import OnnxInferenceHelper

onnx_resource_json_path = os.path.join(local_dir, "outputs", "model_onnx.json")
fitted_onnx_model = OnnxConverter.load_onnx_model(onnx_model_path)

mdl_bytes = fitted_onnx_model.SerializeToString()
onnx_res = get_onnx_res(onnx_resource_json_path)

onnxrt_helper = OnnxInferenceHelper(mdl_bytes, onnx_res)
pred_onnx, pred_prob_onnx = onnxrt_helper.predict(df_data)

print(pred_onnx)
print(pred_prob_onnx)<jupyter_output>Directory /tmp/artifact_downloads/05bb01e9-a479-48ba-a0e3-3bcd868dbcd7_1/outputs already exists. Skipping download.
[0 0]
[[0.9730727  0.02692736]
 [0.9730727  0.02692736]]
{'fraction_non_alphanumeric': 0.0985717436858939, 'fraction_numerical': 0.02327120326458586, 'mean_word_length': 4.840418679549114, 'pattern_counts': {'":': 3, '<': 123, '<?xml version=': 0, '>': 133, 'https://': 5, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 76, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '34', 'dir': 'jupyter-structured-clean-dedup', 'id': '185499', 'license': 'MIT', 'n_tokens_mistral': 11870, 'n_tokens_neox': 11287, 'n_words': 2843, 'path': 'notebooks/3-automl-remote-compute-run/V2-automl-remote-compute-run-safe-driver-classifier.ipynb', 'repo_name': 'sjinthehouse/Easy-AutoML-MLOps', 'size': '36276'}