<jupyter_start><jupyter_text>Simulate CLI Auth from MLops<jupyter_code>import repackage
repackage.add("../../azure-enterprise-scale-ml/esml/common/")
import azureml.core
from azureml.core.authentication import InteractiveLoginAuthentication
from esml import ESMLProject
print("SDK Version:", azureml.core.VERSION)

def init_ws():
    p = ESMLProject() # self-aware about its config sources

    # WRITE ws config - simulate CLI auth with Interactive
    cli_auth = InteractiveLoginAuthentication(tenant_id = p.tenant)
    ws, config_name = p.authenticate_workspace_and_write_config(cli_auth)
    # TEST read
    ws = p.get_workspace_from_config(cli_auth) # Authenticate to DEV
    p.ws = ws
    return p
    
p = init_ws()
p.describe()<jupyter_output>SDK Version: 1.26.0
Using lake_settings.json with ESML version 1.4 - Models array support including LABEL<jupyter_text>10<jupyter_code>"""
Copyright (C) Microsoft Corporation. All rights reserved.​
 ​
Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual,
royalty-free right to use, copy, and modify the software code provided by us
("Software Code"). You may not sublicense the Software Code or any use of it
(except to your affiliates and to vendors to perform work on your behalf)
through distribution, network access, service agreement, lease, rental, or
otherwise. This license does not purport to express any claim of ownership over
data you may have shared with Microsoft in the creation of the Software Code.
Unless applicable law gives you more rights, Microsoft reserves all other
rights not expressly granted herein, whether by implication, estoppel or
otherwise. ​
 ​
THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
#import repackage
#repackage.add("../../azure-enterprise-scale-ml/esml/common/")
#import azureml.core
#from azureml.core.authentication import AzureCliAuthentication
#from esml import ESMLProject
#print("SDK Version:", azureml.core.VERSION)

#p = ESMLProject.get_project_from_env_command_line() # Alt A)
#print("ESML environment (dev, test or prod): {}".format(p.dev_test_prod))
#p.describe()
#cli_auth = AzureCliAuthentication()
#ws, config_name = p.authenticate_workspace_and_write_config(cli_auth) 

p = init_ws() # Simulates the "commented code above"-to authentica and create ESMLProject
p.inference_mode = False # We want "TRAIN" mode
ws = p.ws
print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep="\n")<jupyter_output>Using lake_settings.json with ESML version 1.4 - Models array support including LABEL
msft-weu-DEV-eap-proj02_ai-amls
MSFT-WEU-EAP_PROJECT02_AI-DEV-RG
westeurope
ca0a8c40-b06a-4e4e-8434-63c03a1dee34<jupyter_text>21 Bronze 2 Gold<jupyter_code>p = init_ws()
p.inference_mode = False # We want "TRAIN" mode
p.init(p.ws) # Automapping from datalake to Azure ML datasets, prints status


import repackage
repackage.add("../../azure-enterprise-scale-ml/esml/common/")
from esml import ESMLProject
from baselayer_azure_ml_pipeline import ESMLPipelineFactory, esml_pipeline_types

#p = ESMLProject()
p.inference_mode = False
p.active_model = 12 # Y=11, price=12
p_factory = ESMLPipelineFactory(p)

scoring_date = '2021-01-01 10:35:01.243860'
p_factory.batch_pipeline_parameters[1].default_value = scoring_date # overrides ESMLProject.date_scoring_folder.

p_factory.describe()<jupyter_output>Using lake_settings.json with ESML version 1.4 - Models array support including LABEL
Using GEN2 as Datastore
Searching for setting in ESML datalake...
ESML in-folder settings override = TRUE 
 - Found settings in the ESML AutoLake  [active_in_folder.json,active_scoring_in_folder.json], to override ArgParse/GIT config with.
 - TRAIN in date:  2021/01/01
 - INFERENCE in date: 2021/06/08 and ModelVersion to score with: 1 (0=latest)

Inference mode (False = Training mode): False
Load data as Datasets....
ds01_diabetes
ds02_other

####### Automap & Autoregister - SUCCESS!
1) Auto mapped 2 ESML Dataset with registered Azure ML Datasets (potentially all 3: IN,BRONZE, SILVER) in Datastore project002 

Dataset 'ds01_diabetes' status:
 - IN_Folder_has_files
 - BRONZE_Folder_has_files
 - SILVER_Folder_has_files
Dataset 'ds02_other' status:
 - IN_Folder_has_files
 - BRONZE_Folder_has_files
 - SILVER_Folder_has_files

2) Registered each Dataset with suffixes (_IN, _BRONZE, _SILVER) 
 Tip: Use ESML[...]<jupyter_text>`IN_2_GOLD` or `IN_2_GOLD_TRAIN`
- This since `p.inference_mode=False`
    - Otherwise if we want a SCORNING Pipeline, we flip this flag `p.inference_mode=TRUE`, and create `IN_2_GOLD_SCORING`<jupyter_code>print(p.inference_mode)
## CREATE - IN_2_GOLD
p_factory.create_dataset_scripts_from_template(overwrite_if_exists=False) 
batch_pipeline = p_factory.create_batch_pipeline(esml_pipeline_types.IN_2_GOLD) # Prepare data for training: Either with AutoMLStep or ManualMLStep

## RUN pipeline
pipeline_run = p_factory.execute_pipeline(batch_pipeline)
pipeline_run.wait_for_completion(show_output=False)<jupyter_output>Did NOT overwrite script-files with template-files such as 'scoring_gold.py', since overwrite_if_exists=False
Using GEN2 as Datastore
Searching for setting in ESML datalake...
ESML in-folder settings override = FALSE. [active_in_folder.json,active_scoring_in_folder.json] not found in LAKE. 
 - Using [active_in_folder.json,active_scoring_in_folder.json] from ArgParse or GIT. No override from datalake settings
Path for active folder (where no files exists):
AdlsGen2-ListFiles (req=1, existingItems=0)' for 'https://msftweudevcmnai2.dfs.core.windows.net/lake3?directory=projects/project002/12_car_price_regression/train/active&recursive=true&resource=filesystem' on storage failed with status code 'NotFound' (The specified path does not exist.), client request ID '546c816e-a842-4072-bbaf-30e145da135f', request ID '22cc8ab0-f01f-0023-2e54-c5fa13000000
Note: OVERRIDING enterprise performance settings with project specifics. (to change, set flag in 'dev_test_prod_settings.json' -> override_enter[...]<jupyter_text>22 - TRAIN step<jupyter_code>from azureml.train.automl import AutoMLConfig
from esml import ESMLDataset, ESMLProject
from baselayer_azure_ml import AutoMLFactory,azure_metric_regression,azure_metric_classification

p = init_ws()
p.inference_mode = False # We want "TRAIN" mode
p.init(ws) # Automapping from datalake to Azure ML datasets, prints status

# TRAIN MODEL
automl_performance_config = p.get_automl_performance_config() # 1)Get config
aml_compute = p.get_training_aml_compute(ws) # 2)Get compute, for active environment

label = p.active_model["label"]
train_6, validate_set_2, test_set_2 = p.split_gold_3(0.6) # 3) Auto-registerin AZURE (M03_GOLD_TRAIN | M03_GOLD_VALIDATE | M03_GOLD_TEST)          # Alt: p.Gold.random_split(percentage=0.8, seed=23)

automl_config = AutoMLConfig(task = 'regression', # 4) Override the ENV config, for model(that inhertits from enterprise DEV_TEST_PROD config baseline)
                            primary_metric = azure_metric_regression.MAE, # # Note: Regression(MAPE) are not possible in AutoML
                            compute_target = aml_compute,
                            training_data = p.GoldTrain, # is 'train_6' pandas dataframe, but as an Azure ML Dataset
                            experiment_exit_score = '0.9', # DEMO purpose
                            label_column_name = label,
                            **automl_performance_config
                        )
via_pipeline = False
# Consistent/same return values from both AutoML ALTERNATIVES (run or pipeline)
best_run, fitted_model, experiment = AutoMLFactory(p).train_pipeline(automl_config) if via_pipeline else AutoMLFactory(p).train_as_run(automl_config)
print(esml_pipeline_types.IN_2_GOLD)<jupyter_output>IN_2_GOLD<jupyter_text>23 - compare scoring<jupyter_code>from baselayer_azure_ml import AutoMLFactory
p = init_ws()
p.inference_mode = False # We want "TRAIN" mode

# COMPARE MODEL SCORING - Promote?
target_env = "dev" # If not set or used, it will automatically test for NEXT environment
print("Example: If new model scores better in DEV, we can promote this to TEST")

promote, m1_name, r1_id, m2_name, r2_run_id = AutoMLFactory(p).compare_scoring_current_vs_new_model(target_env)

print("New Model: {} in environment {}".format(m1_name, p.dev_test_prod))
print("Existing Model: {} in environment {}".format(m2_name,target_env))

if (promote and p.dev_test_prod == target_env):# Can only register a model in same workspace (test->test) - need to retrain if going from dev->test
    AutoMLFactory(p).register_active_model(target_env)<jupyter_output>Example: If new model scores better in DEV, we can promote this to TEST
Loading AutoML config settings from: dev
targe=source environement. Compare model version in DEV/TEST/PROD with latest registered in same DEV/TEST/PROD workspace (same workspace & subscriptiom comparison)

WARNING:root:The version of the SDK does not match the version the model was trained on.
WARNING:root:The consistency in the result may not be guaranteed.
WARNING:root:Package:azureml-automl-core, training version:1.30.0, current version:1.26.0
Package:azureml-automl-runtime, training version:1.30.0, current version:1.26.0
Package:azureml-core, training version:1.30.0, current version:1.26.0
Package:azureml-dataprep, training version:2.15.1, current version:2.13.2
Package:azureml-dataprep-native, training version:33.0.0, current version:32.0.0
Package:azureml-dataprep-rslex, training version:1.13.0, current version:1.11.2
Package:azureml-dataset-runtime, training version:1.30.0, current version:1.26.0
Package:azu[...]<jupyter_text>31 - deploy to DEV<jupyter_code>p = init_ws()
#p.init(ws) # Automapping from datalake to Azure ML datasets, prints status

print("Environment:")
print(p.dev_test_prod,p.ws.name)

# DEPLOY!
inference_config, model, best_run = p.get_active_model_inference_config(p.ws) #  AutoML support 
service,api_uri, kv_aks_api_secret= p.deploy_automl_model_to_aks(model,inference_config)<jupyter_output>Environment:
dev msft-weu-DEV-eap-proj02_ai-amls
Note: OVERRIDING enterprise performance settings with project specifics. (to change, set flag in 'dev_test_prod_settings.json' -> override_enterprise_settings_with_model_specific=False)
Loading AutoML config settings from: dev
Loading AutoML config settings from: dev
WARNING:root:The version of the SDK does not match the version the model was trained on.
WARNING:root:The consistency in the result may not be guaranteed.
WARNING:root:Package:azureml-automl-core, training version:1.30.0, current version:1.26.0
Package:azureml-automl-runtime, training version:1.30.0, current version:1.26.0
Package:azureml-core, training version:1.30.0, current version:1.26.0
Package:azureml-dataprep, training version:2.15.1, current version:2.13.2
Package:azureml-dataprep-native, training version:33.0.0, current version:32.0.0
Package:azureml-dataprep-rslex, training version:1.13.0, current version:1.11.2
Package:azureml-dataset-runtime, training version:1.3[...]<jupyter_text>31 - TEST DEV webservice<jupyter_code>p = init_ws()
p.inference_mode = True 
p.connect_to_lake() # To be able to also SAVE scoring autmatically

# Get TEST-data
X_test, y_test, tags = p.get_gold_validate_Xy() # Get  X_test data, ESML knowsSPLIT and LABEL already 
print(tags)

df = p.call_webservice(p.ws, X_test,False) # Auto-fetch key from keyvault, and calls the webservice
df.head()<jupyter_output>Using GEN2 as Datastore
M03_GOLD_VALIDATE : (37, 11)
X_test  (37, 10)
y_test  (37,)
{'split_percentage': '0.2', 'label': 'Y', 'model': '03_diabetes_model_reg'}
Note: OVERRIDING enterprise performance settings with project specifics. (to change, set flag in 'dev_test_prod_settings.json' -> override_enterprise_settings_with_model_specific=False)
Note: Fetching keys automatically via workspace keyvault.
Saving scoring to lake for project folder project002 and inference_model_version: 6 ...
...
..

Saved DATA to score successfully in LAKE, as file 'to_score_False.parquet'
..
Saved SCORED data in LAKE, as file 'scored_False.parquet'<jupyter_text>32 - DEPLOY TEST model to TEST webservice (via DEV ws)<jupyter_code>p = init_ws() # Authenticate to CLI workspace
p.dev_test_prod = "test" # This should be read from config  "GIT-branch-TEST" then ABOVE get_other_workspace is not needed (but not since only 1 branch-MASTER, we can easily switch)
test_ws = p.get_other_workspace(p.dev_test_prod) # Get TEST via DEV workspace/keyvault 6 SP
p.init(test_ws) # Automapping from datalake to Azure ML datasets, prints status)

# DEPLOY! 
inference_config, model, best_run = p.get_active_model_inference_config(test_ws)  #   Get from TEST (Roadmap: Get model from DEV workspace)
print("Found model: {} version {} in TEST".format(model.name, model.version))
service,api_uri, kv_aks_api_secret= p.deploy_automl_model_to_aks(model,inference_config) # Deploy to TEST model to TEST AKS<jupyter_output><empty_output><jupyter_text>32 - SCORE TEST webservice (via 1st authenticaion/DEVws)<jupyter_code>p = init_ws() # Authenticate via CLI to a workspace (DEV workspace in this case)
p.inference_mode = True

# GET TEST workspace, PROJECT are already authenticated via "a" workspace (DEV, TEST or PROD)
test_ws = p.get_other_workspace("test") # Get TEST via DEV workspace/keyvault 6 SP
p.dev_test_prod = "test"
p.init(test_ws) # Automapping from datalake to Azure ML datasets, prints status)


# TEST webservice!
X_test, y_test, tags = p.get_gold_validate_Xy("Y") # Get the X_test data, ESML knows the SPLIT and LABEL already (due to training)
print(tags)

caller_user_id = '81965d9c-40ca-4e47-9723-5a608a32a0e4' # Connect the scoring to a caller/user, globally for all rows
df = p.call_webservice(test_ws, X_test,caller_user_id,False) # Auto-fetch key from keyvault, and calls the webservice
df.head()<jupyter_output><empty_output><jupyter_text>TODO - in Roadmap  32 (TODO) - DEPLOY DEV model to TEST webservice (via DEV ws)<jupyter_code>p = init_ws() # Authenticate via CLI to a workspace (DEV workspace in this case)

p.dev_test_prod = "test" # This should be read from config  "GIT-branch-TEST" then ABOVE get_other_workspace is not needed (but not since only 1 branch-MASTER, we can easily switch)
test_ws = p.get_other_workspace(p.dev_test_prod) # Get TEST via DEV workspace/keyvault 6 SP
p.init(test_ws) # Automapping from datalake to Azure ML datasets, prints status)

# DEPLOY! 

# A DEV to TEST (There is no model with id AutoMLfc209440a6:8 in msft-weu-TEST-eap-proj02_ai-amls)
p.dev_test_prod = "dev"
print("Active workspace: (before fetch model) ", p.ws.name)

inference_config, model, best_run = p.get_active_model_inference_config(p.ws)  #   Get from DEV
print("Found model: {} version {} in DEV".format(model.name, model.version))

print("Active workspace (before deploy): ", p.ws.name)
p.dev_test_prod = "test"
service,api_uri, kv_aks_api_secret= p.deploy_automl_model_to_aks(model,inference_config) # Deploy DEV model to TEST AKS...does this work?

# B) TEST to TEST
#p.dev_test_prod = "test"
#inference_config, model, best_run = p.get_active_model_inference_config(test_ws)  #   Get from TEST (Roadmap: Get model from DEV workspace)
#print("Found model: {} version {} in TEST".format(model.name, model.version))
#service,api_uri, kv_aks_api_secret= p.deploy_automl_model_to_aks(model,inference_config) # Deploy to TEST model to TEST AKS<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08902922238732046, 'fraction_numerical': 0.022969291728578506, 'mean_word_length': 5.217474980754426, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 42, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '532552', 'license': 'MIT', 'n_tokens_mistral': 5629, 'n_tokens_neox': 5292, 'n_words': 1531, 'path': 'copy_my_subfolders_to_my_grandparent/mlops/demo_project02_m12_remote/01-esml-test-steps.ipynb', 'repo_name': 'jostrm/azure-enterprise-scale-ml', 'size': '16152'}