<jupyter_start><jupyter_text>Copyright (c) Microsoft Corporation. All rights reserved.  Licensed under the MIT License.<jupyter_code># Import Workspace
import os
# import dotenv
from azureml.core import Workspace, Datastore
from azureml.exceptions import ProjectSystemException

# _ = dotenv.load_dotenv('.env')<jupyter_output><empty_output><jupyter_text>Create or Import Azure Machine Learning Workspace<jupyter_code># Fetch Workspace Name, Resource Group and Subscription ID from environment variables
# If these aren't defined as Env Vars, then replace the "None" entries below
WORKSPACE_NAME = os.environ.get('AML_WORKSPACE', None)
RESOURCE_GROUP = os.environ.get('AML_RG', None)
SUBSCRIPTION_ID = os.environ.get('AML_SUB_ID', None)
REGION_NAME = os.environ.get('AML_REGION', None)
try:
    ws = Workspace(subscription_id=SUBSCRIPTION_ID,
                   resource_group=RESOURCE_GROUP,
                   workspace_name=WORKSPACE_NAME)
    ws.write_config()
except ProjectSystemException:
    print("Workspace Not Found - please create workspace using code below")
    print()
    print('Workspace Name: \t', WORKSPACE_NAME)
    print('Resource Group: \t', RESOURCE_GROUP)
    print('Subscription ID: \t', SUBSCRIPTION_ID)
# # If the above code fails, uncomment this code block to create a new Workspace
# ws = Workspace.create(subscription_id=SUBSCRIPTION_ID,
#                       resource_group=RESOURCE_GROUP,
#                       name=WORKSPACE_NAME,
#                       location=REGION_NAME)
# ws.write_config()
ws = Workspace.from_config()<jupyter_output><empty_output><jupyter_text>Create CPU ClusterWe can create a CPU-based AML Managed Compute Cluster<jupyter_code>from azureml.core.compute import AmlCompute
cpu_cluster_name = 'cpu-cluster-west'

if ws.compute_targets.get(cpu_cluster_name):
    print(f"'{cpu_cluster_name}' found. Skipping cluster creation")
    ct = ws.compute_targets[cpu_cluster_name]
else:
    print(f"'{cpu_cluster_name}' not found. Creating a new AML Manged Compute cluster.")
    
    cpu_config = AmlCompute.provisioning_configuration(vm_size='Standard_DS13_v2',
                                                       vm_priority='dedicated',
                                                       min_nodes=0,
                                                       max_nodes=5,
                                                       idle_seconds_before_scaledown=360,
                                                       tags={'project_name': "lab-demo"},
                                                       description="An AML Managed compute cluster leveraging DS Series VMs"
                                                      )
    ct = AmlCompute.create(workspace=ws,
                      name=cpu_cluster_name, 
                      provisioning_configuration=cpu_config)
    ct.wait_for_completion(show_output=True)
# # If you would like to see the supported VM sizes, you can get a list with the supported_vmsizes method
# AmlCompute.supported_vmsizes(ws)<jupyter_output><empty_output><jupyter_text>Create GPU Cluster<jupyter_code>gpu_cluster_name = 'gpu-cluster'

if ws.compute_targets.get(gpu_cluster_name):
    print(f"'{gpu_cluster_name}' found. Skipping cluster creation")
    ct = ws.compute_targets[gpu_cluster_name]
else:
    print(f"'{gpu_cluster_name}' not found. Creating a new AML Manged Compute cluster.")
    
    gpu_config = AmlCompute.provisioning_configuration(vm_size='Standard_NC6',
                                                       vm_priority='dedicated',
                                                       min_nodes=0,
                                                       max_nodes=5,
                                                       idle_seconds_before_scaledown=360,
                                                       tags={'project_name': "lab-demo"},
                                                       description="An AML Managed compute cluster leveraging NC Series VMs"
                                                      )
    gpu_ct = AmlCompute.create(workspace=ws,
                      name=gpu_cluster_name, 
                      provisioning_configuration=gpu_config)
    gpu_ct.wait_for_completion(show_output=True)
# Verify clusters are created
ct = ws.compute_targets['cpu-cluster']
cs = ct.get_status()
cs.serialize()<jupyter_output><empty_output><jupyter_text>Attach Data Store<jupyter_code>BLOB_CONTAINER = os.environ.get("BLOB_CONTAINER_NAME", "diabetes")
BLOB_ACCOUNT = os.environ.get("BLOB_ACCT_NAME", "publicmldataeastus")
BLOB_ACCT_KEY = os.environ.get("BLOB_ACCT_KEY", "<KEY>)
datastore_name = 'diabetes'

if ws.datastores.get(datastore_name):
    print(f"'{datastore_name}' datastore found. Skipping registration")
else:
    print(f"'{datastore_name}' datastore not found. Registering with Workspace")
    _ = Datastore.register_azure_blob_container(workspace=ws,
                                                datastore_name='diabetes',
                                                container_name=BLOB_CONTAINER,
                                                account_name=BLOB_ACCOUNT,
                                                account_key=BLOB_ACCT_KEY)
datastore_name = 'hymenoptera'

if ws.datastores.get(datastore_name):
    print(f"'{datastore_name}' datastore found. Skipping registration")
else:
    print(f"'{datastore_name}' datastore not found. Registering with Workspace")
    _ = Datastore.register_azure_blob_container(workspace=ws,
                                                datastore_name='hymenoptera',
                                                container_name='hymenoptera',
                                                account_name=BLOB_ACCOUNT,
                                                account_key=BLOB_ACCT_KEY)
ws.datastores<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07923497267759563, 'fraction_numerical': 0.0023907103825136613, 'mean_word_length': 2.105514316012725, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '230194', 'license': 'MIT', 'n_tokens_mistral': 1594, 'n_tokens_neox': 1542, 'n_words': 308, 'path': 'code/00-Configuration.ipynb', 'repo_name': 'ezwiefel/aml-demo', 'size': '5940'}