<jupyter_start><jupyter_text>preparing folder etc<jupyter_code># Load the workspace from the saved config file
ws = Workspace.from_config()
#Create a tabular dataset from the path on the datastore (this may take a short while)
tab_data_set = Dataset.get_by_name(ws,  'mainMlDataSet', version = 1)
mainPdFrame = tab_data_set.take(100).to_pandas_dataframe()
mainPdFrame
# Create a folder for the pipeline step files
experiment_folder = 'VascProsth_Experiment'
os.makedirs(experiment_folder, exist_ok=True)
# Get the experiment run context
run = Run.get_context()
print(experiment_folder)
#Creating Experiment<jupyter_output><empty_output><jupyter_text>prepare data<jupyter_code>def prepareFrame (df):
    #filling nulls with mean
    df= df.fillna(df.mean())

    # Normalize the numeric columns
    scaler = MinMaxScaler()
    num_cols = ['SuvInFocus','TBR','ageInYearsWhenSurgery']
    df[num_cols] = scaler.fit_transform(df[num_cols])
    #setting categorical columns to boolean  - as here we basically care only about two columns
    df.loc[df['prosthesisType'] == "stentgraft", 'isStentgraft'] = True 
    df.loc[df['simplifiedClassification'] == "Y", 'isY'] = True 

    df.loc[df['prosthesisType'] != "stentgraft", 'isStentgraft'] = False 
    df.loc[df['simplifiedClassification'] != "Y", 'isY'] = False 


    df["isMale"] = df["isMale"].astype(bool)
    return df


mainPdFrame= pd.DataFrame(data= prepareFrame(mainPdFrame))
mainPdFrame<jupyter_output><empty_output><jupyter_text>Gradient boosting with hyperparameter tuning<jupyter_code>%%writefile $experiment_folder/vascProsth_training.py
# this is important as this cell will be written to separate python file
# Import libraries


import argparse, joblib, os
from azureml.core import Run
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.preprocessing import MinMaxScaler

from sklearn.tree import DecisionTreeClassifier




def prepareFrame (df):
    #filling nulls with mean
    df= df.fillna(df.mean())

    # Normalize the numeric columns
    scaler = MinMaxScaler()
    num_cols = ['SuvInFocus','TBR','ageInYearsWhenSurgery']
    df[num_cols] = scaler.fit_transform(df[num_cols])
    #setting categorical columns to boolean  - as here we basically care only about two columns
    df.loc[df['prosthesisType'] == "stentgraft", 'isStentgraft'] = True 
    df.loc[df['simplifiedClassification'] == "Y", 'isY'] = True 

    df.loc[df['prosthesisType'] != "stentgraft", 'isStentgraft'] = False 
    df.loc[df['simplifiedClassification'] != "Y", 'isY'] = False 


    df["isMale"] = df["isMale"].astype(bool)
    return df





# Get the experiment run context
run = Run.get_context()

#learning_rate = 0.1
#n_estimators =100

################ part below to set arguments when this python file will be called

# Get script arguments
parser = argparse.ArgumentParser()

# Input dataset
parser.add_argument("--input-data", type=str, dest='input_data', help='training dataset')

############ Hyperparameters


#from https://medium.datadriveninvestor.com/decision-tree-adventures-2-explanation-of-decision-tree-classifier-parameters-84776f39a28
#max_depth: int or None, optional (default=None) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]


#min_samples_split: int, float, optional (default=2) [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]


#min_impurity_decrease   [0.00005,0.0001,0.0002,0.0005,0.001,0.0015,0.002,0.005,0.01]


parser.add_argument('--max_depth', type=int, dest='max_depth', default=15, help='max_depth')
parser.add_argument('--min_samples_split', type=int, dest='min_samples_split', default=2, help='min_samples_split')
parser.add_argument('--min_impurity_decrease', type=float, dest='min_impurity_decrease', default=0.00005, help='min_impurity_decrease')
parser.add_argument('--isToExplain', type=bool, dest='isToExplain', default=False , help='true if it is a model we want to interpret - the best model')

# Add arguments to args collection
args = parser.parse_args()



mainPdFr = run.input_datasets['training_data'].to_pandas_dataframe() # Get the training data from the estimator input
mainPdFr= prepareFrame(mainPdFr)
#################### preapre data

# Separate features and labels
X, y = mainPdFr[
    ['SuvInFocus','TBR','isStentgraft','isY','ageInYearsWhenSurgery'
     ,'isMale']].values, mainPdFr['isStudy'].values

# Split data into training set and test set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=0)


################ model
model = DecisionTreeClassifier(max_depth =args.max_depth,
                                min_samples_split= args.min_samples_split,
                               min_impurity_decrease= args.min_impurity_decrease
                                                       ).fit(X_train, y_train)


########################## metrics


# calculate accuracy
y_hat = model.predict(X_test)
acc = np.average(y_hat == y_test)
print('Accuracy:', acc)
run.log('Accuracy', np.float(acc))

# calculate AUC
y_scores = model.predict_proba(X_test)
auc = roc_auc_score(y_test,y_scores[:,1])
print('AUC: ' + str(auc))
run.log('AUC', np.float(auc))

######################## Save the model in the run outputs
os.makedirs('outputs', exist_ok=True)
joblib.dump(value=model, filename='outputs/vascProsth_model.pkl')

########## explanations used only in best case
if(args.isToExplain):
    from interpret.ext.blackbox import TabularExplainer
    import os, shutil
    from azureml.interpret import ExplanationClient
    # Get explanation
    explainer = TabularExplainer(model, X_train, features=['SuvInFocus','TBR','isStentgraft','isY','ageInYearsWhenSurgery'
         ,'isMale'], classes=['noInfection', 'infection'])
    explanation = explainer.explain_global(X_test)

    # Get an Explanation Client and upload the explanation
    explain_client = ExplanationClient.from_run(run)
    explain_client.upload_model_explanation(explanation, comment='Tabular Explanation')



run.complete()
############# preparing compute enviroment
from azureml.core.compute import ComputeTarget, AmlCompute
from azureml.core.compute_target import ComputeTargetException

cluster_name = "main"

try:
    # Check for existing compute target
    training_cluster = ComputeTarget(workspace=ws, name=cluster_name)
    print('Found existing cluster, use it.')
except ComputeTargetException:
    # If it doesn't already exist, create it
    try:
        compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2S_V3', max_nodes=2)
        training_cluster = ComputeTarget.create(ws, cluster_name, compute_config)
        training_cluster.wait_for_completion(show_output=True)
    except Exception as ex:
        print(ex)
    
# Create a Python environment for the experiment
sklearn_env = Environment("sklearn-env")

# Ensure the required packages are installed (we need scikit-learn, Azure ML defaults, and Azure ML dataprep)
packages = CondaDependencies.create(conda_packages=['scikit-learn','pip'],
                                    pip_packages=['azureml-defaults','azureml-dataprep[pandas]'])
sklearn_env.python.conda_dependencies = packages

# Get the training dataset
ds = ws.datasets.get("mainMlDataSet")

# Create a script config
script_config = ScriptRunConfig(source_directory=experiment_folder,
                                script='vascProsth_training.py',
                                # Add non-hyperparameter arguments -in this case, the training dataset
                                arguments = ['--input-data', ds.as_named_input('training_data')],
                                environment=sklearn_env,
                                compute_target = training_cluster)



# Sample a range of parameter values
params = BayesianParameterSampling(
    {
        # Hyperdrive will try  combinations, adding these as script arguments
        '--max_depth': choice(1 ,2,3,4,5,6,7,8,9),
        '--min_samples_split' : choice(2,3,4,5,6,7,8,9,10,11,12,13,14,15),
        '--min_impurity_decrease' : choice(0.00005,0.0001,0.0002,0.0005,0.001,0.0015,0.002,0.005,0.01)
    }
)

# Configure hyperdrive settings
hyperdrive = HyperDriveConfig(run_config=script_config, 
                          hyperparameter_sampling=params, 
                          policy=None, # No early stopping policy
                          primary_metric_name='AUC', # Find the highest AUC metric
                          primary_metric_goal=PrimaryMetricGoal.MAXIMIZE, 
                          max_total_runs=50, # Restict the experiment to 200 iterations
                          max_concurrent_runs=2) # Run up to 2 iterations in parallel

# Run the experiment
experiment = Experiment(workspace=ws, name='mslearn-vascProsth-hyperdrive')
run = experiment.submit(config=hyperdrive)

# Show the status in the notebook as the experiment runs
RunDetails(run).show()
run.wait_for_completion()

# Print all child runs, sorted by the primary metric
# Get the experiment run context

for child_run in run.get_children_sorted_by_primary_metric():
    print(child_run)

# Get the best run, and its metrics and arguments
best_run = run.get_best_run_by_primary_metric()
best_run_metrics = best_run.get_metrics()
script_arguments = best_run.get_details() ['runDefinition']['arguments']
print('Best Run Id: ', best_run.id)
print(' -AUC:', best_run_metrics['AUC'])
print(' -Accuracy:', best_run_metrics['Accuracy'])
print(' -Arguments:',script_arguments)<jupyter_output><empty_output><jupyter_text>Model Interpreting<jupyter_code>from decimal import Decimal
bestRunDetails = best_run.get_details() ['runDefinition']['arguments']

max_depth = bestRunDetails[3]
min_samples_split = bestRunDetails[5]
min_impurity_decrease = bestRunDetails[7]


#################### preapre data

# Separate features and labels
X, y = mainPdFrame[
    ['SuvInFocus','TBR','isStentgraft','isY','ageInYearsWhenSurgery'
     ,'isMale']].values, mainPdFrame['isStudy'].values

# Split data into training set and test set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=0)


################ model
modelToInterpret = DecisionTreeClassifier(max_depth =int(max_depth),
                                min_samples_split= int(min_samples_split),
                               min_impurity_decrease= Decimal(min_impurity_decrease)
                                                       ).fit(X_train, y_train)
modelToInterpret
from sklearn import tree

tree.plot_tree(modelToInterpret) 


# Get explanation
explainer = TabularExplainer(modelToInterpret, X_train, features=['SuvInFocus','TBR','isStentgraft','isY','ageInYearsWhenSurgery'
     ,'isMale'], classes=['noInfection', 'infection'])
explanation = explainer.explain_global(X_test)

#Get an Explanation Client and upload the explanation
explain_client = ExplanationClient.from_run(run)
explain_client.upload_model_explanation(explanation, comment='Tabular Explanation')
explain_client.list_model_explanations()
# Create a Python environment for the experiment
explain_env = Environment("explain-env")

# Create a set of package dependencies (including the azureml-interpret package)
packages = CondaDependencies.create(conda_packages=['scikit-learn','pandas','pip'],
                                    pip_packages=['azureml-defaults','azureml-interpret'])
explain_env.python.conda_dependencies = packages


ds = ws.datasets.get("mainMlDataSet")

max_depth = bestRunDetails[3]
min_samples_split = bestRunDetails[5]
min_impurity_decrease = bestRunDetails[7]

# Create a script config
script_config = ScriptRunConfig(source_directory=experiment_folder,
                      script='vascProsth_training.py'
                      ,arguments=['--input-data', ds.as_named_input('training_data')
                                 ,'--max_depth',max_depth
                                 ,'--min_samples_split',min_samples_split
                                 ,  '--min_impurity_decrease',min_impurity_decrease
                                  ,'--isToExplain',True
                                 ],
                       environment=explain_env) 

# submit the experiment
experiment_name = 'mslearn-vascProsth-explain'
experiment = Experiment(workspace=ws, name=experiment_name)
run = experiment.submit(config=script_config)
RunDetails(run).show()
run.wait_for_completion()
# Get the feature explanations
client = ExplanationClient.from_run(run)
engineered_explanations = client.download_model_explanation()
feature_importances = engineered_explanations.get_feature_importance_dict()

# Overall feature importance
print('Feature\tImportance')
for key, value in feature_importances.items():
    print(key, '\t', value)<jupyter_output>Feature	Importance
TBR 	 0.40010683760683763
SuvInFocus 	 0.08764245014245015
isMale 	 0.0
ageInYearsWhenSurgery 	 0.0
isY 	 0.0
isStentgraft 	 0.0
{'fraction_non_alphanumeric': 0.1174468085106383, 'fraction_numerical': 0.01941972920696325, 'mean_word_length': 4.220516962843296, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '247476', 'license': 'Apache-2.0', 'n_tokens_mistral': 4268, 'n_tokens_neox': 3975, 'n_words': 852, 'path': 'AzureMl/mlB (1) (2).ipynb', 'repo_name': 'jakubMitura14/Azure-Medicine-Data-Engineering', 'size': '12925'}