<jupyter_start><jupyter_text>Random layer 1<jupyter_code>model = load_model('YOUR_MODEL')
for layer in model.layers: 
  if hasattr(layer, 'kernel_initializer'):
    #print(layer.name, layer)
    if layer.name == 'dense_1':
      layer.kernel.initializer.run(session=session)
y_pred = model.predict(features2, verbose=1)

# Create a new dataframe with entries for each element of the test set
# Include an ID, diagnosis, and % likelihoods for each diagnosis from the model
df = pd.DataFrame(columns=['ID', 'Dx', '% Mel', '% Nev'],index=[i for i in range(400)])
df['ID'] = df.index

# Create dictionaries to contain actual diagnosis and probabilities from the model
dx_d = {}
Pmel = {}
Pnev = {}
# Take the actual diagnoses from where we retrieved them earlier
y_test_cat = target_cat

# For each element in the test set:
for ind in range(400):
    # Append the diagnosis and predictions to their respective dictionaries
    if y_test_cat[ind][0] == 1.0:
        diagnosis = 'Melanoma'
    elif y_test_cat[ind][1] == 1.0:
        diagnosis = 'Nevus'
    dx_d[ind] = diagnosis
    Pmel[ind] = y_pred[ind][0]
    Pnev[ind] = y_pred[ind][1]
    
# Take the above dictionaries and insert them into the data frame
df['Dx'] = df['ID'].map(dx_d)
df['% Mel'] = df['ID'].map(Pmel)
df['% Nev'] = df['ID'].map(Pnev)

# Change the prediction likelihoods to be floats 
df = df.astype({"% Mel": float, "% Nev": float})
df = df.iloc[id_list]

# Print the first 5 entries in the data frame
print('Unseen set') 
print(df.head())
df.to_csv('scram_layer_1.csv')
#Model has been scrambled
#Regnerate shap / grad  with these images 
from vis.utils import utils
from keras import layers, activations
import copy
#Assorted modifications for model compatibility with gradCAM
gmodel = copy.deepcopy(model)
layer_idx = utils.find_layer_idx(gmodel,'dense_2')

#swap output activation with linear classifier for the reasons mentioned above
gmodel.layers[layer_idx].activation = activations.linear
gmodel = utils.apply_modifications(gmodel)
batch_shap2(features, id_list, model, gmodel, 'Explanations_TEST_Scramble_1/')
del(gmodel)<jupyter_output><empty_output><jupyter_text>Random layer 2<jupyter_code>for layer in model.layers: 
  if hasattr(layer, 'kernel_initializer'):
    #print(layer.name, layer)
    if layer.name == 'conv2d_94':
      layer.kernel.initializer.run(session=session)
y_pred = model.predict(features2, verbose=1)

# Create a new dataframe with entries for each element of the test set
# Include an ID, diagnosis, and % likelihoods for each diagnosis from the model
df = pd.DataFrame(columns=['ID', 'Dx', '% Mel', '% Nev'],index=[i for i in range(400)])
df['ID'] = df.index

# Create dictionaries to contain actual diagnosis and probabilities from the model
dx_d = {}
Pmel = {}
Pnev = {}
# Take the actual diagnoses from where we retrieved them earlier
y_test_cat = target_cat

# For each element in the test set:
for ind in range(400):
    # Append the diagnosis and predictions to their respective dictionaries
    if y_test_cat[ind][0] == 1.0:
        diagnosis = 'Melanoma'
    elif y_test_cat[ind][1] == 1.0:
        diagnosis = 'Nevus'
    dx_d[ind] = diagnosis
    Pmel[ind] = y_pred[ind][0]
    Pnev[ind] = y_pred[ind][1]
    
# Take the above dictionaries and insert them into the data frame
df['Dx'] = df['ID'].map(dx_d)
df['% Mel'] = df['ID'].map(Pmel)
df['% Nev'] = df['ID'].map(Pnev)

# Change the prediction likelihoods to be floats 
df = df.astype({"% Mel": float, "% Nev": float})
df = df.iloc[id_list]

# Print the first 5 entries in the data frame
print('Unseen set') 
print(df.head())
df.to_csv('scram_layer_2.csv')
#Model has been scrambled
#Regnerate shap / grad  with these images 
from vis.utils import utils
from keras import layers, activations
import copy
#Assorted modifications for model compatibility with gradCAM
gmodel = copy.deepcopy(model)
layer_idx = utils.find_layer_idx(gmodel,'dense_2')

#swap with softmax with linear classifier for the reasons mentioned above
gmodel.layers[layer_idx].activation = activations.linear
gmodel = utils.apply_modifications(gmodel)
batch_shap2(features, id_list, model, gmodel, 'Explanations_TEST_Scramble_2/')
del(gmodel)<jupyter_output><empty_output><jupyter_text>Random layer 3<jupyter_code>for layer in model.layers: 
  if hasattr(layer, 'kernel_initializer'):
    #print(layer.name, layer)
    if layer.name == 'conv2d_77':
      layer.kernel.initializer.run(session=session)
y_pred = model.predict(features2, verbose=1)

# Create a new dataframe with entries for each element of the test set
# Include an ID, diagnosis, and % likelihoods for each diagnosis from the model
df = pd.DataFrame(columns=['ID', 'Dx', '% Mel', '% Nev'],index=[i for i in range(400)])
df['ID'] = df.index

# Create dictionaries to contain actual diagnosis and probabilities from the model
dx_d = {}
Pmel = {}
Pnev = {}
# Take the actual diagnoses from where we retrieved them earlier
y_test_cat = target_cat

# For each element in the test set:
for ind in range(400):
    # Append the diagnosis and predictions to their respective dictionaries
    if y_test_cat[ind][0] == 1.0:
        diagnosis = 'Melanoma'
    elif y_test_cat[ind][1] == 1.0:
        diagnosis = 'Nevus'
    dx_d[ind] = diagnosis
    Pmel[ind] = y_pred[ind][0]
    Pnev[ind] = y_pred[ind][1]
    
# Take the above dictionaries and insert them into the data frame
df['Dx'] = df['ID'].map(dx_d)
df['% Mel'] = df['ID'].map(Pmel)
df['% Nev'] = df['ID'].map(Pnev)

# Change the prediction likelihoods to be floats 
df = df.astype({"% Mel": float, "% Nev": float})
df = df.iloc[id_list]

# Print the first 5 entries in the data frame
print('Unseen set') 
print(df.head())
df.to_csv('scram_layer_3.csv')
#Model has been scrambled
#Regnerate shap / grad  with these images 
from vis.utils import utils
from keras import layers, activations
import copy
#Assorted modifications for model compatibility with gradCAM
gmodel = copy.deepcopy(model)
layer_idx = utils.find_layer_idx(gmodel,'dense_2')

#swap with softmax with linear classifier for the reasons mentioned above
gmodel.layers[layer_idx].activation = activations.linear
gmodel = utils.apply_modifications(gmodel)
batch_shap2(features, id_list, model, gmodel, 'Explanations_TEST_Scramble_3/')
del(gmodel)<jupyter_output><empty_output><jupyter_text>Random layer 4<jupyter_code>for layer in model.layers: 
  if hasattr(layer, 'kernel_initializer'):
    #print(layer.name, layer)
    if layer.name == 'conv2d_65':
      layer.kernel.initializer.run(session=session)
y_pred = model.predict(features2, verbose=1)

# Create a new dataframe with entries for each element of the test set
# Include an ID, diagnosis, and % likelihoods for each diagnosis from the model
df = pd.DataFrame(columns=['ID', 'Dx', '% Mel', '% Nev'],index=[i for i in range(400)])
df['ID'] = df.index

# Create dictionaries to contain actual diagnosis and probabilities from the model
dx_d = {}
Pmel = {}
Pnev = {}
# Take the actual diagnoses from where we retrieved them earlier
y_test_cat = target_cat

# For each element in the test set:
for ind in range(400):
    # Append the diagnosis and predictions to their respective dictionaries
    if y_test_cat[ind][0] == 1.0:
        diagnosis = 'Melanoma'
    elif y_test_cat[ind][1] == 1.0:
        diagnosis = 'Nevus'
    dx_d[ind] = diagnosis
    Pmel[ind] = y_pred[ind][0]
    Pnev[ind] = y_pred[ind][1]
    
# Take the above dictionaries and insert them into the data frame
df['Dx'] = df['ID'].map(dx_d)
df['% Mel'] = df['ID'].map(Pmel)
df['% Nev'] = df['ID'].map(Pnev)

# Change the prediction likelihoods to be floats 
df = df.astype({"% Mel": float, "% Nev": float})
df = df.iloc[id_list]

# Print the first 5 entries in the data frame
print('Unseen set') 
print(df.head())
df.to_csv('scram_layer_4.csv')
#Model has been scrambled
#Regnerate shap / grad  with these images 
from vis.utils import utils
from keras import layers, activations
import copy
#Assorted modifications for model compatibility with gradCAM
gmodel = copy.deepcopy(model)
layer_idx = utils.find_layer_idx(gmodel,'dense_2')

#swap with softmax with linear classifier for the reasons mentioned above
gmodel.layers[layer_idx].activation = activations.linear
gmodel = utils.apply_modifications(gmodel)
batch_shap2(features, id_list, model, gmodel, 'Explanations_TEST_Scramble_4/')
del(gmodel)<jupyter_output><empty_output><jupyter_text>Random layer 5<jupyter_code>for layer in model.layers: 
  if hasattr(layer, 'kernel_initializer'):
    #print(layer.name, layer)
    if layer.name == 'conv2d_51':
      layer.kernel.initializer.run(session=session)
y_pred = model.predict(features2, verbose=1)

# Create a new dataframe with entries for each element of the test set
# Include an ID, diagnosis, and % likelihoods for each diagnosis from the model
df = pd.DataFrame(columns=['ID', 'Dx', '% Mel', '% Nev'],index=[i for i in range(400)])
df['ID'] = df.index

# Create dictionaries to contain actual diagnosis and probabilities from the model
dx_d = {}
Pmel = {}
Pnev = {}
# Take the actual diagnoses from where we retrieved them earlier
y_test_cat = target_cat

# For each element in the test set:
for ind in range(400):
    # Append the diagnosis and predictions to their respective dictionaries
    if y_test_cat[ind][0] == 1.0:
        diagnosis = 'Melanoma'
    elif y_test_cat[ind][1] == 1.0:
        diagnosis = 'Nevus'
    dx_d[ind] = diagnosis
    Pmel[ind] = y_pred[ind][0]
    Pnev[ind] = y_pred[ind][1]
    
# Take the above dictionaries and insert them into the data frame
df['Dx'] = df['ID'].map(dx_d)
df['% Mel'] = df['ID'].map(Pmel)
df['% Nev'] = df['ID'].map(Pnev)

# Change the prediction likelihoods to be floats 
df = df.astype({"% Mel": float, "% Nev": float})
df = df.iloc[id_list]

# Print the first 5 entries in the data frame
print('Unseen set') 
print(df.head())
df.to_csv('scram_layer_5.csv')
#Model has been scrambled
#Regnerate shap / grad  with these images 
from vis.utils import utils
from keras import layers, activations
import copy
#Assorted modifications for model compatibility with gradCAM
gmodel = copy.deepcopy(model)
layer_idx = utils.find_layer_idx(gmodel,'dense_2')

#swap with softmax with linear classifier for the reasons mentioned above
gmodel.layers[layer_idx].activation = activations.linear
gmodel = utils.apply_modifications(gmodel)
batch_shap2(features, id_list, model, gmodel, 'Explanations_TEST_Scramble_5/')
del(gmodel)<jupyter_output><empty_output><jupyter_text>Random layer 6<jupyter_code>for layer in model.layers: 
  if hasattr(layer, 'kernel_initializer'):
    #print(layer.name, layer)
    if layer.name == 'conv2d_1':
      layer.kernel.initializer.run(session=session)
y_pred = model.predict(features2, verbose=1)

# Create a new dataframe with entries for each element of the test set
# Include an ID, diagnosis, and % likelihoods for each diagnosis from the model
df = pd.DataFrame(columns=['ID', 'Dx', '% Mel', '% Nev'],index=[i for i in range(400)])
df['ID'] = df.index

# Create dictionaries to contain actual diagnosis and probabilities from the model
dx_d = {}
Pmel = {}
Pnev = {}
# Take the actual diagnoses from where we retrieved them earlier
y_test_cat = target_cat

# For each element in the test set:
for ind in range(400):
    # Append the diagnosis and predictions to their respective dictionaries
    if y_test_cat[ind][0] == 1.0:
        diagnosis = 'Melanoma'
    elif y_test_cat[ind][1] == 1.0:
        diagnosis = 'Nevus'
    dx_d[ind] = diagnosis
    Pmel[ind] = y_pred[ind][0]
    Pnev[ind] = y_pred[ind][1]
    
# Take the above dictionaries and insert them into the data frame
df['Dx'] = df['ID'].map(dx_d)
df['% Mel'] = df['ID'].map(Pmel)
df['% Nev'] = df['ID'].map(Pnev)

# Change the prediction likelihoods to be floats 
df = df.astype({"% Mel": float, "% Nev": float})
df = df.iloc[id_list]

# Print the first 5 entries in the data frame
print('Unseen set') 
print(df.head())
df.to_csv('scram_layer_6.csv')
#Model has been scrambled
#Regnerate shap / grad  with these images 
from vis.utils import utils
from keras import layers, activations
import copy
#Assorted modifications for model compatibility with gradCAM
gmodel = copy.deepcopy(model)
layer_idx = utils.find_layer_idx(gmodel,'dense_2')

#swap with softmax with linear classifier for the reasons mentioned above
gmodel.layers[layer_idx].activation = activations.linear
gmodel = utils.apply_modifications(gmodel)
batch_shap2(features, id_list, model, gmodel, 'Explanations_TEST_Scramble_6/')
del(gmodel)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11889172788246567, 'fraction_numerical': 0.011737464068987544, 'mean_word_length': 4.852803738317757, 'pattern_counts': {'":': 12, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '639770', 'license': 'MIT', 'n_tokens_mistral': 4267, 'n_tokens_neox': 4146, 'n_words': 1293, 'path': 'Shap_GradCAM_Notebooks/RandomLayersShapGradCAM.ipynb', 'repo_name': 'yelk314/DNN-or-Dermatologist---Supporting-Code', 'size': '12524'}