<jupyter_start><jupyter_text>Artificial Dermatologist Sagemaker model---This notebook is for the execution of the model 1) Dowload Data---run this section only once<jupyter_code>%mkdir data
!wget -O train.zip https://s3-us-west-1.amazonaws.com/udacity-dlnfd/datasets/skin-cancer/train.zip
!unzip train.zip -d data
!rm train.zip
!wget -O test.zip https://s3-us-west-1.amazonaws.com/udacity-dlnfd/datasets/skin-cancer/test.zip
!unzip test.zip -d data
!rm test.zip
!wget -O valid.zip https://s3-us-west-1.amazonaws.com/udacity-dlnfd/datasets/skin-cancer/valid.zip
!unzip valid.zip -d data
!rm valid.zip<jupyter_output><empty_output><jupyter_text>2) Load Data and Preprocess data---<jupyter_code>data_dir = 'data/'
train_dir = 'train/'
test_dir = 'test/'
valid_dir = 'valid/'
classes = ['melanoma', 'nevus', 'seborrheic_keratosis']

import pickle as pkl
import os
import matplotlib.pyplot as plt
import numpy as np
import torch
from torchvision import models, transforms
from torchvision import datasets
from torchvision import transforms
from torch.utils.data import DataLoader

%matplotlib inline
train_on_gpu = torch.cuda.is_available()

if not train_on_gpu:
    print('CUDA is not available.  Training on CPU ...')
else:
    print('CUDA is available!  Training on GPU ...')
def get_data_loader(batch_size,  data_dir):
    
    transform = transforms.Compose([transforms.Resize(299),
                                    transforms.RandomCrop(299),#for inception network
                                    transforms.RandomHorizontalFlip(), #random horizontal flip
                                    transforms.RandomRotation(359),# random rotaion of 1-359 degrees
                                    transforms.ToTensor()])

    image_path = './' + data_dir
    
    dataset = datasets.ImageFolder(image_path, transform)
                                             
    loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True, num_workers=0)
    
    return loader
                                             
batch_size = 128 
train_loader = get_data_loader(batch_size, data_dir+train_dir)
valid_loader = get_data_loader(batch_size, data_dir+valid_dir)
test_loader = get_data_loader(batch_size, data_dir+test_dir)
def scale(x, feature_range=(-1, 1)): #scale for tanh function
    x=x*(feature_range[1]-feature_range[0])+feature_range[0]
    return x
img = images[0]
scaled_img = scale(img)

print('Min: ', scaled_img.min())
print('Max: ', scaled_img.max())<jupyter_output>Min:  tensor(-1.)
Max:  tensor(0.7255)<jupyter_text>3) Save to S3---<jupyter_code>cd ..
import pandas as pd

data_dir = 'data/artificial-dermatologist'
if not os.path.exists(data_dir):
    os.makedirs(data_dir)
pd.DataFrame(valid_loader).to_csv(os.path.join(data_dir, 'valid.csv'), header=False, index=False)
pd.DataFrame(train_loader).to_csv(os.path.join(data_dir, 'train.csv'), header=False, index=False)
pd.DataFrame(test_loader).to_csv(os.path.join(data_dir, 'test.csv'), header=False, index=False)
import sagemaker

session = sagemaker.Session()

bucket = session.default_bucket()

role = sagemaker.get_execution_role()

prefix = 'artificial-dermatologist'

test_location = session.upload_data(os.path.join(data_dir, 'test.csv'), key_prefix=prefix)
val_location = session.upload_data(os.path.join(data_dir, 'valid.csv'), key_prefix=prefix)
train_location = session.upload_data(os.path.join(data_dir, 'train.csv'), key_prefix=prefix)
import torch
import torch.utils.data
#redownload data for testing purposes
train_sample = pd.read_csv(os.path.join(data_dir, 'train.csv'), header=None, names=None, nrows=250)
valid_sample = pd.read_csv(os.path.join(data_dir, 'valid.csv'), header=None, names=None, nrows=250)

train_sample_ds = torch.from_numpy((train_sample).values.astype(np.float32))
valid_sample_ds = torch.from_numpy((valid_sample).values.astype(np.float32))

train_sample_dl = torch.utils.data.DataLoader(train_sample_ds, batch_size=50)
valid_sample_dl = torch.utils.data.DataLoader(valid_sample_ds, batch_size=50)<jupyter_output><empty_output><jupyter_text>3) Training---<jupyter_code>from sagemaker.pytorch import PyTorch

estimator = PyTorch(entry_point="train.py",
                    source_dir="train",
                    role=role,
                    framework_version='0.4.0',
                    train_instance_count=1,
                    train_instance_type='ml.p2.xlarge',
                    hyperparameters={
                        'epochs': 10,
                        'hidden_dim': 200,
                    })<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11949133961850471, 'fraction_numerical': 0.014032010524007894, 'mean_word_length': 3.6127401415571283, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '570168', 'license': 'MIT', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1492, 'n_words': 279, 'path': '.ipynb_checkpoints/Sagemaker-model-checkpoint.ipynb', 'repo_name': 'JRPCF/Artificial-Dermatologist', 'size': '4561'}