<jupyter_start><jupyter_text>PreprocessAfter doing this once, you can skip to the training if the script broke<jupyter_code>df_download = pd.read_csv(os.path.join(PROCESSED_DIR, 'image_download_locs.csv'))
downloaded = os.listdir(os.path.join(COUNTRIES_DIR, 'ethiopia_2015', 'images'))
print(f"actually downloaded: {len(downloaded)}, expected: {len(df_download)}")
df_download.head()
df_download['row'] = np.arange(len(df_download))
idx_not_download = df_download.set_index('image_name').drop(downloaded)['row'].values.tolist()
df_download.drop(idx_not_download, inplace=True)
df_download.drop('row', axis=1, inplace=True)
# the distribution
(df_download['harvest_bin']==0).mean(), (df_download['harvest_bin']==1).mean(), (df_download['harvest_bin']==2).mean()<jupyter_output><empty_output><jupyter_text>Split images into train/valid.Each cluster will contribute 80% of images for training, and 20% for validation.<jupyter_code>df_download.reset_index(drop=True, inplace=True)
df_download.head()
df_download['is_train'] = True
np.random.seed(RANDOM_SEED)
groups = df_download.groupby(['cluster_lat', 'cluster_lon'])
for _, g in groups:
    n_ims = len(g)
    n_train = int(0.8 * n_ims)
    n_valid = n_ims - n_train
    valid_choices = np.random.choice(np.arange(n_ims), replace=False, size=n_valid).tolist()
    current_index = g.index
    idx_valid = current_index[valid_choices]
    df_download['is_train'].loc[idx_valid] = False
df_download['is_train'].mean()
# save this new dataframe
df_download.to_csv(os.path.join(PROCESSED_DIR, 'image_download_actual.csv'), index=False)
os.makedirs(os.path.join(CNN_TRAIN_IMAGE_DIR, 'train'), exist_ok=False)
os.makedirs(os.path.join(CNN_TRAIN_IMAGE_DIR, 'valid'), exist_ok=False)

labels = ['0', '1', '2']
for l in labels:
    os.makedirs(os.path.join(CNN_TRAIN_IMAGE_DIR, 'train', l), exist_ok=False)
    os.makedirs(os.path.join(CNN_TRAIN_IMAGE_DIR, 'valid', l), exist_ok=False)
t = df_download[df_download['is_train']]
v = df_download[~df_download['is_train']]
len(t), len(v)
# uses symlinking to save disk space
print('copying train images')
for im_name, nl, country in tqdm(zip(t['image_name'], t['harvest_bin'], t['country']), total=len(t)):
    country_dir = None
    if country == 'mw':
        country_dir = 'malawi_2016'
    elif country == 'eth':
        country_dir = 'ethiopia_2015'
    elif country == 'ng':
        country_dir = 'nigeria_2015'
    else:
        print(f"no match for country {country}")
        raise ValueError()
    src = os.path.abspath(os.path.join(COUNTRIES_DIR, country_dir, 'images', im_name))
    dest = os.path.join(CNN_TRAIN_IMAGE_DIR, 'train', str(nl), im_name)
    if os.symlink(src, dest, target_is_directory = False) != None:
        print("error creating symlink")
        raise ValueError()

print('copying valid images')
for im_name, nl, country in tqdm(zip(v['image_name'], v['harvest_bin'], v['country']), total=len(v)):
    country_dir = None
    if country == 'mw':
        country_dir = 'malawi_2016'
    elif country == 'eth':
        country_dir = 'ethiopia_2015'
    elif country == 'ng':
        country_dir = 'nigeria_2015'
    else:
        print(f"no match for country {country}")
        raise ValueError()
    src = os.path.abspath(os.path.join(COUNTRIES_DIR, country_dir, 'images', im_name))
    dest = os.path.join(CNN_TRAIN_IMAGE_DIR, 'valid', str(nl), im_name)
    if os.symlink(src, dest, target_is_directory = False) != None:
        print("error creating symlink")
        raise ValueError()
# shows count distribution in train folder, make sure this matches above
counts = []
for l in ['0', '1', '2']:
    counts.append(len(os.listdir(os.path.join(CNN_TRAIN_IMAGE_DIR, 'train', l))))
print(counts)
print([c/sum(counts) for c in counts])
print(sum(counts))
# shows count distribution in valid folder
counts = []
for l in ['0', '1', '2']:
    counts.append(len(os.listdir(os.path.join(CNN_TRAIN_IMAGE_DIR, 'valid', l))))
print(counts)
print([c/sum(counts) for c in counts])
print(sum(counts))<jupyter_output>[1220, 500, 610]
[0.5236051502145923, 0.2145922746781116, 0.26180257510729615]
2330<jupyter_text>Train ModelHeavily adapted from the PyTorch CNN training tutorial.<jupyter_code>import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copy
# Top level data directory.

data_dir = CNN_TRAIN_IMAGE_DIR

# Models to choose from [resnet, alexnet, vgg, squeezenet, densenet, inception]
model_name = "vgg"

# Number of classes in the dataset
num_classes = 3

# Batch size for training (change depending on how much memory you have)
batch_size = 8

# Number of epochs to train for, first 10 will be training the new layers, last 10 the whole model
num_epochs = 2

# Flag for feature extracting. When False, we finetune the whole model,
#   when True we only update the reshaped layer params
feature_extract = True
def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
    # Initialize these variables which will be set in this if statement. Each of these
    #   variables is model specific.
    model_ft = models.vgg11_bn(pretrained=use_pretrained)
    set_parameter_requires_grad(model_ft, feature_extract)
    num_ftrs = model_ft.classifier[6].in_features
    model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
    input_size = 224
    return model_ft, input_size

def set_parameter_requires_grad(model, feature_extracting):
    if feature_extracting:
        for param in model.parameters():
            param.requires_grad = False
# Initialize the model for this run
model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)

# Print the model we just instantiated
model_ft
# you can modify the classifier part of the model by doing this
# model_ft.classifier = nn.Sequential(
#     nn.Linear(in_features=25088, out_features=4096, bias=True),
#     nn.ReLU(inplace=True),
#     nn.Dropout(p=0.5),
#     nn.Linear(in_features=4096, out_features=256, bias=True),
#     nn.ReLU(inplace=True),
#     nn.Dropout(p=0.5),
#     nn.Linear(in_features=256, out_features=3, bias=True),
# )
# Data augmentation and normalization for training
# Just normalization for validation
data_transforms = {
    'train': transforms.Compose([
        transforms.RandomResizedCrop(input_size),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'valid': transforms.Compose([
        transforms.Resize(input_size),
        transforms.CenterCrop(input_size),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}
print("Initializing Datasets and Dataloaders...")

# Create training and validation datasets
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'valid']}
# Create training and validation dataloaders
dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'valid']}

# Detect if we have a GPU available
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('device:', device)
# Send the model to GPU
model_ft = model_ft.to(device)

# Gather the parameters to be optimized/updated in this run. If we are
#  finetuning we will be updating all parameters. However, if we are
#  doing feature extract method, we will only update the parameters
#  that we have just initialized, i.e. the parameters with requires_grad
#  is True.
params_to_update = model_ft.parameters()
print("Params to learn:")
if feature_extract:
    params_to_update = []
    for name,param in model_ft.named_parameters():
        if param.requires_grad == True:
            params_to_update.append(param)
            print("\t",name)
else:
    for name,param in model_ft.named_parameters():
        if param.requires_grad == True:
            print("\t",name)

# Observe that all parameters are being optimized
optimizer_ft = optim.SGD(params_to_update, lr=1e-4, momentum=0.9)
def train_model(model, dataloaders, criterion, optimizer, num_epochs=25):
    since = time.time()

    val_acc_history = []
    
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        if epoch > 10:
            # fine tune whole model
            for param in model_ft.parameters():
                param.requires_grad = True
            optimizer = optim.SGD(model_ft.parameters(), lr=1e-4, momentum=0.9)

        # Each epoch has a training and validation phase
        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in tqdm(dataloaders[phase]):
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    loss = criterion(outputs, labels)

                    _, preds = torch.max(outputs, 1)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
            if phase == 'valid':
                val_acc_history.append(epoch_acc)
                
        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model, val_acc_history
# Setup the loss fxn
criterion = nn.CrossEntropyLoss()

# Train and evaluate
model_ft, hist = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft, num_epochs=num_epochs)
path = os.path.join(CNN_SAVE_DIR, 'trained_model.pt')
assert not os.path.isfile(path), print('A model is already saved at this location')
print(f'Saving model to {path}')
torch.save(model_ft, path)
# you can run below if you want to see the final accuracy on nightlights over the train set
model_ft.eval()   # Set model to evaluate mode

criterion = nn.CrossEntropyLoss()
running_loss = 0.0
running_corrects = 0
total = 0

# Iterate over data.
for inputs, labels in tqdm(dataloaders_dict['train']):
    inputs = inputs.to(device)
    labels = labels.to(device)

    # forward
    # track history if only in train
    with torch.set_grad_enabled(False):
        outputs = model_ft(inputs)
        loss = criterion(outputs, labels)

        _, preds = torch.max(outputs, 1)

    # statistics
    running_loss += loss.item() * inputs.size(0)
    running_corrects += torch.sum(preds == labels.data)
    
    total += len(preds)
        
print(running_corrects.double()/total)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11743241001824514, 'fraction_numerical': 0.019737933322275667, 'mean_word_length': 3.5283514832895233, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '379862', 'license': 'MIT', 'n_tokens_mistral': 4094, 'n_tokens_neox': 3876, 'n_words': 976, 'path': 'scripts/train_cnn.ipynb', 'repo_name': 'omshah2006/predicting-poverty-ethiopia', 'size': '12058'}