<jupyter_start><jupyter_text>INFERENCE<jupyter_code>if not TRAIN:
    model_name = '19-08-2018_09:09:51_ternausv2.06-0.1506.pt'
    model = load_model(SAVE_PATH + model_name)

ids= ['1f1cc6b3a4','5b7c160d0d','6c40978ddf'] #,'7dfdf6eeb8','7e5a6e5013']
def show(ids):
    for j, img_name in enumerate(ids):
        q = j + 1
        img = cv2.imread(DIRECTORY + '/train/images/' + img_name + '.png')
        img_mask = cv2.imread(DIRECTORY + '/train/masks/' + img_name + '.png')

        plt.figure(figsize=(15,10))
        plt.subplot(1, 2, 1)
        plt.imshow(img)
        plt.subplot(1, 2, 2)
        plt.imshow(img_mask)

depths = read_csv(DIRECTORY + '/depths.csv', index_col=0)['z']

img_id = '5b7c160d0d'
orig_img = imread(DIRECTORY + '/train/images/' + img_id +'.png')
orig_mask = imread(DIRECTORY + '/train/masks/' + img_id + '.png')
pad = aug.PadIfNeeded(min_height=128, min_width=128)
crop = aug.CenterCrop(height=101, width=101)
img = pad(image=orig_img)["image"] / 255.

#img[:, :, 1] = np.cumsum(img[:, :, 0], axis=0)
#img[:, :, 2] = depths[img_id] / 1000.

img = np.transpose(img, (2, 0, 1)).astype('float32') 
img = np.expand_dims(img, 0)
#print(img.shape)
img = torch.from_numpy(img)

class Batch(object):
    pass
batch = Batch()
batch.image = torch.autograd.Variable(img).cuda()
pred = model(batch)
#pred = model({ 'image': batch.image })
#pred
mask = pad(image=orig_mask)["image"]
ones = np.expand_dims(mask, 2) > 0
zeros = ones != True
#ones = !zeros
mask = np.concatenate((ones, zeros), axis=2)
mask[:, :, 0].astype('float32')
y_pred = pred.data[0][0].cpu().numpy()
import matplotlib.pyplot as plt
%matplotlib inline
print(mask.shape)
plt.figure(figsize=(15,10))
plt.subplot(1, 4, 1)
plt.imshow(orig_img)
plt.subplot(1, 4, 2)
plt.imshow(orig_mask)
plt.subplot(1, 4, 3)
plt.imshow(crop(image=y_pred)['image'])
plt.subplot(1, 4, 4)
plt.imshow(crop(image=mask[:,:,1])['image'])
from torchbearer import Model
from lenin.augmentors.image import Augmentor

def predict(model, dataset, **options):
    batch_size = options.pop('batch_size', 32)

    augmentors = {}
    for fields, augment in options.pop('augment', {}).items():
        if isinstance(fields, str):
            fields = [fields]
        augmentor = Augmentor(augment)
        for field in fields:
            augmentors[field] = augmentor

    preload_opts = options.pop('preloader', { 'shuffle': False })
    testgen = __preloader_predict(model, dataset, 'test_fields', dataset.test, batch_size, preload_opts, augmentors)

    device = options.pop('device', 'cuda')
    bearer = Model(model, torch.optim.Adam(model.parameters())).to(device)
    return bearer.predict_generator(testgen)


def __preloader_predict(model, dataset, fields_name, records, batch_size, preload_opts, augmentors={}):
    fields = __fields(model, dataset, fields_name)
    def collate_fn(records):
        batch = { 'records': torch.utils.data.dataloader.default_collate(records) }
        for name in fields:
            if augmentors.get(name):
                values = [getattr(dataset, name)(record, augmentors[name]) for record in records]
            else:
                values = [getattr(dataset, name)(record) for record in records]
            batch[name] = torch.utils.data.dataloader.default_collate(values)
        return batch
    return torch.utils.data.DataLoader(records, batch_size, collate_fn=collate_fn, **preload_opts)


def __fields(model, dataset, fields_name):
    return getattr(model, fields_name, None) or getattr(model, 'fields', None) or dataset.fields
predictions = test(model, dataset,
    augment={ ('image',): [{'type': 'PadIfNeeded', 'min_height': 128, 'min_width': 128}] }
)
val_dataset = Dataset(DIRECTORY)
val_dataset.test = dataset.train[:2000]
val_predictions = test(model, val_dataset,
    augment={ ('image',): [{'type': 'PadIfNeeded', 'min_height': 128, 'min_width': 128}] }
)
_img = val_predictions[1803].data[0].cpu().numpy()
plt.figure(figsize=(15,10))
plt.subplot(1, 4, 1)
plt.imshow(_img)
val_preds = np.array([crop(image=pred)['image'] for pred in val_predictions.data.cpu().numpy()[:, 0, :, :]])
val_masks = [crop(image=val_dataset.mask(rec).numpy()[0])['image'] for rec in val_dataset.test]
plt.figure(figsize=(15,10))
plt.subplot(1, 4, 1)
plt.imshow(val_preds[1803])
plt.subplot(1, 4, 2)
plt.imshow(val_masks[1803])
from sklearn.metrics import jaccard_similarity_score

metric_by_threshold = []
for threshold in np.linspace(0, 1, 11):
    val_binary_prediction = (val_preds > threshold).astype(int)
    
    iou_values = []
    for y_mask, p_mask in zip(val_masks, val_binary_prediction):
        iou = jaccard_similarity_score(y_mask.flatten(), p_mask.flatten())
        iou_values.append(iou)
    iou_values = np.array(iou_values)
    
    accuracies = [
        np.mean(iou_values > iou_threshold)
        for iou_threshold in np.linspace(0.5, 0.95, 10)
    ]
    print('Threshold: %.1f, Metric: %.3f' % (threshold, np.mean(accuracies)))
    metric_by_threshold.append((np.mean(accuracies), threshold))
    
best_metric, best_threshold = max(metric_by_threshold)
predictions = np.array([crop(image=pred)['image'] for pred in predictions.data.cpu().numpy()[:, 0, :, :]])
binary_predictions = (predictions > best_threshold).astype(int)

def rle_encoding(x):
    dots = np.where(x.T.flatten() == 1)[0]
    run_lengths = []
    prev = -2
    for b in dots:
        if (b > prev+1): run_lengths.extend((b + 1, 0))
        run_lengths[-1] += 1
        prev = b
    return run_lengths

all_masks = []
for p_mask in list(binary_predictions):
    p_mask = rle_encoding(p_mask)
    all_masks.append(' '.join(map(str, p_mask)))
import pandas as pd
submit = pd.DataFrame([[rec.split('-')[1] for rec in dataset.test], all_masks]).T
submit.columns = ['id', 'rle_mask']
submit.to_csv('/home/saint/submissions/salt/%s.csv' % model_name, index = False)
submit = SUBMISSIONS_PATH + model_name + '.csv'
!kaggle c submit -f {submit.replace(':', '\:')} -m '{submit}' -c tgs-salt-identification-challenge<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16503403619458742, 'fraction_numerical': 0.032541922629918646, 'mean_word_length': 4.261135371179039, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '422677', 'license': 'MIT', 'n_tokens_mistral': 2379, 'n_tokens_neox': 2249, 'n_words': 436, 'path': 'code_gazay/salt/notebooks/Ternaus with lenin.ipynb', 'repo_name': 'artyompal/kaggle_salt', 'size': '6023'}