<jupyter_start><jupyter_text>note: implement auc metric   imports<jupyter_code>%reload_ext autoreload
%autoreload 2
from fastai.vision import *
import fastai; fastai.__version__
chexpert_path = Path("../../data/chexpert/")
mimic_path = Path("../../data/mimic-cxr")
chexpert_path.ls()
mimic_path.ls()
torch.cuda.set_device(3)<jupyter_output><empty_output><jupyter_text>data<jupyter_code>chx_train_df = pd.read_csv(chexpert_path/"CheXpert-v1.0-small/train.csv")
chx_valid_df = pd.read_csv(chexpert_path/"CheXpert-v1.0-small/valid.csv")
chx_train_df.head(2)
mimic_train_df = pd.read_csv(mimic_path/"train.csv").rename(columns={"path":"Path", "Airspace Opacity": "Lung Opacity"})
mimic_valid_df = pd.read_csv(mimic_path/"valid.csv").rename(columns={"path":"Path", "Airspace Opacity": "Lung Opacity"})
mimic_train_df.head(2)
label_columns = list(chx_train_df.columns[5:])
columns = ['Path'] + label_columns
mimic_train_df['Path'] = mimic_train_df['Path'].apply(lambda x: 'mimic-cxr/'+x)
mimic_valid_df['Path'] = mimic_valid_df['Path'].apply(lambda x: 'mimic-cxr/'+x)
chx_train_df['Path'] = chx_train_df['Path'].apply(lambda x: 'chexpert/'+x)
chx_valid_df['Path'] = chx_valid_df['Path'].apply(lambda x: 'chexpert/'+x)
# join chexpert and mimic train - valid
train_df = pd.concat([chx_train_df[columns], mimic_train_df[columns]])
valid_df = pd.concat([chx_valid_df[columns], mimic_valid_df[columns]])
train_df.shape, valid_df.shape
# https://github.com/stanfordmlgroup/chexpert-labeler/issues/9
train_df[label_columns] = train_df[label_columns].fillna(-1)
{c: train_df[c].unique() for c in label_columns}
{c: valid_df[c].unique() for c in label_columns}
train_df.head()<jupyter_output><empty_output><jupyter_text>U-ignore<jupyter_code>train_df[label_columns].apply(lambda x: (x == -1).sum()/len(x))
train_df[label_columns].apply(lambda x: (x != -1).sum())
train_df.head()
train_df.tail()<jupyter_output><empty_output><jupyter_text>training size<jupyter_code>sz = 320
train_df.Path = train_df.Path.apply(lambda x: x.replace("train", f"train_{sz}"))
valid_df.Path = valid_df.Path.apply(lambda x: x.replace("train", f"valid_{sz}"))
# shape of Chexpert small
data_path = Path("../../data")
open_image(data_path/train_df.Path.values[4]).data.shape
# shape of MIMIC CXR
open_image(data_path/train_df.Path.values[-1]).data.shape
train_df.shape, valid_df.shape
train_df = train_df.fillna(-1); 
valid_df = valid_df.fillna(-1)
train_df.to_csv(mimic_path/"chx_mimic_train_320.csv", index=False)
valid_df.to_csv(mimic_path/"chx_mimic_valid_320.csv", index=False)<jupyter_output><empty_output><jupyter_text>databunch<jupyter_code>train_df = pd.read_csv(mimic_path/"chx_mimic_train_320.csv")
valid_df = pd.read_csv(mimic_path/"chx_mimic_valid_320.csv")
label_columns = train_df.columns[1:]
label_columns
sz,bs=320,64
data_path = Path("../../data")
train_valid_df = pd.concat([train_df, valid_df])
il = ImageList.from_df(train_valid_df, data_path, cols=['Path'])
ill = il.split_by_rand_pct(valid_pct=0.02, seed=42)
assert train_valid_df.shape[0] == (len(ill.train) + len(ill.valid))
train_fnames = ["/".join(s.split("/")[4:]) for s in ill.train.items]
valid_fnames = ["/".join(s.split("/")[4:]) for s in ill.valid.items]
assert train_valid_df.shape[0] == (len(train_fnames) + len(valid_fnames))
# create labels
is_train = train_valid_df.Path.apply(lambda x: '/'.join(x.split('/')[1:])).isin(train_fnames)
train_labels = np.array(train_valid_df.loc[is_train, label_columns])
valid_labels = np.array(train_valid_df.loc[~is_train, label_columns])
assert train_valid_df.shape[0] == (len(train_labels) + len(valid_labels))
tfms = get_transforms()
pretrained = True

data = (ill.label_from_lists(list(train_labels), list(valid_labels), label_cls=FloatList)
        .transform(tfms, size=sz)
        .databunch(bs=bs))
data = data.normalize(imagenet_stats) if pretrained else data.normalize()
assert train_valid_df.shape[0] == (len(data.train_ds) + len(data.valid_ds))
# set data classes
data.classes = label_columns
data.classes, data.stats
# data.show_batch(3)<jupyter_output><empty_output><jupyter_text>model<jupyter_code>from chexpert_dev.uignore import *
pretrained = True
# eps = 1e-4
# opt_func= partial(optim.Adam, betas=(0.9,0.99), eps=eps)
if pretrained:
    learn = cnn_learner(data, models.densenet121, pretrained=True,
                        loss_func=MaskedBCEWithLogitsLoss()).to_fp16()
else:                                  
    learn = Learner(data, models.xresnet34(c_out=data.c),
                    loss_func=MaskedBCEWithLogitsLoss()).to_fp16()
roc_auc_cb = MaskedMultiLabelROCAUC(learn=learn, verbose=False)
learn.metrics = [partial(masked_accuracy_thres, thres=0.5, m=-1)]
learn.lr_find()
learn.recorder.plot()
# training from scratch
learn.fit_one_cycle(5, 1e-2, callbacks=[roc_auc_cb])
# # transfer learning

lr=1e-2
# learn.fit_one_cycle(5, lr)

learn.freeze_to(-1)

learn.fit_one_cycle(5, slice(lr/5))

learn.unfreeze()

learn.fit_one_cycle(5, slice(lr/10))
# learn.save(f"u-ignore-resnet34-{sz}")
# learn.to_fp32().save(f"u-ignore-resnext34-{sz}")
learn.to_fp32().save(f"chx-mimic-u-ignore-densenet121-{sz}")
learn.load(f"chx-mimic-u-ignore-densenet121-{sz}");
learn.validate(callbacks=[roc_auc_cb])<jupyter_output><empty_output><jupyter_text>add test<jupyter_code>test = ImageList.from_df(valid_df, data_path, cols=['Path'])
learn.data.add_test(test)
# learn.load(f"u-ignore-resnext34-{sz}");
learn.load(f"u-ignore-densenet121-{sz}");<jupyter_output>/home/turgutluk/.conda/envs/my_fastai/lib/python3.7/site-packages/torch/serialization.py:256: UserWarning: Couldn't retrieve source code for container of type MaskedBCEWithLogitsLoss. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "<jupyter_text>per class AUCs of test<jupyter_code>preds, targs = learn.get_preds(DatasetType.Test)
probas = preds.sigmoid()
targs = tensor(np.array(valid_df[label_columns]))
targs.shape, probas.shape
from sklearn.metrics import roc_auc_score, auc
{c:roc_auc_score(targs[:,i], probas[:, i]) for i,c in enumerate(label_columns[:-2])}
{c:roc_auc_score(targs[:,i], probas[:, i]) for i,c in enumerate(label_columns[:-2])}
np.mean([roc_auc_score(targs[:,i], probas[:, i]) for i,c in enumerate(label_columns[:-2])])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16605577689243028, 'fraction_numerical': 0.017370517928286852, 'mean_word_length': 6.815691158156912, 'pattern_counts': {'":': 4, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '194985', 'license': 'MIT', 'n_tokens_mistral': 2571, 'n_tokens_neox': 2512, 'n_words': 339, 'path': '1b) chexpert+mimic cxr (u-ignore).ipynb', 'repo_name': 'KeremTurgutlu/siim-acr-pneumothorax', 'size': '6275'}