<jupyter_start><jupyter_text>Get files<jupyter_code>labels_dict = dict(zip(train_df['SOPInstanceUID'], train_df['pe_present_on_image']))
len(labels_dict), len(train_df)
imgdatapath = (datapath/'full_raw_512')
files = get_image_files(imgdatapath)
train_files = [o for o in files if o.parent.name in train_pids]
valid_files = [o for o in files if o.parent.name in valid_pids]
len(train_files), len(valid_files)
files = train_files + valid_files
len(files)<jupyter_output><empty_output><jupyter_text>Load Model<jupyter_code>def get_label(o): return labels_dict[o.stem.split("_")[1]]
resize = 512
cnn_learner = load_learner(f"./models/xresnet34-{resize}-PR-fold{FOLD}-export.pkl", cpu=False)
valid_dl = cnn_learner.dls.test_dl(train_files+valid_files, with_labels=True)<jupyter_output><empty_output><jupyter_text>Generate Embeddings<jupyter_code>embs_dir = Path(datapath/f"cnn_embs")
if not embs_dir.exists(): embs_dir.mkdir()
len(valid_dl.dataset)
class EmbeddingHook:
    def __init__(self, m, subdir, csz=5000000, n_init=0):
        store_attr("m,subdir,csz")
        self.embeddings = tensor([])
        if len(m._forward_hooks) > 0: self.reset()
        self.hook = Hook(m, self.hook_fn, cpu=True)
        self.save_iter = n_init        
        if not (embs_dir/self.subdir).exists(): (embs_dir/self.subdir).mkdir()
    
    def hook_fn(self, m, inp, out): 
        "Stack and save computed embeddings"
        self.embeddings = torch.cat([self.embeddings, out])
        if self.embeddings.shape[0] > self.csz:
            self.save()
            self.embeddings = tensor([])
    
    def reset(self): 
        self.m._forward_hooks = OrderedDict()
        
    def save(self): 
        torch.save(self.embeddings, embs_dir/self.subdir/f"train_embs-{self.save_iter}.pkl")
        self.save_iter += 1
emb_hook = EmbeddingHook(cnn_learner.model[1][1], subdir=f"full_sz{resize}_FOLD{FOLD}", n_init=0)
cnn_learner.model[1][1]._forward_hooks, emb_hook.embeddings
preds, targs = cnn_learner.get_preds(dl=valid_dl, act=noop)<jupyter_output><empty_output><jupyter_text>Save<jupyter_code>torch.save(emb_hook.embeddings, embs_dir/f"full_sz{resize}_FOLD{FOLD}"/"embeddings.pkl")
pd.to_pickle(train_files+valid_files, embs_dir/f"full_sz{resize}_FOLD{FOLD}"/"files.pkl")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1402359108781127, 'fraction_numerical': 0.010921799912625601, 'mean_word_length': 5.090425531914893, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '208187', 'license': 'Apache-2.0', 'n_tokens_mistral': 908, 'n_tokens_neox': 886, 'n_words': 122, 'path': 'rsna_pe_notebooks/02 - 2D CNN features.ipynb', 'repo_name': 'KeremTurgutlu/medical', 'size': '2289'}