<jupyter_start><jupyter_text>Setup<jupyter_code>exp_dir = "/srv/nas/mk2/projects/pan-cancer/experiments"
tcga, tcga_meta = process_data()
feature_idx = generate_fsets(tcga, steps=20)
hierarchy = OrderedDict({'Disease':['BRCA', 'LUAD', 'KIRC', 'THCA', 'PRAD', 'SKCM'],
                         'Sample Type':['Primary Tumor', 'Solid Tissue Normal']})
# Define experiment
exp = Experiment(meta_data=tcga_meta,
                 hierarchy=hierarchy,
                 index='CGHubAnalysisID',
                 cases='Case ID',
                 min_samples=20)
# Train / Test split
exp.train_test_split(cases='Case ID')
# Return data 
train_data, train_labels = exp.get_data(tcga, subset="train", dtype=np.float32)
test_data, test_labels = exp.get_data(tcga, subset="test", dtype=np.float32)
experiments = {'experiment': exp,
               'train':(train_data, train_labels),
               'test': (test_data, test_labels)}
pd.to_pickle(experiments, os.path.join(exp_dir, "experiment_meta_data.pkl"))<jupyter_output><empty_output><jupyter_text>Network<jupyter_code>def feature_training(train_data, train_labels, test_data, test_labels, feature_idx, exp_dir, cuda=True):
    # Meta data
    meta_data = {"n_features":[],
                 "model":[],
                 "ANMI":[]}
    # Params
    batch_size = 8
    kwargs = {'num_workers': 10, 'pin_memory': True} if cuda else {'num_workers': 10}
    
    # Feature Index
    for batch, feat in enumerate(feature_idx):
        print("Batch {}\n".format(batch))
        exp_data = {'feature_idx':feat}
        # Define data
        siamese_train_dataset = SiameseDataset(data=train_data.iloc[:,feat],
                                           labels=train_labels,
                                           train=True)
        siamese_test_dataset = SiameseDataset(data=test_data.iloc[:,feat],
                                          labels=test_labels,
                                          train=False)
        # Loaders
        siamese_train_loader = torch.utils.data.DataLoader(siamese_train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
        siamese_test_loader = torch.utils.data.DataLoader(siamese_test_dataset, batch_size=batch_size, shuffle=False, **kwargs)
        # Instantiate model
        n_samples, n_features = siamese_train_dataset.train_data.shape
        for i in range(3):
            nmodel = 'model_{}'.format(i)
            print("\t{}".format(nmodel))
            embedding_net = EmbeddingNet(n_features, 2)
            model = SiameseNet(embedding_net)
            if cuda:
                model.cuda()
            # Parameters
            margin = 1.
            loss_fn = ContrastiveLoss(margin)
            lr = 1e-3
            optimizer = optim.Adam(model.parameters(), lr=lr)
            scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
            n_epochs = 10
            log_interval = round(len(siamese_train_dataset)/4/batch_size)
            # Train
            train_loss, val_loss = fit(siamese_train_loader, siamese_test_loader, model, loss_fn, optimizer, scheduler, 
                                       n_epochs, cuda, log_interval)
            # Test Embeddings
            val_embeddings_baseline, val_labels_baseline = vis.extract_embeddings(siamese_test_dataset.test_data, siamese_test_dataset.labels, model)
            # Evaluation
            n_clusters = len(np.unique(test_labels))
            kmeans = KMeans(n_clusters=n_clusters)
            siamese_clusters = kmeans.fit_predict(val_embeddings_baseline)
            anmi = ANMI(siamese_clusters, val_labels_baseline)
            # Store
            meta_data['n_features'].append(len(feat))
            meta_data['model'].append(nmodel)
            meta_data['ANMI'].append(anmi)
            exp_data[nmodel] = {'data': (val_embeddings_baseline, val_labels_baseline),
                                'loss': (train_loss, val_loss),
                                'ANMI': anmi}
        pd.to_pickle(exp_data, os.path.join(exp_dir, "model_{}.pkl".format(len(feat))))
    pd.to_pickle(meta_data, os.path.join(exp_dir, "model_meta_data.pkl"))
feature_training(train_data, train_labels, test_data, test_labels, feature_idx, exp_dir)
foo = pd.read_pickle("/srv/nas/mk2/projects/pan-cancer/experiments/meta_data.pkl")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11395617070357555, 'fraction_numerical': 0.006228373702422145, 'mean_word_length': 2.5109311740890687, 'pattern_counts': {'":': 3, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '242124', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1353, 'n_words': 211, 'path': 'notebook/.ipynb_checkpoints/2020.03.23_feat_sel_long_range-checkpoint.ipynb', 'repo_name': 'wconnell/metrx', 'size': '4335'}