<jupyter_start><jupyter_text>Load data<jupyter_code>df = pd.read_pickle(folder + "pickles/DAASP_RNN_dataset_with_hemolysis.plk")

df_training = df.query("Set == 'training' and (baumannii == True or aeruginosa == True) and isNotHemolytic==1")
df_test = df.query("Set == 'test' and (baumannii == True or aeruginosa == True) and isNotHemolytic==1")

if torch.cuda.is_available():
    device = "cuda" 
else:
    device = "cpu" 
print("Against A. baumannii or P. aeruginosa:\nactive training "+ str(len(df_training[df_training["activity"]==1])) \
      + "\nactive test " + str(len(df_test[df_test["activity"]==1])) \
      + "\ninactive training "+ str(len(df_training[df_training["activity"]==0])) \
      + "\ninactive test " + str(len(df_test[df_test["activity"]==0])))<jupyter_output>Against A. baumannii or P. aeruginosa:
active training 242
active test 77
inactive training 97
inactive test 25<jupyter_text>Define helper functions<jupyter_code>def randomChoice(l):
    return l[random.randint(0, len(l) - 1)]

def categoryFromOutput(output):
    top_n, top_i = output.topk(1)
    category_i = top_i[0].item()
    return category_i

def nan_equal(a,b):
    try:
        np.testing.assert_equal(a,b)
    except AssertionError:
        return False
    return True

def models_are_equal(model1, model2):
    model1.vocabulary == model2.vocabulary
    model1.hidden_size == model2.hidden_size
    for a,b in zip(model1.model.parameters(), model2.model.parameters()):
        if nan_equal(a.detach().numpy(), b.detach().numpy()) == True:
            print("true")<jupyter_output><empty_output><jupyter_text>Define hyper parameters<jupyter_code>n_embedding  = 100
n_hidden = 400
n_layers = 2
n_epoch = 200
learning_rate = 0.00001
momentum = 0.9
batch_size = 10
epoch = 22<jupyter_output><empty_output><jupyter_text>Loading and Training<jupyter_code>if not os.path.exists(folder+"pickles/generator_TL_gramneg_results_hem.pkl"):
    
    model = Generator.load_from_file(folder+"models/RNN-generator/ep{}.pkl".format(epoch))
    model.to(device)
    vocabulary = model.vocabulary

    df_training_active = df_training.query("activity == 1")
    df_test_active = df_test.query("activity == 1")
    df_training_inactive = df_training.query("activity == 0")
    df_test_inactive = df_test.query("activity == 0")

    training_dataset_active = Dataset(df_training_active, vocabulary, with_activity=False)
    test_dataset_active = Dataset(df_test_active, vocabulary, with_activity=False)
    training_dataset_inactive = Dataset(df_training_inactive, vocabulary, with_activity=False)
    test_dataset_inactive = Dataset(df_test_inactive, vocabulary, with_activity=False)

    
    optimizer = optim.SGD(model.model.parameters(), lr = learning_rate, momentum=momentum)

    # the only one used for training
    training_dataloader_active = torch.utils.data.DataLoader(training_dataset_active, batch_size=batch_size, shuffle=True, collate_fn = collate_fn, drop_last=True, pin_memory=True, num_workers=4)

    # used for evaluation
    test_dataloader_active = torch.utils.data.DataLoader(test_dataset_active, batch_size=batch_size, shuffle=False, collate_fn = collate_fn, drop_last=False, pin_memory=True, num_workers=4)
    training_dataloader_inactive = torch.utils.data.DataLoader(training_dataset_inactive, batch_size=batch_size, shuffle=False, collate_fn = collate_fn, drop_last=False, pin_memory=True, num_workers=4)
    test_dataloader_inactive = torch.utils.data.DataLoader(test_dataset_inactive, batch_size=batch_size, shuffle=False, collate_fn = collate_fn, drop_last=False, pin_memory=True, num_workers=4)
    training_dataloader_active_eval = torch.utils.data.DataLoader(training_dataset_active, batch_size=batch_size, shuffle=False, collate_fn = collate_fn, drop_last=False, pin_memory=True, num_workers=4)

    training_dictionary = {}

    for e in trange(1, n_epoch + 1):
        print("Epoch {}".format(e))
        for i_batch, sample_batched in tqdm(enumerate(training_dataloader_active), total=len(training_dataloader_active) ):

            seq_batched = sample_batched[0].to(model.device, non_blocking=True) 
            seq_lengths = sample_batched[1].to(model.device, non_blocking=True)

            nll = model.likelihood(seq_batched, seq_lengths)

            loss = nll.mean()

            optimizer.zero_grad()
            loss.backward()  
            torch.nn.utils.clip_grad_value_(model.model.parameters(), 2)
            optimizer.step()

        model.save(folder+"models/RNN-generator-TL-hem/gramneg_ep{}.pkl".format(e))


        print("\tExample Sequences")
        sampled_seq = model.sample(5)
        for s in sampled_seq:
            print("\t\t{}".format(model.vocabulary.tensor_to_seq(s, debug=True)))

        nll_training = []
        with torch.no_grad():
            for i_batch, sample_batched in enumerate(training_dataloader_active_eval):    
                seq_batched = sample_batched[0].to(model.device, non_blocking=True) 
                seq_lengths = sample_batched[1].to(model.device, non_blocking=True) 

                nll_training += model.likelihood(seq_batched, seq_lengths)

        nll_training_active_mean = torch.stack(nll_training).mean().item()
        print("\tNLL Train Active: {}".format(nll_training_active_mean))
        del nll_training

        nll_test = []
        with torch.no_grad():
            for i_batch, sample_batched in enumerate(test_dataloader_active):    
                seq_batched = sample_batched[0].to(model.device, non_blocking=True) 
                seq_lengths = sample_batched[1].to(model.device, non_blocking=True) 

                nll_test += model.likelihood(seq_batched, seq_lengths)

        nll_test_active_mean = torch.stack(nll_test).mean().item()
        print("\tNLL Test Active: {}".format(nll_test_active_mean))
        del nll_test

        nll_training = []
        with torch.no_grad():
            for i_batch, sample_batched in enumerate(training_dataloader_inactive):    
                seq_batched = sample_batched[0].to(model.device, non_blocking=True) 
                seq_lengths = sample_batched[1].to(model.device, non_blocking=True) 

                nll_training += model.likelihood(seq_batched, seq_lengths)

        nll_training_inactive_mean = torch.stack(nll_training).mean().item()
        print("\tNLL Train Inactive: {}".format(nll_training_inactive_mean))
        del nll_training

        nll_test = []
        with torch.no_grad():
            for i_batch, sample_batched in enumerate(test_dataloader_inactive):    
                seq_batched = sample_batched[0].to(model.device, non_blocking=True) 
                seq_lengths = sample_batched[1].to(model.device, non_blocking=True) 

                nll_test += model.likelihood(seq_batched, seq_lengths)

        nll_test_inactive_mean = torch.stack(nll_test).mean().item()
        print("\tNLL Test Inactive: {}".format(nll_test_inactive_mean))
        del nll_test
        print()

        training_dictionary[e]=[nll_training_active_mean, nll_test_active_mean, nll_training_inactive_mean, nll_test_inactive_mean]
    
    with open(folder+"pickles/generator_TL_gramneg_results_hem.pkl",'wb') as fd:
        pickle.dump(training_dictionary, fd)
    
else:
    with open(folder+"pickles/generator_TL_gramneg_results_hem.pkl",'rb') as fd:
        training_dictionary = pickle.load(fd)

min_nll_test_active = float("inf")
for epoch, training_values in training_dictionary.items():
    nll_test_active = training_values[1]

    if nll_test_active < min_nll_test_active:
        best_epoch = epoch
        min_nll_test_active = nll_test_active<jupyter_output><empty_output><jupyter_text>Sampling evaluation<jupyter_code>print(best_epoch)
model = Generator.load_from_file(folder+"models/RNN-generator-TL-hem/gramneg_ep{}.pkl".format(best_epoch))<jupyter_output>200<jupyter_text>199<jupyter_code>training_seq = df_training.Sequence.values.tolist()

def _sample(model, n):
    sampled_seq = model.sample(n)
    sequences = []
    for s in sampled_seq:
        sequences.append(model.vocabulary.tensor_to_seq(s))
    return sequences

def novelty(seqs, list_):
    novel_seq = []
    for s in seqs:
        if s not in list_:
            novel_seq.append(s)
    return novel_seq, (len(novel_seq)/len(seqs))*100

def is_in_training(seq, list_ = training_seq):
    if seq not in list_:
        return False
    else:
        return True

def uniqueness(seqs):
    unique_seqs = defaultdict(int)
    for s in seqs:
        unique_seqs[s] += 1
    return unique_seqs, (len(unique_seqs)/len(seqs))*100
# sample
seqs = _sample(model, 50000)
unique_seqs, perc_uniqueness = uniqueness(seqs)
notintraining_seqs, perc_novelty = novelty(unique_seqs, training_seq)


# create dataframe
df_generated = pd.DataFrame(list(unique_seqs.keys()), columns =['Sequence']) 
df_generated["Repetition"] = df_generated["Sequence"].map(lambda x: unique_seqs[x])
df_generated["inTraining"] = df_generated["Sequence"].map(is_in_training)
df_generated["Set"] = "generated-TL-GN-hem"

# save
df_generated.to_pickle(folder+"pickles/Generated-TL-gramneg-hem.pkl")
print(perc_uniqueness, perc_novelty)<jupyter_output>82.89999999999999 99.61158021712907
{'fraction_non_alphanumeric': 0.11315417256011315, 'fraction_numerical': 0.013382656947013382, 'mean_word_length': 4.109505280711506, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '8958', 'license': 'MIT', 'n_tokens_mistral': 3263, 'n_tokens_neox': 3087, 'n_words': 490, 'path': '06-TL-GramNegative.ipynb', 'repo_name': 'reymond-group/MLpeptide', 'size': '9191'}