<jupyter_start><jupyter_text>Input<jupyter_code>class BertDataSet(Dataset):
    """Face Landmarks dataset."""

    def __init__(self, path_or_data, seq_length, vocab_size=20, records_to_test = 1000000):
        #self.data = np.load(path)[:records_to_test]
        if isinstance(path_or_data, str):
            self.data = np.load(path_or_data)[2]
        else: 
            self.data = path_or_data
        self.seq_length = seq_length
        self.vocab_size = vocab_size

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        row = self.data[idx]
#         seq = np.asarray(row[-1])
        seq = row
        length = (0 == seq).argmax(axis=0) if (0 in seq) else len(seq)
        mask = np.ones(length)
        masked_seq = np.copy(seq)        
        indicies = np.sort(np.random.randint(0, length-1, 20))
        np.put(masked_seq, indicies, 21)
#         seq, mask, masked_seq = self.pad([seq, mask, masked_seq], self.seq_length-length)
        mask = self.pad([mask], self.seq_length-length)[0]
        label = np.take(seq, indicies)
        output = self.to_int64([masked_seq, mask, indicies, seq, label])
        return output[0:4], output[4]
    
    def pad(self, items_to_pad, pad_width):
        for i in range(len(items_to_pad)):
            items_to_pad[i] = np.pad(items_to_pad[i], mode="constant", pad_width=(0,pad_width))        
        return items_to_pad
        
    def to_int64(self, items_to_convert):
        for i in range(len(items_to_convert)):
            items_to_convert[i] = np.int64(items_to_convert[i])
        return items_to_convert
        
def validate_ds(test_ds):
    (masked_seq, mask, indicies, seq), label = test_ds.__getitem__(1)
    assert (0 == masked_seq).argmax(axis=0) == (0 == mask).argmax(axis=0)
    assert len(masked_seq) == 512
    assert len(mask) == 512
    assert len(indicies) == 20
    assert len(seq) == 512
    assert len(label) == 20<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>from pytorch_pretrained_bert.modeling import *
class BertTest(nn.Module):
    def __init__(self, config):
        super(BertTest, self).__init__()
        self.bert = BertForPreTraining(config)
        
    def forward(self, masked_seq, attention_mask, indicies, masked_lm_labels, token_ids=None):
        prediction_scores, _ = self.bert(masked_seq, 
                                         token_type_ids=token_ids, 
                                         attention_mask=attention_mask, 
                                         masked_lm_labels=masked_lm_labels)
        return prediction_scores
from pytorch_pretrained_bert.modeling import BertConfig
bert_config = BertConfig.from_json_file(BERT_CONFIG_FILE)
bert_test = BertTest(bert_config)
bert_test.bert.load_state_dict(torch.load(BERT_WEIGHTS, map_location='cpu'))
bert_test.to('cuda')
bert_test.bert.bert.embeddings.word_embeddings.weight[0][:10]
for n, p in bert_test.bert.named_parameters():
    if p.requires_grad: 
        p.requires_grad=False
sum(p.numel() for p in bert_test.parameters() if p.requires_grad)
# test_ds = BertDataSet("../../../bert/weights/eval_full/20181217_073706_412768.npy", SEQUENCE_LENGTH)
seq = (np.load("../../../bert/weights/eval_full/20181217_073706_412768.npy")[2][165])
seqs = np.repeat(np.expand_dims(seq, axis=0), 16, axis=0)
test_ds = BertDataSet(seqs, SEQUENCE_LENGTH)
validate_ds(test_ds)
dl = DataLoader(test_ds, batch_size=16, shuffle=False)
def accuracy(input, targs):
    n = targs.shape[0]
    input = input.argmax(dim=-1).view(n,-1)
    targs = targs.view(n,-1)
    return (input==targs).float().mean(dim=1)
model = bert_test
#loss_func = nn.CrossEntropyLoss(reduction='none')
loss_func = bert_loss
model.eval()
xbs, ybs, preds, loss, acc = [None, None, None, None], None, None, None, None
with torch.no_grad():
    val_losses, val_acc, steps = [],[], 0
    for xb,yb in NBProgressBar(dl):
        xb, yb = to_device((xb, yb), torch.device('cuda'))
        out = model(*xb)
        for i in range(len(xb)):
            xbs[i] = append(xbs[i], xb[i])
        ybs = append(ybs, yb)
        pred = torch.nn.functional.log_softmax(batched_index_select(out,1, xb[2]), dim=-1).reshape([-1, 20, 22])
        preds = append(preds, pred)
        loss = append(loss, loss_func(pred, yb, bert_config.vocab_size).mean(dim=1))
        acc = append(acc, accuracy(pred, yb))
        steps = steps + 1
        if steps == 100:
            break
             
loss.mean(), acc.mean()
accuracies = to_np(acc)
losses = to_np(loss)
predictions = to_np(preds.transpose(1,2))
masked_seq, mask, indicies, seq = [to_np(x) for x in xbs]
labels = to_np(ybs)<jupyter_output><empty_output><jupyter_text>Accuracy<jupyter_code>unique, counts=np.unique(accuracies, return_counts=True)
dict(zip(unique, counts))
import matplotlib.pyplot as plt
plt.hist(accuracies, bins='auto')
plt.show()<jupyter_output><empty_output><jupyter_text>Losses<jupyter_code>import matplotlib.pyplot as plt
plt.hist(losses, bins='auto')
plt.show()<jupyter_output><empty_output><jupyter_text>Accuracy per amino acid<jupyter_code>list(zip(np.unique(labels, return_counts=True)))
unique_g, counts_g = np.unique(predictions.argmax(axis=2), return_counts=True)
unique_t, counts_t = np.unique(labels, return_counts=True)
dict(zip(unique_g, counts_g)), dict(zip(unique_t, counts_t))
predictions.shape, labels.shape
data = np.column_stack((predictions.argmax(axis=1).flatten(),labels.flatten(),np.repeat(accuracies, 20, axis=0)))
amino_acids = pd.DataFrame(data, columns=["Guess", "Actual", "Accuracy"])
amino_acids.shape
amino_acids["Correct"] = amino_acids["Guess"] == amino_acids["Actual"]
pd.concat([amino_acids.groupby(by="Guess").agg({"Correct":['mean','count'], "Accuracy": ["mean"]}), 
           amino_acids.groupby(by="Actual").agg({"Correct":['mean','count'], "Accuracy": ["mean"]})]
          ,axis=1)
amino_acids['Correct'].sum()
df_top_freq = amino_acids[~amino_acids['Correct']].groupby(["Guess", "Actual"])['Correct'].agg(
    {"code_count": len}).sort_values(
    "code_count", ascending=False).head(20).reset_index()
df_top_freq
df_top_freq = amino_acids[(~amino_acids['Correct']) & (amino_acids['Accuracy'] >= 0.5)].groupby(["Guess", "Actual"])['Correct'].agg(
    {"code_count": len}).sort_values(
    "code_count", ascending=False).head(11).reset_index()
df_top_freq
df_top_freq = amino_acids[(~amino_acids['Correct']) & (amino_acids['Accuracy'] < 0.2)].groupby(["Guess", "Actual"])['Correct'].agg(
    {"code_count": len}).sort_values(
    "code_count", ascending=False).head(11).reset_index()
df_top_freq
df_top_freq = amino_acids[(amino_acids['Correct']) & (amino_acids['Accuracy'] < 1.5)].groupby(["Guess", "Actual"])['Correct'].agg(
    {"code_count": len}).sort_values(
    "code_count", ascending=False).head(20).reset_index()
df_top_freq<jupyter_output>c:\users\donatas\appdata\local\programs\python\python36\lib\site-packages\ipykernel_launcher.py:2: FutureWarning: using a dict on a Series for aggregation
is deprecated and will be removed in a future version<jupyter_text>Analysis<jupyter_code>worst = np.argmin(accuracies)
best = np.argmax(accuracies)
accuracies[worst], accuracies[best]
def get_table(index):
    print("Accuracy:{} | Loss: {}".format(accuracies[index], losses[index]))
    in_seq, original = "", ""
    for i, a in enumerate(masked_seq[index]):
        if a == 0:
            break
        if a == 21:
            position = np.argwhere(indicies[index] == i)[0][0]
            in_seq = in_seq + "("+ID_TO_AMINO_ACID[predictions[index].argmax(axis=0)[position]]+")"
            original = original + "("+ID_TO_AMINO_ACID[seq[index][i]]+")"
        else:
            in_seq = in_seq + ID_TO_AMINO_ACID[a]
            original = original + ID_TO_AMINO_ACID[seq[index][i]]

    print(in_seq) 
    print(original)
    data = list(zip(labels[index], predictions[index].argmax(axis=0), 
                    np.transpose(predictions[index].argsort(axis=0))[:, ::-1],
                    np.sort(np.transpose(predictions[index]))[:, ::-1]))
    df = pd.DataFrame(data, columns=['Correct', 'Guess', 'Other guess', 'Probs'], index=indicies[index]) 
    df = df[~df.index.duplicated(keep='first')]
    return df.sort_index()
len(np.where(accuracies <0.2)[0])
len(np.where(accuracies >0.7)[0])
arr = seq[np.where(accuracies <0.2)]
(arr ==0).argmax(axis=1).mean()
arr = seq[np.where(accuracies >0.8)]
(arr ==0).argmax(axis=1).mean()
losses[np.where(accuracies <0.2)].mean()
losses[np.where(accuracies >0.8)].mean()
losses
losses.argsort(axis=0)[:10], losses.argsort(axis=0)[-10:]
losses[accuracies.argsort(axis=0)[:5]], losses[accuracies.argsort(axis=0)[-5:]]
"".join([ID_TO_AMINO_ACID[x] for x in  seq[0] if x != 0])
for i in zip(losses.argsort(axis=0)[:10], losses.argsort(axis=0)[-10:]):#range(256):
    good = np.sort(np.transpose(predictions[i[0]]))[:, ::-1]
    bad = np.sort(np.transpose(predictions[i[1]]))[:, ::-1]
    g, b = 0, 0
    for p in zip(good, bad):
        g += p[0][0]-p[0][-3]    
        b += p[1][0]-p[1][-3]
    print("{}:\t{}({})\t{}:\t{}({})".format(i[0],g, losses[i[0]], i[1], b, losses[i[1]]))
ps = np.sort(np.transpose(predictions[72]))[:, ::-1]
for p in ps:
    print(p) 
labels[0]
pd.set_option('display.width', 1000)
pd.set_option('max_colwidth', 350)
get_table(8)
"".join([ID_TO_AMINO_ACID[x] for x in  seq[165] if x != 0])
get_table(113)
from Bio import Seq
from Bio.Blast import NCBIWWW
from Bio.Blast import NCBIXML

seq_string="MSRSSPDQLTRFLAQVQGRPKKGLSQNFLIDGNILRKILAVSCVEAGDWVLEIGPGFGALTEVLVNQGAHVVALEKDPMFEETLKQLPIDLEITDACKYPLSQLEEKGWQGKGRVVANLPYHITTPLLTKLFLEVPNQWKTVTVMMQDEVARRITAQPGGKEYGSLTIFLQFFADVRYAFKVSPGCFFPKPQVSSAVVHMTVKETFPLETSLHQKFFSLTRAAFGQRRKLLANALKDLYPKELVFSALNQLNFSEKTRPETLSLNEYLQLFHLLSSNA"
seq_string = seq_string.replace('0', '')
testSeq = Seq.Seq(seq_string)
fasta = ('>\n' + '\n>\n'.join(["".join([ID_TO_AMINO_ACID[x] for x in  seq[i] if x != 0]) for i in losses.argsort(axis=0)[-20:-1]]))
losses.argsort(axis=0)[20:40][3]
seq[469]
print(fasta)
result_handle = NCBIWWW.qblast(program="blastp", database="nr", perc_ident=95, sequence=fasta)
all_titles = ""
identities = []
blast_records = NCBIXML.parse(result_handle)
for blast_record in blast_records:
    for alignment in blast_record.alignments:
        for hsp in alignment.hsps:
            identities.append(hsp.identities/hsp.align_length)
print(identities)
sum(identities)/len(identities)
sum(counts)/len(counts)
len(blast_records)
len(alignment.hsps)
len("MNQGAHVVALEKDSMLEETLKQLPIHLEITDACKYPLSQLQDQGWQGKGRVVANLPYHITTPLLRKLFLEAPNQWKTVTVMIQDEVARRITAQPGGKEYGSLTIFLQFFVDVHYAFKVSPGCFLPKPQVASAVVHMTVKENFPLEEPLRTKFFSLTRAAFGQRRKLLANALKDLYPKERVFEALSQLHFSDKTRPETLSLDDYLKLFYLLS")
alignment.title
hsp.align_length
        for hsp in alignment.hsps:
            print('****Alignment****')
            print('Identity:', hsp.identities)
            print('Score:', hsp.score)
            print('e value:', hsp.expect)
            print(hsp.query)
            print(hsp.match)
            print(hsp.sbjct)<jupyter_output>****Alignment****
Identity: 183
Score: 978.0
e value: 9.83642e-132
VNQGAHVVALEKDPMFEETLKQLPIDLEITDACKYPLSQLEEKGWQGKGRVVANLPYHITTPLLTKLFLEVPNQWKTVTVMMQDEVARRITAQPGGKEYGSLTIFLQFFADVRYAFKVSPGCFFPKPQVSSAVVHMTVKETFPLETSLHQKFFSLTRAAFGQRRKLLANALKDLYPKELVFSALNQLNFSEKTRPETLSLNEYLQLFHLLS
+NQGAHVVALEKD M EETLKQLPI LEITDACKYPLSQL+++GWQGKGRVVANLPYHITTPLL KLFLE PNQWKTVTVM+QDEVARRITAQPGGKEYGSLTIFLQFF DV YAFKVSPGCF PKPQV+SAVVHMTVKE FPLE  L  KFFSLTRAAFGQRRKLLANALKDLYPKE VF AL+QL+FS+KTRPETLSL++YL+LF+LLS
MN<KEY>
{'fraction_non_alphanumeric': 0.1562201164913501, 'fraction_numerical': 0.023037468486481787, 'mean_word_length': 4.431539187913126, 'pattern_counts': {'":': 8, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '306000', 'license': 'MIT', 'n_tokens_mistral': 4714, 'n_tokens_neox': 4401, 'n_words': 666, 'path': 'notebooks/amino_acid_embeddings/test_bert.ipynb', 'repo_name': 'duttaprat/proteinGAN', 'size': '11707'}