<jupyter_start><jupyter_text>Drop non numeric columns<jupyter_code>data = data.drop(["3-letter", "AminoAcid"], axis =1 )
data = data.drop(["Pc", "Residue mass"], axis =1 )
#data = data.drop(["3-letter", "AminoAcid", "Chou-Fasman code for helix propensity","Chou-Fasman code for sheet propensity"], axis=1)
data.abs().max(axis=0)
std = data.std(ddof=0, axis=0)
std
data_std = data.div(std, axis=1)
data_std
data_std["Hyd"] = data_std["Hyd"] * (-1)
transposed = data_std.transpose()
transposed["0"] = [0, 0, 0, 0, 0, 0, 0, 7/1.724522]
data_std = transposed.transpose()
#data = pd.merge(left=data, right=generated, how="inner", left_index=True, right_index=True)
data_std.sort_index()
data = data_std
delta_0_center = (data.max(axis=0)+data.min(axis=0))/2 - np.abs(np.random.normal(scale=0.0001, size=data.shape[1]))
data = (data.subtract(delta_0_center, axis=1))#.div(data.std(ddof=0, axis=0), axis=1))
# data = (data.subtract(data.mean(axis=0), axis=1).div(data.std(ddof=0, axis=0), axis=1))
data = data.div(data.abs().max(axis=0)+np.abs(np.random.normal(scale=0.04, size=data.shape[1])), axis=1)
data = data + np.random.normal(scale=0.0001, size=data.shape)
data.max(), data.min()
data
# transposed = data.transpose()
# transposed[0] = -1 + np.abs(np.random.normal(scale=0.001, size=data.shape[1]))
# final = transposed.transpose()
final=data.sort_index()
final
final = final.fillna(value=0.00000000000000001)
def calculate_variation(data, col):
    one_column = data[[col]]
    one_column = one_column.append(pd.DataFrame([-0.999, 0.999], index = ["N_0", "N_1"], columns=[col]))
    one_column[col+"_min"] =one_column.sort_values(col).rolling(window=2).apply(lambda x: (x[0] - x[1])/2, raw=True)
    one_column[col +"_max"] =one_column.sort_values(col, ascending=False)[col].rolling(window=2).apply(lambda x: (x[0] - x[1])/2, raw=True)
    one_column = one_column.drop(["N_0", "N_1"], axis=0)
    return list(zip(one_column[col+"_min"].values.tolist(), one_column[col+"_max"].values.tolist()))
    #return one_column
col = "Lip"
one_column = final[[col]]
one_column = one_column.append(pd.DataFrame([-0.999, 0.999], index = ["N_0", "N_1"], columns=[col]))
one_column[col+"_min"] =one_column.sort_values(col).rolling(window=2).apply(lambda x: (x[0] - x[1])/2, raw=True)
one_column[col +"_max"] =one_column.sort_values(col, ascending=False)[col].rolling(window=2).apply(lambda x: (x[0] - x[1])/2, raw=True)
one_column = one_column.drop(["N_0", "N_1"], axis=0)
one_column
final.shape
variation = [] 
for c in final.columns:
    variation.append(calculate_variation(final, c))
np.array(variation).shape
np.array(variation).shape
final.sum(axis=1)
final_values = final.values
final_values
np.save("../data/protein/hand_crafted_embeddings_8.npy", final_values)
np.save("../data/protein/embeddings_variation_8.npy", variation)
variation[0][0]
variation[0][0][0] + variation[0][0][1] 
abs(variation[0][0][0] - variation[0][0][1])/1.95
variation[0][0][0] + variation[0][0][1] 
hist, bin_edges = np.histogram(final.values)
hist, bin_edges
np.max(final.values)
np.min(final.values)
import tensorflow as tf
tf.enable_eager_execution()
embeddings = tf.constant([[ 0.        ,  0.        ,  0.        ],
       [-0.78874256,  0.43069857, -0.47393251],
       [-0.68765258,  0.19845299, -0.64110991],
       [-0.19540625, -0.46831037, -0.43458175],
       [ 0.39009043, -0.46831037, -0.14043892],
       [-0.4217928 ,  0.31515591,  0.5829058 ],
       [-1.        ,  0.43069857, -0.93603945],
       [ 0.088746  , -0.32018371,  0.23223556],
       [ 0.49063027,  0.43069857,  0.32813161],
       [ 0.97943816, -0.1715736 ,  0.46384033],
       [ 0.74644982,  0.43069857,  0.26766763],
       [ 0.45239487,  0.1340585 ,  0.05631338],
       [-0.14891861, -0.26652163, -0.23952655],
       [-0.58491215,  0.43069857,  0.03943025],
       [ 0.3517175 , -0.26652163,  0.0522144 ],
       [ 0.68015679, -1.        ,  0.360837  ],
       [-0.70594505, -0.15329949, -0.60368124],
       [ 0.93336313,  0.00826686, -0.29583345],
       [-0.2963587 ,  0.43069857,  0.16728669],
       [ 0.12175498,  0.01397148,  1.        ],
       [-0.40501324, -0.13937636,  0.21428112]])
embeddings
acid_embeddings = tf.get_variable("acid_embeddings", shape=[21, 3],
                                              initializer=tf.constant_initializer(final[:, :3]), trainable=False)
real_x = tf.nn.embedding_lookup(acid_embeddings, [0,1,2,3,4,5,6])
real_x
emb_distances = tf.matmul(
            tf.nn.l2_normalize(acid_embeddings, axis=1),
            tf.nn.l2_normalize(real_x, axis=1),
            transpose_b=True)

emb_distances
tf.argmax(emb_distances, axis=0)
tf.nn.l2_normalize(acid_embeddings, axis=1) #[-0.78874254,  0.43069857, -0.4739325 ]
a = [[1,2,3],[50,60,70],[800, 900,1000], [-1, -2, -3]]
a = tf.expand_dims(a, axis=0)
a = tf.expand_dims(a, axis=3)
tf.squeeze(a)
x = tf.image.resize_nearest_neighbor(a, [4, 3 * 2])
tf.squeeze(x)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1976909054081426, 'fraction_numerical': 0.14766052258456552, 'mean_word_length': 4.095975232198143, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '410955', 'license': 'MIT', 'n_tokens_mistral': 2437, 'n_tokens_neox': 2078, 'n_words': 311, 'path': 'notebooks/preprocessing/amino_acid_embedding.ipynb', 'repo_name': 'duttaprat/proteinGAN', 'size': '4937'}