<jupyter_start><jupyter_text>Here is where we would select difference scalers<jupyter_code>scaler = None

if scaler is None:
    scaler = StandardScaler()
    scaler.fit(genes_df)

genes_df_scaled = pd.DataFrame(scaler.transform(genes_df),index=genes_df.index,columns=genes_df.columns).fillna(0)
genes_df_scaled.head()
data = genes_df_scaled.join(df[['Subtype']])
pca_data = data.set_index('Subtype').loc['LumB']
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
pca.fit(pca_data)
pca.explained_variance_ratio_
data_transformed = pd.DataFrame(pca.transform(data.drop('Subtype',axis=1))).join(df[['Subtype']])
data_transformed.columns = ["PC1","PC2","Subtype"]
data_transformed
import altair as alt
alt.Chart(data_transformed).mark_circle(size=60).encode(
    x='PC1',
    y='PC2',
    color='Subtype:N'
    #tooltip=['Name', 'Origin', 'Horsepower', 'Miles_per_Gallon']
)#.interactive()<jupyter_output><empty_output><jupyter_text>Look at the genes individually<jupyter_code>source = genes_df_scaled.join(df[['Subtype','target']]).melt(id_vars=['Subtype','target'])
source.columns = ["Subtype","target","Gene","Value"]
counts = source.groupby('Subtype')['target'].count().to_frame()
counts.columns = ['Count']
source = source.set_index('Subtype').join(counts).reset_index()
# use the url approach to keep notebook smaller
url = 'Luminal_B_101_to_graph_fig1.json'
pyknowledge.plot.prepare_url(source,url)
source
pyknowledge.plot.histogram_1(url,'Value','Subtype','Gene')<jupyter_output><empty_output><jupyter_text>Knowledge OverviewThese genes are deemed important based on previous research into the LuminalB subtype. The pattern is unspecified and is a set of genes. As previously mentioned, the relationship between pairs of samples is the input to the modified cross-entropy loss function. This requires defining a distance between samples. We define the distance as the number of the number of LumB genes within one standard deviation of one another.<jupyter_code># get us ready to parallize distance
ncores = 7
pair_ixs = pyknowledge.common.get_pair_inxs(df.shape[0])
chunks_pair_ixs = list(pyknowledge.common.divide_chunks(pair_ixs,pyknowledge.common.calc_len_chunk(len(pair_ixs),ncores)))
from joblib import Parallel, delayed
helper = lambda distance_func: pd.concat(Parallel(n_jobs=-1)(delayed(pyknowledge.distance.chunk_distance)(genes_df_scaled,chunk,distance_func) for chunk in chunks_pair_ixs))
DL1 = helper(pyknowledge.distance.L1)
DL2 = helper(pyknowledge.distance.L2)
DFSIGN = helper(pyknowledge.distance.FSIGN)
alt.data_transformers.disable_max_rows()
Ds = {'DL1':DL1,'DL2':DL2,'DFSIGN':DFSIGN}
distances = {}
for key in Ds.keys():
    distances[key] = pd.DataFrame(list(Ds[key]['distance']),columns=[key],index=Ds[key].index)
for key in distances.keys():
    distances[key] = pyknowledge.distance.remove_self_ref(distances[key]).dropna()
for key in distances.keys():
    distances[key] = pyknowledge.common.add_labels(df,distances[key])
distances[key].head()<jupyter_output><empty_output><jupyter_text>Prepare our distances for plotting<jupyter_code>subtypes = df.Subtype.unique()
for subtype in subtypes:
    for key in distances.keys():
        source = pyknowledge.plot.prepare_source1(df,distances[key],subtype,subtypes)
        group_counts = source.groupby('label1_label2')[key].count().to_frame()
        group_counts.columns = ['Count']
        source = source.set_index('label1_label2').join(group_counts).reset_index()
        url = f"{subtype}_{key}_hist.json"
        source.to_json(url,orient='records')
source.head()
subtypes = df.Subtype.unique()
for key in distances.keys():
    for subtype in ["LumA","LumB"]:
        url = f"{subtype}_{key}_hist.json"
        g = alt.Chart(url).transform_calculate(
            pct='1 / datum.Count'
        ).mark_area(
            opacity=0.3,
            interpolate='step'
        ).encode(
            alt.X('%s:Q'%key, bin=alt.Bin(maxbins=50)),
            alt.Y('sum(pct):Q', axis=alt.Axis(format='%'),stack=None),
            alt.Color('label1_label2:N',title='Label - Label')
        )
        display(Markdown(f'### {key}: {subtype}'))
        display(g)<jupyter_output><empty_output><jupyter_text>Old below<jupyter_code>alt.Chart(source.sample(n=10000)).mark_area(
    opacity=0.3,
    interpolate='step'
).encode(
    alt.X('DL2:Q', bin=alt.Bin(maxbins=50)),
    alt.Y('count()', stack=None),
    alt.Color('label1_label2:N')
)
g.save('LumB_LumB_distances.png')
print('test')
constant = 'LumA'
distance1_12 = get_distance(distance1,constant,constant)
for n in df['Subtype'].unique():
    if n != constant:
        distance1_12 = distance1_12.append(get_distance(distance1,constant,n))

alt.Chart(distance1_12.reset_index()).mark_area(
    opacity=0.3,
    interpolate='step'
).encode(
    alt.X('D1:Q', bin=alt.Bin(maxbins=100)),
    alt.Y('count()', stack=None),
    alt.Color('label1_label2:N')
)
constant = 'LumB'
distance1_12 = get_distance(distance1,constant,constant)
for n in df['Subtype'].unique():
    if n != constant:
        distance1_12 = distance1_12.append(get_distance(distance1,constant,n))

alt.Chart(distance1_12.reset_index()).mark_area(
    opacity=0.3,
    interpolate='step'
).encode(
    alt.X('D1:Q', bin=alt.Bin(maxbins=100)),
    alt.Y('count()', stack=None),
    alt.Color('label1_label2:N')
)
D2 = pd.concat(Parallel(n_jobs=-1)(delayed(chunk_distance)(chunk,distance2) for chunk in chunks_pair_ixs))<jupyter_output><empty_output><jupyter_text>Set some minimum values for sparsity<jupyter_code>min_value = genes_df_scaled.shape[1]*0.3 # Choose a minimum value
min_value
D2.plot.hist()
D2.unstack()
D2_visual = D2.copy() # or D2
D2_visual.loc[D2.distance <= min_value] = 0 # just to make it more sparse
D2_visual=D2_visual.fillna(0).unstack()
D2_visual.index.name='index'
D2_visual.columns = D2_visual.index
D2_visual = D2_visual/genes_df_scaled.shape[1]
D2_visual.to_csv(f'graphs/D2_gt_{min_value}.csv')
!ls graphs
min_value
D2_visual
min_value = genes_df_scaled.shape[1]*0.6 # Choose a minimum value
min_value
D1_visual = D1.copy() # or D2
D1_visual.loc[D1.distance <= min_value] = 0 # just to make it more sparse
D1_visual = D1_visual.fillna(0).unstack()
D1_visual.index.name='index'
D1_visual.columns = D1_visual.index
D1_visual.to_csv(f'graphs/D1_gt_{min_value}.csv')
D1_visual#.dropna()<jupyter_output><empty_output><jupyter_text>Test out the knowledge functionality<jupyter_code>import knowledge
k = knowledge.Knowledge()
k.distances
#import tensorflow as tf
def cosine_sim(logits):
    x_ = tf.expand_dims(logits, 0)
    y_ = tf.expand_dims(logits, 1)
    xN = tf.sqrt(tf.reduce_sum(tf.reshape((x_[None]*y_[:,None])*0+x_[None]**2, [-1, logits.shape[1]]),axis=1))
    yN = tf.sqrt(tf.reduce_sum(tf.reshape((x_[None]*y_[:,None])*0+y_[:,None]**2, [-1, logits.shape[1]]),axis=1))
    #z = tf.reshape(cosine_loss(y_true, y_pred), [-1, logits.shape[1]]), reduction_indices=[1])),axis=0)    
    z = tf.reduce_sum(tf.reshape((x_[None]*y_[:,None]), [-1, logits.shape[1]]),axis=1)/(xN*yN)
    return z

#sess = tf.Session()
#logits = tf.constant(genes_df_scaled.values[:3,:], dtype=tf.float32)
#A = sess.run(cosine_sim(logits))
A
import tensorflow as tf
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))

from sklearn.metrics.pairwise import cosine_similarity

pd.DataFrame(cosine_similarity(genes_df_scaled.values[:3,:])).stack()<jupyter_output><empty_output><jupyter_text>This is looking good. We just need to do all combinations and then set the index.<jupyter_code>np.reshape(A,(3,3))



D_visual.dropna()
from IPython.display import Image

import networkx as nx

G = nx.Graph()
temp = D_visual.loc[D_visual.distance > 0].reset_index()
for n in np.unique(list(temp.index1) + list(temp.index2)):
    c = df.loc[n,'color']
    G.add_node(n,color=c,style='filled',fillcolor=c)

for index,values in D_visual.loc[D_visual.distance > 0].iterrows():
    n1,n2 = index
    G.add_edge(n1,n2,weight=genes_df_scaled.shape[1]-values.loc['distance'])
def save(A,file="graph.png"):
    g = A.draw(format=file.split(".")[-1], prog='dot')
    open(file,"wb").write(g)
    return Image(g)

pos = nx.drawing.nx_agraph.graphviz_layout(G, prog='dot')
A = nx.nx_agraph.to_agraph(G)
A.graph_attr["rankdir"] = "LR"
# draw it in the notebook
save(A)<jupyter_output><empty_output><jupyter_text>The additional terms to the loss function are broken into label-label, label-unknown, and unknown-unknown. We will model the label-label case first where mw stands for the match weight, mm stands for the mismatch weight.label1  label2 w---------------------------LumB    LumB    mwUnknown LumB    0|mmwLumB    Unknown 0|mmwUnknown Unknown 0If both samples are LumB subtype, then the weight to the loss function is mw > 0. If only a single sample is LumB subtype, then we may also set the mmw = 0 or mmw > 0. If you set mmw > 0, then you may use labeled and unlabelled.What I would like to test:1. Does this improve performance and/or convergence of DeepType (at least during the supervised learning phase)2. When running repeated experiments with varying random seed and/or parameters, do we have more consistency of results?   Let's work on a sample model<jupyter_code>nsample = 10
genes_df_scaled_sampled = genes_df_scaled.sample(n=nsample,random_state=0)
samples = genes_df_scaled_sampled.join(df[['Subtype','target']])
(100*samples[['CCND1','FAM83H','C17orf28','FAM83D']]).round().astype(int).join(df[['Subtype','target']])
import itertools

def get_diff(samples,y,cls):
    diff_samples = pd.DataFrame(columns = list(genes_df_scaled_sampled.columns)+["Subtype_1","Subtype_2","ix1","ix2"])
    diff_samples = diff_samples.set_index(["ix1","ix2"])
    for ix1,ix2 in itertools.permutations(samples.index,2):
        if ix1 == ix2:
            continue
        new_sample = samples.loc[ix1,genes_df_scaled_sampled.columns] - samples.loc[ix2,genes_df_scaled_sampled.columns]
        new_sample = new_sample.to_frame().T
        #new_sample["label_1"] = samples.loc[ix1,"label"]
        #new_sample["label_2"] = samples.loc[ix2,"label"]
        new_sample["Subtype_1"] = y.loc[ix1] #samples.loc[ix1,"Subtype"]
        new_sample["Subtype_2"] = y.loc[ix2] #samples.loc[ix2,"Subtype"]
        new_sample["ix1"] = ix1
        new_sample["ix2"] = ix2
        new_sample = new_sample.set_index(["ix1","ix2"])
        diff_samples = diff_samples.append(new_sample)
        
    diff_samples['target'] = "MatchIgnore" # Case 4
    diff_samples.loc[(diff_samples['Subtype_1'] != cls) & (diff_samples['Subtype_2'] == cls),'target'] = "Mismatch1"
    diff_samples.loc[(diff_samples['Subtype_1'] == cls) & (diff_samples['Subtype_2'] != cls),'target'] = "Mismatch2"
    diff_samples.loc[(diff_samples['Subtype_1'] == cls) & (diff_samples['Subtype_2'] == cls),'target'] = "Match"
    diff_samples['target'] = diff_samples['target'].astype('category')
    return diff_samples<jupyter_output><empty_output><jupyter_text>ID label1 label2 f(id1,id2)---------------------------1  LumB   LumB   Match2  Other  LumB   Mismatch13  LumB   Other  Mismatch24  Other  Other  MatchIgnore  Now construct the model<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold, cross_val_score, RepeatedStratifiedKFold, StratifiedShuffleSplit
import imblearn

oversample = imblearn.over_sampling.RandomOverSampler()
undersample = imblearn.under_sampling.RandomUnderSampler()

#X,y = diff_samples[genes_df_scaled_sampled.columns], diff_samples['target']
X,y = samples[genes_df_scaled_sampled.columns], samples['Subtype']

#X, y = oversample.fit_resample(X_orig, y_orig)
#cv = RepeatedStratifiedKFold(n_splits=10, n_repeats = 2, random_state=1)

#print("f1_micro")
#print(cross_val_score(clf, X, y, cv=cv, scoring='f1_micro'))
#print("f1_macro")
#print(cross_val_score(clf, X, y, cv=cv, scoring='f1_macro'))
#print("f1_weighted")
#print(cross_val_score(clf, X, y, cv=cv, scoring='f1_weighted'))
from sklearn.metrics import classification_report
clf = RandomForestClassifier(max_depth=2, random_state=0)
from sklearn.tree import DecisionTreeClassifier
#clf = DecisionTreeClassifier()

from sklearn.neighbors import KNeighborsClassifier
clf = KNeighborsClassifier(n_neighbors=3)

from sklearn.svm import OneClassSVM
from sklearn.neighbors import LocalOutlierFactor

labels = y.unique()
cnt = 1

all_scores_df = []
random_states = [0]
for random_state in random_states:
    cv = StratifiedShuffleSplit(n_splits=2, test_size=0.3, random_state=random_state)
    scores = {}
    for train_index, test_index in cv.split(X, y):
        X_train, X_test = X.iloc[train_index], X.iloc[test_index]
        y_train, y_test = y.iloc[train_index], y.iloc[test_index]

        print(f'Round {cnt}')
        for cls in labels:
            diff_samples = get_diff(X_train,y_train,cls)
            Xdiff = diff_samples.drop(["target","Subtype_1","Subtype_2"],axis=1)
            ydiff = diff_samples['target']
            Xone = Xdiff.loc[ydiff == "Match"]
            
            clf = OneClassSVM(gamma='auto')
            #clf = LocalOutlierFactor(n_neighbors=2,novelty=True)
            clf.fit(Xone)

            #X_over, y_over = oversample.fit_resample(X_train, y_train)
            #X_under, y_under = undersample.fit_resample(X_train, y_train)
            #clf.fit(X_under,y_under)

            diff_samples_test = get_diff(X_test,y_test,cls)
            Xdiff_test = diff_samples_test.drop(["target","Subtype_1","Subtype_2"],axis=1)
            ydiff_test = diff_samples_test['target']
            decf = clf.decision_function(Xdiff_test)
            new_data =  pd.DataFrame(decf,index=ydiff_test.index).join(ydiff_test)
            if cls not in scores:
                scores[cls] = new_data
            else:
                scores[cls].append(new_data)
        cnt+=1
Xone
scores['LumA']
scores['LumA'].loc[scores['LumA'].target=='Match'].mean()
scores['LumA'].loc[scores['LumA'].target!='Match'].mean()
scores_df_target.loc[scores_df_target.Subtype!='LumB']
means = pd.DataFrame(columns=["Match","Mismatch","Subtype"]).set_index("Subtype")
for label in labels:
    mismatch = scores_df_target.loc[scores_df_target.Subtype!=label][label].mean()
    match = scores_df_target.loc[scores_df_target.Subtype==label][label].mean()
    means = means.append(pd.Series([match,mismatch],name=label,index=means.columns))
means
scores_df_target.loc[scores_df_target.Subtype=='Basal']['Basal'].mean()
clf.predict_proba(diff_samples[genes_df_scaled_sampled.columns].iloc[[0]])
import numpy as np
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
pca.fit(genes_df_scaled)

print(pca.explained_variance_ratio_)

print(pca.singular_values_)
scores = pca.transform(genes_df_scaled)
scores = pd.DataFrame(scores,columns=["PC1","PC2"])
scores['Subtype'] = df.label.map({1:'Basal',2:'HER2+',3:'LumA',4:'LumB',5:'Normal Like',6:'Normal'})
scores
import altair as alt
points = alt.Chart(scores).mark_point().encode(
    x='PC1:Q',
    y='PC2:Q',
    color='Subtype:N'
)
points
import tensorflow as tf
def cosine_sim(logits):
    x_ = tf.expand_dims(logits, 0)
    y_ = tf.expand_dims(logits, 1)
    xN = tf.sqrt(tf.reduce_sum(tf.reshape((x_[None]*y_[:,None])*0+x_[None]**2, [-1, logits.shape[1]]),axis=1))
    yN = tf.sqrt(tf.reduce_sum(tf.reshape((x_[None]*y_[:,None])*0+y_[:,None]**2, [-1, logits.shape[1]]),axis=1))
    #z = tf.reshape(cosine_loss(y_true, y_pred), [-1, logits.shape[1]]), reduction_indices=[1])),axis=0)    
    z = tf.reduce_sum(tf.reshape((x_[None]*y_[:,None]), [-1, logits.shape[1]]),axis=1)/(xN*yN)
    return z
import tensorflow as tf
def cosine_sim(logits):
    x_ = tf.expand_dims(logits, 0)
    y_ = tf.expand_dims(logits, 1)
    xN = tf.sqrt(tf.reduce_sum(tf.reshape((x_[None]*y_[:,None])*0+x_[None]**2, [-1, logits.shape[1]]),axis=1))
    yN = tf.sqrt(tf.reduce_sum(tf.reshape((x_[None]*y_[:,None])*0+y_[:,None]**2, [-1, logits.shape[1]]),axis=1))
    #z = tf.reshape(cosine_loss(y_true, y_pred), [-1, logits.shape[1]]), reduction_indices=[1])),axis=0)    
    z = tf.reduce_sum(tf.reshape((x_[None]*y_[:,None]), [-1, logits.shape[1]]),axis=1)/(xN*yN)
    return z

sess = tf.Session()
logits = tf.constant(genes_df_scaled.values, dtype=tf.float32)
A = sess.run(cosine_sim(logits))
m = int(np.sqrt(len(A)))
A = np.reshape(A,(m,m))
pd.DataFrame(A).to_csv('graph_Luminal_B_101_BRCA1View20000.csv')
import networkx as nx
G=nx.from_numpy_matrix(A)
from IPython.display import Image

def save(A):
    open("graph.png","wb").write(A.draw(format='png', prog='dot'))

pos = nx.drawing.nx_agraph.graphviz_layout(G, prog='neato')
#print(edge_labels)
# Modify node fillcolor and edge color.
#D.node_attr.update(color='blue', style='filled', fillcolor='yellow')
#D.edge_attr.update(color='blue', arrowsize=1)
A = nx.nx_agraph.to_agraph(G)
A.graph_attr["rankdir"] = "LR"
# draw it in the notebook
save(A)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15567765567765568, 'fraction_numerical': 0.01979203592106818, 'mean_word_length': 5.311334824757644, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '486243', 'license': 'MIT', 'n_tokens_mistral': 6442, 'n_tokens_neox': 6230, 'n_words': 1046, 'path': 'metabric/Luminal_B_101/Luminal_B_101_to_graph.ipynb', 'repo_name': 'calpoly-bioinf/knowledgelib', 'size': '16926'}