<jupyter_start><jupyter_text>Dataset Overview<jupyter_code>path = "../datasets/combined_Bat_Cat_flu.fa"
gene,gene_len = read_fasta(path)
max(gene_len)
min(gene_len)
sum(gene_len)/len(gene_len)<jupyter_output><empty_output><jupyter_text>Unsupervised<jupyter_code>bat_len = len(get_gene_sequences("../datasets/bat_flu.fa"))
cat_len = len(get_gene_sequences("../datasets/cat_flu.fa"))
zeros = [0]*bat_len
labels_all = np.append(zeros, [1]*cat_len, axis=None)
path = "../datasets/combined_Bat_Cat_flu.fa"
k_min = 2 
k_max = 6 
num_class = 2 
cov_type = 'full' 
seed = 0
predictions1 = get_predictions(path,k_min,k_max,num_class,cov_type,seed)
predictions1
number = list(range(1, len(predictions1)+1))
df = pd.DataFrame(list(zip(number, predictions1)), 
               columns =['Number', 'Labels']) 
df.to_csv('predictions_unsup.csv',index = False)
df2 = pd.DataFrame(list(zip(number, labels_all)), 
               columns =['Number', 'Labels']) 
df2.to_csv('actual_labels.csv',index = False)
cal_accuracy(labels_all,predictions1)
df_0 = get_kmer_table("../datasets/combined_Bat_Cat_flu.fa",2,6)
df_0
x = df_0
pca = PCA(n_components=2)
principalComponents = pca.fit_transform(x)
principalDf = pd.DataFrame(data = principalComponents,columns = ['principal component 1', 'principal component 2'])
finalDf = pd.concat([principalDf, pd.Series(predictions1)], axis = 1)
finalDf.columns = ['principal Component 1', 'Principal Component 2','Predicted Label']
finalDf.to_csv('pca_unsup.csv',index = False)
finalDf = pd.concat([principalDf, pd.Series(labels_all)], axis = 1)
finalDf.columns = ['principal Component 1', 'Principal Component 2','True Label']
finalDf.to_csv('pca_labels.csv',index = False)
PCA_plot(df_0,predictions1,2,'figures/pca_0.png','PCA of Predictions')
PCA_plot(df_0,labels_all,2,'figures/pca_labels.png','PCA of All Labels')
tsne_plot(df_0,predictions1,'figures/tsne_0.png','TSNE of Predictions from Unsupervised Model')
tsne_plot(df_0,labels_all,'figures/tsne_labels.png','TSNE of All Labels')<jupyter_output>/Users/zhiwen/Applications/miniconda3/lib/python3.7/site-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning<jupyter_text>Semi-supervised 50%<jupyter_code>labels_50 = pd.read_csv('../datasets/labels_fifty_percent.csv')
labels_50 = pd.Series(labels_50['Labels'])
path = "../datasets/combined_Bat_Cat_flu.fa"
num_class = 2
predictions_50_0 = model_selection(path,labels_50,num_class)
cal_accuracy(labels_all,predictions_50_0)
df_50_0 = get_kmer_table("../datasets/combined_Bat_Cat_flu.fa",2,3)
tsne_plot(df_50_0,predictions_50_0,'figures/tsne_50_best.png','TSNE of Predictions from 50% Known Labels')
PCA_plot(df_50_0,predictions_50_0,2,'figures/pca_50_best.png','PCA of Predictions from 50% Known Labels')
x = df_50_0
pca = PCA(n_components=2)
principalComponents = pca.fit_transform(x)
principalDf = pd.DataFrame(data = principalComponents,columns = ['principal component 1', 'principal component 2'])
finalDf = pd.concat([principalDf, pd.Series(predictions_50_0)], axis = 1)
finalDf.columns = ['principal Component 1', 'Principal Component 2','Predicted Label']
finalDf.to_csv('pca_50%.csv',index = False)
df3 = pd.DataFrame(list(zip(number, predictions_50_0)), 
               columns =['Number', 'Labels']) 
df3.to_csv('predictions_50%_bestModel.csv',index = False)
tsne_plot(df_50_0,predictions_50_0,'figures/tsne_50.png','TSNE of Predictions from 50% Known Labels')<jupyter_output>/Users/zhiwen/Applications/miniconda3/lib/python3.7/site-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning<jupyter_text>Semi-supervised 10%<jupyter_code>labels_10 = pd.read_csv('../datasets/labels_ten_percent.csv')
labels_10 = pd.Series(labels_10['Labels'])
predictions_10_0 = model_selection(path,labels_10,num_class)
cal_accuracy(labels_all,predictions_10_0)
df_10_0 = get_kmer_table("../datasets/combined_Bat_Cat_flu.fa",2,4)
tsne_plot(df_10_0,predictions_10_0,'figures/tsne_10_best.png','TSNE of Predictions from 10% Known Labels')
PCA_plot(df_10_0,predictions_10_0,2,'figures/pca_10_best.png','PCA of Predictions from 10% Known Labels')
x = df_10_0
pca = PCA(n_components=2)
principalComponents = pca.fit_transform(x)
principalDf = pd.DataFrame(data = principalComponents,columns = ['principal component 1', 'principal component 2'])
finalDf = pd.concat([principalDf, pd.Series(predictions_10_0)], axis = 1)
finalDf.columns = ['principal Component 1', 'Principal Component 2','Predicted Label']
finalDf.to_csv('pca_10%.csv',index = False)
df4 = pd.DataFrame(list(zip(number, predictions_10_0)), 
               columns =['Number', 'Labels']) 
df4.to_csv('predictions_10%_bestModel.csv',index = False)<jupyter_output><empty_output><jupyter_text>Semi-supervised 10% only 0s<jupyter_code>labels_10_0 = pd.read_csv('../datasets/labels_ten_percent_only0s.csv')
labels_10_0 = labels_10_0['Labels']
predictions_10_only0 = model_selection(path,labels_10_0,num_class)
cal_accuracy(labels_all,predictions_10_only0)
df_10_only0 = get_kmer_table("../datasets/combined_Bat_Cat_flu.fa",2,3)
tsne_plot(df_10_only0,predictions_10_only0,'figures/tsne_10_0_best.png','TSNE of Predictions from 10% Known Labels')
PCA_plot(df_10_only0,predictions_10_only0,2,'figures/pca_10_0_best.png','PCA of Predictions from 10% Known Labels')
df5 = pd.DataFrame(list(zip(number, predictions_10_only0)), 
               columns =['Number', 'Labels']) 
df5.to_csv('predictions_10%_only0s_bestModel.csv',index = False)
x = df_10_only0
pca = PCA(n_components=2)
principalComponents = pca.fit_transform(x)
principalDf = pd.DataFrame(data = principalComponents,columns = ['principal component 1', 'principal component 2'])
finalDf = pd.concat([principalDf, pd.Series(predictions_10_only0)], axis = 1)
finalDf.columns = ['principal Component 1', 'Principal Component 2','Predicted Label']
finalDf.to_csv('pca_10%_only0.csv',index = False)
predictions1
cal_accuracy(labels_all,predictions1)
predictions_50_0
cal_accuracy(labels_all,predictions_50_0)
predictions_10_0
cal_accuracy(labels_all,predictions_10_0)
predictions_10_only0
cal_accuracy(labels_all,predictions_10_only0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13028222730739894, 'fraction_numerical': 0.04363081617086194, 'mean_word_length': 7.847503373819163, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '468165', 'license': 'MIT', 'n_tokens_mistral': 2570, 'n_tokens_neox': 2386, 'n_words': 395, 'path': 'manuscript/scripts/GMM-Tests-for-Paper.ipynb', 'repo_name': 'solislemuslab/mycovirus-website', 'size': '6555'}