<jupyter_start><jupyter_text>Classification of Lab Samples<jupyter_code>import init
from common import constants as cn
from common.trinary_data import TrinaryData
from common.data_provider import DataProvider
from common_python.plots import util_plots
from common_python.classifier import classifier_ensemble
from common_python.classifier import classifier_collection
from common_python.classifier.classifier_ensemble_random_forest import ClassifierEnsembleRandomForest
from common import transform_data

import collections
import os
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn
from sklearn.decomposition import PCA
from sklearn import svm
from sklearn.model_selection import cross_val_score

%matplotlib inline
data = TrinaryData()
data.df_X.head()
df_sampleAM = transform_data.trinaryReadsDF(
    csv_file="AM_MDM_Mtb_transcripts_DEseq.csv")
df_sampleAW = transform_data.trinaryReadsDF(
"AW_plus_v_AW_neg_Mtb_transcripts_DEseq.csv")
df_sampleAM = df_sampleAM.T
df_sampleAM.head()
df_sampleAW = df_sampleAW.T
df_sampleAW.head()<jupyter_output><empty_output><jupyter_text>Classification ValidationsClassify T2-T25 and see if result is same as original class<jupyter_code>provider = DataProvider(is_normalize=False)
provider.do()
def getTimeSample(time_index):
    df0 = provider.dfs_cenered_adjusted_read_count[0]
    num = len(provider.dfs_cenered_adjusted_read_count)
    ser = pd.Series(np.repeat(0, len(df0.index)), index=df0.index)
    for idx in range(num):
        ser += provider.dfs_cenered_adjusted_read_count[idx][time_index]
    df = pd.DataFrame(ser/num)
    df_result = transform_data.trinaryReadsDF(df_sample=df)
    return df_result.T
getTimeSample(2).head()
provider.dfs_cenered_adjusted_read_count[1].head()
svm_ensemble = classifier_ensemble.ClassifierEnsemble(
        classifier_ensemble.ClassifierDescriptorSVM(), filter_high_rank=15, size=30)
df_X = data.df_X.copy()
df_X.columns = data.features
svm_ensemble.fit(df_X, data.ser_y)
if False:
    data.ser_y
    df_expected = pd.DataFrame(data.ser_y)
    df_expected.columns = ["columns"]
    df_expected["values"] = 1.0
    dff_expected = df_expected.pivot(columns="columns", values="values")
    dff_expected = dff_expected.fillna(0)
    dff_expected.index = [int(v[1:]) for v in dff_expected.index]
    dff_expected 
    df_diff = (dff_expected - dfff)
    df_diff = df_diff.applymap(lambda v: np.abs(v))
    df_diff.sum().sum()/len(df_diff)<jupyter_output><empty_output><jupyter_text>Classification of Lab Samples<jupyter_code>svm_ensemble = classifier_ensemble.ClassifierEnsemble(
        classifier_ensemble.ClassifierDescriptorSVM(), filter_high_rank=15, size=30)
df_X = data.df_X.copy()
df_X.columns = data.features
svm_ensemble.fit(df_X, data.ser_y)
svm_ensemble.predict(df_sampleAM)
svm_ensemble.predict(df_sampleAW)<jupyter_output><empty_output><jupyter_text>Comparisons With Random Forest<jupyter_code>dummy_columns = [v+"-" for v in data.features]
truncated_columns = [f[0:f.index("-")] for f in dummy_columns]
len(truncated_columns)
df_X = data.df_X.copy()
df_X.columns = truncated_columns
clf = ClassifierEnsembleRandomForest(size=150, filter_high_rank=30)
clf.fit(df_X, data.ser_y)
df_sampleAM[truncated_columns].head()
clf.predict(df_sampleAM[clf.columns])
,Resuscitation,Transition,StageII,Stage1a,Stage1b
AM_D20_1,0.0,0.3,0.0,0.0,0.7
AM_D20_3,0.0,0.3,0.0,0.0,0.7
AM_D20_4,0.0,0.0,0.0,0.0,1.0
AM_D20_5,0.0,0.3,0.0,0.0,0.7
MDM_D20_1,0.0,0.3,0.0,0.0,0.7
MDM_D20_3,0.0,0.3,0.0,0.0,0.7
MDM_D20_4,0.0,1.0,0.0,0.0,0.0
MDM_D20_5,0.0,0.3,0.0,0.0,0.7

clf.predict(df_sampleAW[truncated_columns])<jupyter_output><empty_output><jupyter_text>Random Forest Validations<jupyter_code>set(clf.columns).difference(data.features)
df_X = data.df_X.copy()
df_X.columns = data.features
df_X[clf.columns]
def calcCVR(num_features, num_classifiers, num_iterations=10):
     clf = ClassifierEnsembleRandomForest(
         filter_high_rank=num_features, size=num_classifiers)
     return classifier_collection.ClassifierCollection.crossValidateByState(
         clf, df_X, data.ser_y, num_iterations)
calcCVR(30, 100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10859619552130989, 'fraction_numerical': 0.0339513604623164, 'mean_word_length': 6.735567970204841, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '209131', 'license': 'MIT', 'n_tokens_mistral': 1663, 'n_tokens_neox': 1611, 'n_words': 237, 'path': 'notebooks/Classification of Lab Samples.ipynb', 'repo_name': 'uwescience/expression_state', 'size': '4153'}