<jupyter_start><jupyter_text>1. Data loading and processing<jupyter_code>def read_excle(path):
    data_xls = pd.ExcelFile(path)
    data = {}
    for name in data_xls.sheet_names:
        df = pd.read_excel(path,sheet_name=name,header=0,index_col=None)
        data[name]=df
    return data
data = read_excle("finaldata.xlsx")
origin = data["internal"]
case1 = origin[(origin.label==1)]# COVID-19
case2 = origin[(origin.label==-1)] # CONTROL
# fill missing value
all_case = []
for k in case1.columns[:-1]:
    line1 = case1[k].values
    line2 = case2[k].values
    line_all = line1.tolist() + line2.tolist()
    line_clean = []
    info = []
    for x in line_all:
        if x!=-2: #not missing value
            line_clean.append(float(x))
    mean_value = np.mean(line_clean)
    for x in line_all:
        if x !=-2:
            info.append(x)
        else:
            info.append(mean_value)
    all_case.append(info)
# normalized
final_case = []
trans_data = np.array(all_case)
for i in range(len(trans_data)):
    line = trans_data[i]
    max_v = max(line)
    min_v = min(line)
    val = max_v-min_v
    final_case.append([(x-min_v)/(val) for x in line])
final_case1 = np.array(final_case).T.tolist()
all_names = [k for k in case1.columns.tolist()[:-1]]
dataset = [] # all 26 indicators
set1 = [7, 22, 8, 24, 19, 25, 9, 12, 3, 4, 6, 18, 21]
dataset2 = [] #13 statistically significant indictaors
dataset3 = [] #13 statistically insignificant indictaors
for i in range(len(final_case1)):
    if i < 75:
        da = final_case1[i] + [1]
        dataset.append(da)
        text = []
        text1 = []
        trans = np.array(final_case1[i]).T
        for j in range(len(trans)):
            if j<=1:
                continue
            else:
                if j in set1:
                    text.append(trans[j])
                else:
                    text1.append(trans[j])
        dataset2.append(np.array(text).T.tolist()+[1])
        dataset3.append(np.array(text1).T.tolist()+[1])
    else:
        da = final_case1[i] + [-1]
        dataset.append(da)
        text = []
        text1 = []
        trans = np.array(final_case1[i]).T
        for j in range(len(trans)):
            if j<=1:
                continue
            else:
                if j in set1:
                    text.append(trans[j])
                else:
                    text1.append(trans[j])
        dataset2.append(np.array(text).T.tolist()+[-1])
        dataset3.append(np.array(text1).T.tolist()+[-1])<jupyter_output><empty_output><jupyter_text>2. Testing iCov on internal held-out validation data set 2.1 On the 26 indicators internal cohort<jupyter_code>clfs = {"svm":SVC(C=5,kernel="rbf",max_iter=100000,probability=True),\
         "LR":LogisticRegressionCV(n_jobs=30,max_iter=10000),\# for classifier comparation
         "Adaboost":AdaBoostClassifier(n_estimators=200),\ # for classifier comparation
         "RandomForest":RandomForestClassifier(n_estimators=200,n_jobs=40)# for classifier comparation
}
def try_different_method(clf,clf_key):
    roc = []
    tpr = []
    recall = []
    for i in range(5000):
        random.seed(i)
        np.random.shuffle(dataset)
        split = int(len(dataset)*0.7)
        X = np.array([line[:-1] for line in dataset1[:split]])
        Y = np.array([line[-1] for line in dataset1[:split]])
        X_eval = np.array([line[:-1] for line in dataset1[split:]])
        Y_eval = np.array([line[-1] for line in dataset1[split:]])
        if clf_key =="svm":
            smo = SMOTE(random_state=0)
            X,Y = smo.fit_sample(X,Y)
        clf.fit(X,Y)
        pre = clf.predict(X_eval)
        recall.append(recall_score(Y_eval,pre))
        roc.append(roc_auc_score(Y_eval,pre))
        maxtri = list(confusion_matrix(Y_eval,pre,labels=[1,-1]))
        tpr.append(maxtri[1][1]/(maxtri[1][1]+maxtri[1][0]))
    recall = np.sort(np.array(recall))
    roc = np.sort(np.array(roc))
    tpr = np.sort(np.array(tpr))
    ca = int(len(roc)*0.05/2)
    st = ca if ca!=0 else 0
    ed = (-1)*ca if ca!=0 else -1
    print("Sensitivity: ",round(recall.mean(),3),round(recall.std(),3),round(recall[st],3),round(recall[ed],3))
    print("AUC: ",round(roc.mean(),3),round(roc.std(),3),round(roc[st],3),round(roc[ed],3))
    print("Specificity: ",round(tpr.mean(),3),round(tpr.std(),3),round(tpr[st],3),round(tpr[ed],3))
for clf_key in clfs.keys():
    print("the classifier is :",clf_key)
    clf = clfs[clf_key]
    try_different_method(clf,clf_key)<jupyter_output><empty_output><jupyter_text>2.2 On the 13 statistical significance indicators internal cohort<jupyter_code>clfs = {"svm":SVC(C=5,kernel="rbf",max_iter=100000,probability=True)}
def try_different_method(clf,clf_key):
    roc = []
    tpr = []
    recall = []
    for i in range(5000):
        random.seed(i)
        np.random.shuffle(dataset2)
        split = int(len(dataset1)*0.7)
        X = np.array([line[:-1] for line in dataset2[:split]])
        Y = np.array([line[-1] for line in dataset2[:split]])
        X_eval = np.array([line[:-1] for line in dataset2[split:]])
        Y_eval = np.array([line[-1] for line in dataset2[split:]])
        smo = SMOTE(random_state=0)
        X,Y = smo.fit_sample(X,Y)
        clf.fit(X,Y)
        pre = clf.predict(X_eval)
        recall.append(recall_score(Y_eval,pre))
        roc.append(roc_auc_score(Y_eval,pre))
        maxtri = list(confusion_matrix(Y_eval,pre,labels=[1,-1]))
        tpr.append(maxtri[1][1]/(maxtri[1][1]+maxtri[1][0]))
    recall = np.sort(np.array(recall))
    roc = np.sort(np.array(roc))
    tpr = np.sort(np.array(tpr))
    ca = int(len(roc)*0.05/2)
    st = ca if ca!=0 else 0
    ed = (-1)*ca if ca!=0 else -1
    print("Sensitivity: ",round(recall.mean(),3),round(recall.std(),3),round(recall[st],3),round(recall[ed],3))
    print("AUC: ",round(roc.mean(),3),round(roc.std(),3),round(roc[st],3),round(roc[ed],3))
    print("Specificity: ",round(tpr.mean(),3),round(tpr.std(),3),round(tpr[st],3),round(tpr[ed],3))
for clf_key in clfs.keys():
    print("the classifier is :",clf_key)
    clf = clfs[clf_key]
    try_different_method(clf,clf_key)<jupyter_output><empty_output><jupyter_text>2.3 On the 13 statistical insignificance indicators internal cohort<jupyter_code>clfs = {"svm":SVC(C=5,kernel="rbf",max_iter=100000,probability=True)}
def try_different_method(clf,clf_key):
    roc = []
    tpr = []
    recall = []
    for i in range(5000):
        random.seed(i)
        np.random.shuffle(dataset3)
        split = int(len(dataset1)*0.7)
        X = np.array([line[:-1] for line in dataset3[:split]])
        Y = np.array([line[-1] for line in dataset3[:split]])
        X_eval = np.array([line[:-1] for line in dataset3[split:]])
        Y_eval = np.array([line[-1] for line in dataset3[split:]])
        smo = SMOTE(random_state=0)
        X,Y = smo.fit_sample(X,Y)
        clf.fit(X,Y)
        pre = clf.predict(X_eval)
        recall.append(recall_score(Y_eval,pre))
        roc.append(roc_auc_score(Y_eval,pre))
        maxtri = list(confusion_matrix(Y_eval,pre,labels=[1,-1]))
        tpr.append(maxtri[1][1]/(maxtri[1][1]+maxtri[1][0]))
    recall = np.sort(np.array(recall))
    roc = np.sort(np.array(roc))
    tpr = np.sort(np.array(tpr))
    ca = int(len(roc)*0.05/2)
    st = ca if ca!=0 else 0
    ed = (-1)*ca if ca!=0 else -1
    print("Sensitivity: ",round(recall.mean(),3),round(recall.std(),3),round(recall[st],3),round(recall[ed],3))
    print("AUC: ",round(roc.mean(),3),round(roc.std(),3),round(roc[st],3),round(roc[ed],3))
    print("Specificity: ",round(tpr.mean(),3),round(tpr.std(),3),round(tpr[st],3),round(tpr[ed],3))
for clf_key in clfs.keys():
    print("the classifier is :",clf_key)
    clf = clfs[clf_key]
    try_different_method(clf,clf_key)<jupyter_output><empty_output><jupyter_text>3. Indicators selection by isCov 3.1 Get ranking scores<jupyter_code>max_ranges1 = defaultdict(list)
max_ranges2 = defaultdict(list)
for line in all_names[:-1]:
    max_ranges1[line] = []
for i in range(0,5000):
    random.seed(i)
    np.random.shuffle(dataset1)
    X = np.array([line[:-1] for line in dataset1])
    Y = np.array([line[-1] for line in dataset1])
    list1 = []
    list2 = []
    lists1 = []
    lists2 = []    
    pram = []
    KF = KFold(n_splits=5)
    for train_index,test_index in KF.split(X):
        X_train,X_test = X[train_index],X[test_index]
        Y_train,Y_test = Y[train_index],Y[test_index]
        smo = SMOTE(random_state=0)
        X,Y = smo.fit_sample(X,Y)
        clf = LassoCV(max_iter=10000,n_jobs=40)
        clf.fit(X,Y)
        pre = clf.predict(X_test)
        label = [1 if line >0 else -1 for line in pre]
        list1 = [abs(line) for line in clf.coef_]
        list2 = [line for line in clf.coef_]
        lists1.append(list1)
        lists2.append(list2)
        
    lists1 = np.array(lists1).T
    lists2 = np.array(lists2).T
    mean_list1 = [line.mean() for line in lists1]
    mean_list2 = [line.mean() for line in lists2]
    name_order1 = dict(zip(all_names[:-1],mean_list1))
    name_order1 = dict(sorted(name_order1.items(),key=lambda x:x[1],reverse=True))    
    name_order2 = dict(zip(all_names[:-1],mean_list2))
    name_order2 = dict(sorted(name_order2.items(),key=lambda x:x[1],reverse=True)) 
    max_order = list(name_order1.keys())
    number = 25
    for j in range(len(max_order)):
        in_name = max_order[j]
        max_ranges1[in_name].append(number)
        max_ranges2[in_name].append(name_order2[in_name])
        number = number - 1
max_ranges = dict()
for k in max_ranges1.keys():
    max_ranges[k] = sum(max_ranges1[k])
max_ranges = dict(sorted(max_ranges.items(),key = lambda x:x[1],reverse = True))
max_index = dict()
for key in max_ranges.keys():
    index = all_names.index(key)
    max_index[key]=index
print(max_ranges.keys())
info_name = [line for line in max_index.keys()]
info = [line for line in max_index.values()]<jupyter_output><empty_output><jupyter_text>3.2 Feeding indicators into iCov according to the isCov model ranking scores<jupyter_code>#多个指标添加
nums = []
nums_name = []
for i in range(len(info)):
    if i == 0:
        nums.append([info[0]])
        nums_name.append([info_name[0]])
    else:
        nums.append([info[:i+1]])
        nums_name.append([info_name[:i+1]]) 
#单个指标测试
nums1 = []
nums_name1 = []
for i in range(len(info)):
    nums1.append([info[i]])
    nums_name1.append([info_name[i]]) 
roc = []
roc_std = []
roc_ci = [[],[]]
recall = []
recall_std = []
recall_ci = [[],[]]
tpr = []
tpr_std = []
tpr_ci = [[],[]]
for j in range(len(nums)):
    dataset_tmp = []
    print(nums_name[j])
    for line in dataset1:
        col = []
        line_index = nums[j][0] if j !=0 else nums[j]
        for n in range(len(line)):
            if n in line_index:
                col.append(line[n])
        dataset_tmp.append(col+[line[-1]])

    s2 = []
    s5 = []
    s6 = []
    model = SVC(C=5,kernel="rbf",max_iter=100000)
    for i in range(5000):
        random.seed(i)
        np.random.shuffle(dataset_tmp)
        split = int(len(dataset_tmp)*0.7)
        X = np.array([line[:-1] for line in dataset_tmp[:split]])
        Y = np.array([line[-1] for line in dataset_tmp[:split]])
        X_eval = np.array([line[:-1] for line in dataset_tmp[split:]])
        Y_eval = np.array([line[-1] for line in dataset_tmp[split:]])
        smo = SMOTE(random_state=i)
        X,Y = smo.fit_sample(X,Y)
        if len(X[0])==1:
            X = X.reshape(-1,1)
            X_eval = X_eval.reshape(-1,1)
        model.fit(X,Y)
        pre = model.predict(X_eval)
        s2.append(recall_score(Y_eval,pre))
        s5.append(roc_auc_score(Y_eval,pre))
        maxtri = list(confusion_matrix(Y_eval,pre,labels=[1,-1]))
        s6.append(maxtri[1][1]/(maxtri[1][1]+maxtri[1][0]))
        

    s2 = np.sort(np.array(s2))
    s5 = np.sort(np.array(s5))
    s6 = np.sort(np.array(s6))
    
    #0.95 CI
    ca = int(len(roc)*0.05/2)
    st = ca if ca!=0 else 0
    ed = (-1)*ca if ca!=0 else -1
    recall_ci[0].append(s2[st])
    recall_ci[1].append(s2[ed])        
    roc_ci[0].append(s5[st])
    roc_ci[1].append(s5[ed])
    tpr_ci[0].append(s6[st])
    tpr_ci[1].append(s6[ed])
    
    v2 = [round(s2.mean(),2),round(s2.std(),2)]
    v5 = [round(s5.mean(),2),round(s5.std(),2)]
    v6 = [round(s6.mean(),2),round(s6.std(),2)]
    print("Sensitivity:",s2.mean(),recall_ci[0][j],recall_ci[1][j])
    print("AUC:",s5.mean(),roc_ci[0][j],roc_ci[1][j])
    print("Specificity:",s6.mean(),tpr_ci[0][j],tpr_ci[1][j])
    recall.append(v2[0])
    recall_std.append(v2[1])
    roc.append(v5[0])
    roc_std.append(v5[1])    
    tpr.append(v6[0])
    tpr_std.append(v6[1]) 
#roc for 13 indicators
dataset_tmp = []
print(nums_name[12])
for line in dataset1:
    col = []
    for n in range(len(line)):
        if n in nums[12][0]:
            col.append(line[n])
    dataset_tmp.append(col+[line[-1]])
model = SVC(C=5,kernel="rbf",max_iter=100000)
random.seed(0)
np.random.shuffle(dataset_tmp)
split = int(len(dataset_tmp)*0.7)
X = np.array([line[:-1] for line in dataset_tmp[:split]])
Y = np.array([line[-1] for line in dataset_tmp[:split]])
X_eval = np.array([line[:-1] for line in dataset_tmp[split:]])
Y_eval = np.array([line[-1] for line in dataset_tmp[split:]])
smo = SMOTE(random_state=0)
X,Y = smo.fit_sample(X,Y)
y_score = model.fit(X,Y).decision_function(X_eval)
fpr,tprs,threshold = roc_curve(Y_eval, y_score) 
roc_auc = auc(fpr,tprs)
print(roc_auc)
#roc for 6 indicators
dataset_tmp1 = []
print(nums_name[5])
for line in dataset1:
    col = []
    for n in range(len(line)):
        if n in nums[5][0]:
            col.append(line[n])
    dataset_tmp1.append(col+[line[-1]])
model = SVC(C=5,kernel="rbf",max_iter=100000)
random.seed(0)
np.random.shuffle(dataset_tmp1)
split = int(len(dataset_tmp1)*0.7)
X = np.array([line[:-1] for line in dataset_tmp1[:split]])
Y = np.array([line[-1] for line in dataset_tmp1[:split]])
X_eval = np.array([line[:-1] for line in dataset_tmp1[split:]])
Y_eval = np.array([line[-1] for line in dataset_tmp1[split:]])
smo = SMOTE(random_state=0)
X,Y = smo.fit_sample(X,Y)
y_score = model.fit(X,Y).decision_function(X_eval)
fpr1,tpr1,threshold1 = roc_curve(Y_eval, y_score) 
roc_auc1 = auc(fpr1,tpr1)
print(roc_auc1)
#roc for 26 indicators
dataset_tmp2 = []
print(nums_name[25])
for line in dataset1:
    col = []
    for n in range(len(line)):
        if n in nums[25][0]:
            col.append(line[n])
    dataset_tmp2.append(col+[line[-1]])
model = SVC(C=5,kernel="rbf",max_iter=100000)
random.seed(9)
np.random.shuffle(dataset_tmp2)
split = int(len(dataset_tmp2)*0.7)
X = np.array([line[:-1] for line in dataset_tmp2[:split]])
Y = np.array([line[-1] for line in dataset_tmp2[:split]])
X_eval = np.array([line[:-1] for line in dataset_tmp2[split:]])
Y_eval = np.array([line[-1] for line in dataset_tmp2[split:]])
smo = SMOTE(random_state=0)
X,Y = smo.fit_sample(X,Y)
y_score = model.fit(X,Y).decision_function(X_eval)
fpr2,tpr2,threshold2 = roc_curve(Y_eval, y_score) 
roc_auc2 = auc(fpr2,tpr2)
print(roc_auc2)<jupyter_output><empty_output><jupyter_text>4. Testing iCov and isCov model on external cohort<jupyter_code>origin1 = data["external"]
test_case1 = origin1[origin1.label==1]
test_case2 = origin1[origin1.label==-1]<jupyter_output><empty_output><jupyter_text>4.1 Data processing<jupyter_code>all_case = []
for k in case1.columns[:-1]:
    line1 = case1[k].values
    line2 = case2[k].values
    line3 = test_case1[k].values
    line4 = test_case2[k].values
    line_all = line1.tolist() + line2.tolist()
    line_all1 = line1.tolist() + line2.tolist()+line3.tolist() + line4.tolist()
    line_clean = []
    info = []
    for x in line_all:
        if x!=-2: 
            line_clean.append(float(x))
    mean_value = np.mean(line_clean)
    for x in line_all1:
        if x !=-2:
            info.append(x)
        else:
            info.append(mean_value)
    all_case.append(info)
final_case = []
trans_data = np.array(all_case)
for i in range(len(trans_data)):
    line = trans_data[i]
    max_v = max(line)
    min_v = min(line)
    val = max_v-min_v
    final_case.append([(x-min_v)/(val) for x in line])
final_case1 = np.array(final_case).T.tolist()
dataset = []
set1 = [9, 24, 10, 26, 21, 27, 11, 14, 5, 6, 8, 20, 23]
dataset2 = []
dataset3 = []
for i in range(len(final_case1)):
    if (i < 75) or (i>=732 and i<751):
        da = final_case1[i] + [1]
        dataset.append(da)
        text = []
        text1 = []
        trans = np.array(final_case1[i]).T
        for j in range(len(trans)):
            if j<=1:
                continue
            else:
                if j in set1:
                    text.append(trans[j])
                else:
                    text1.append(trans[j])
        dataset2.append(np.array(text).T.tolist()+[1])
        dataset3.append(np.array(text1).T.tolist()+[1])
    else:
        da = final_case1[i] + [-1]
        dataset.append(da)
        text = []
        text1 = []
        trans = np.array(final_case1[i]).T
        for j in range(len(trans)):
            if j<=1:
                continue
            else:
                if j in set1:
                    text.append(trans[j])
                else:
                    text1.append(trans[j])
        dataset2.append(np.array(text).T.tolist()+[-1])
        dataset3.append(np.array(text1).T.tolist()+[-1])
Trainset = dataset[:732]
Testset = dataset[732:]
nums1 = [7, 22, 8, 24, 19, 25, 9, 12, 3, 4, 6, 18, 21] # 13 statistically significant indictaors
nums2 = [15, 22, 7, 6, 8, 12] # Top 6 isCov selected indictaors
nums3 = [15, 22, 7, 6, 8, 12, 17, 2, 11, 16, 20, 21, 24] # Top 13 isCov selected indictaors
nums4 = [i for i in range(26)] # all 26 indictaors
dataset_select = []
dataset_select1 = []
# build training set
for line in Trainset:
    col = []
    for n in range(len(line)):
        if n in nums1:# varible nums1 can be replaced for testing different indicators data set. For example, nums2.
            col.append(line[n])
    dataset_select.append(col+[line[-1]])
# build testing set    
for line in Testset:
    col = []
    for n in range(len(line)):
        if n in nums1: # varible nums1 can be replaced for testing different indicators data set. For example, nums2.
            col.append(line[n])
    dataset_select1.append(col+[line[-1]])<jupyter_output><empty_output><jupyter_text>4.2 Testing<jupyter_code>clf = SVC(C=5,kernel="rbf",max_iter=100000)
# clf = LogisticRegressionCV(n_jobs=30,max_iter=10000) # for testing other classifer
# clf = AdaBoostClassifier(n_estimators=200) # for testing other classifer
# clf = RandomForestClassifier(n_estimators=200,n_jobs=40) # for testing other classifer
roc = []
recall = []
tpr = []
np.random.shuffle(dataset_select)
np.random.shuffle(dataset_select1)
X = np.array([line[:-1] for line in dataset_select])
Y = np.array([line[-1] for line in dataset_select])
X_eval = np.array([line[:-1] for line in dataset_select1])
Y_eval = np.array([line[-1] for line in dataset_select1])
smo = SMOTE(random_state=i)
X,Y = smo.fit_sample(X,Y)
clf.fit(X,Y)
pre = clf.predict(X_eval)
recall.append(recall_score(Y_eval,pre))
roc.append(roc_auc_score(Y_eval,pre))
maxtri = list(confusion_matrix(Y_eval,pre,labels=[1,-1]))
tpr.append(maxtri[1][1]/(maxtri[1][1]+maxtri[1][0]))
tpr = np.sort(np.array(tpr))
recall = np.sort(np.array(recall))
roc = np.sort(np.array(roc))

print("auc: ",round(roc.mean(),3))
print("sensitivity: ",round(recall,3))
print("specificity: ",round(tpr,3))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1684852210440039, 'fraction_numerical': 0.0389836586240459, 'mean_word_length': 3.2282867662984622, 'pattern_counts': {'":': 7, '<': 44, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '415585', 'license': 'MIT', 'n_tokens_mistral': 7929, 'n_tokens_neox': 7605, 'n_words': 1365, 'path': 'iCov_isCov.ipynb', 'repo_name': 'yangkexin/prediction-of-COVID-19', 'size': '19521'}