<jupyter_start><jupyter_text>到目前为止，最终提取了总共2150维特征。1.amino_acid：20维。20种氨基酸在序列中出现的频率。2.local-DPP：120维。基于PSSM提出的一种能够保留局部PSSM特性的120维特征。3.PSSM-DT：2000维。基于PSSM的在ACC思想上进行的变种，分为SDT（100维），DDT（1900维）。4.amino_acid_top10:10维。根据论文结果统计的特性占比在前十位的两个氨基酸组合。在各种分类算法对比下，准确率最高的是GradientBoost，达到83.886%。 1.提取20种氨基酸在序列中出现的频率  输入文件名，得到相应的特征值。  输出文件feature_amino_acid_freq.csv，20维的特征向量值。<jupyter_code>from numpy import random
file="data/temp_test.txt"
def load_data(filename):
    returnval = []
    with open(filename) as file1:
        content2 = file1.read().replace("\n", " ").split(">")
        content = [a.split(" ", 1) for a in content2]
        for a in content[1:]:
            if a != "" and a!= [""] and a != []:
                returnval.append([a[0], a[1].replace(" ", "")])
    return returnval
dataset = load_data(file)
amino_acid="<KEY>"
def get_amino_acid_frequencies(seq):
    return [ seq.count(base)/len(seq)
        for base in amino_acid]
import csv
for data in dataset:
    frequencies = get_amino_acid_frequencies(data[1])
    with open("final1225/1_feature_amino_acid_freq.csv","a+",newline='') as file:
        writer = csv.writer(file)
        writer.writerow(frequencies)
2.提取基于PSSM提出的一种能够保留局部PSSM特性的120维特征。
  输入文件为PSSM矩阵
    输出文件2_feature_LocalDPP.csv，120维的特征向量值
import scipy.io as sio
import numpy as np
import pandas as pd
import csv
pssm_file="PSSM/protein_neg.mat"
load_data = sio.loadmat(pssm_file)
#print (load_data)
n=3
lamd=1
for pssm in load_data['train_pssm'][0]:
    # get the submatrix 
    L = len(pssm)
    length=int(L/n)
    # the length of L1 and L2
    if(L%n==0):
        L1=L2=length
    else:
        L1=length
        L2=L-L1*(n-1)
    #print('L,L1,L2',L,L1,L2)
    # get the first block of feautre F
    feature1=[]
    for k in range(1,n):
        temp_fea=[]
        for j in range(0,20):
            temp=0
            for i in range((k-1)*L1,k*L1):
                temp+=pssm[i][j]
            temp_=temp/L1
            temp_fea.append(temp_)
        feature1.extend(temp_fea)
    #print('feature1......',feature1)
    #print('len(feature1):',len(feature1)) #20
    feature2=[]
    for k in range(1,n):
        for lam in range(1,lamd+1):
            temp_fea=[]
            for j in range(0,20):
                temp=0
                for i in range((k-1)*L1,k*L1-lam):
                    temp+=(pssm[i][j]-pssm[i+lam][j])**2
                temp_=temp/(L1-lam)
                temp_fea.append(temp_)
            feature2.extend(temp_fea)
    #print('feature2......',feature2)
    #print('len(feature2):',len(feature2)) #40
    feature1_n=[]
    for j in range(0,20):
        temp=0
        for i in range((n-1)*L1,(n-1)*L1+L2):
            temp+=pssm[i][j]
        temp_=temp/L2
        feature1_n.append(temp_)
    #print ('feature1_n......',feature1_n)
    #print ('len(feature1_n):',len(feature1_n)) #20
    feature2_n=[]
    for lam in range(1,lamd+1):
        for j in range(0,20):
            temp=0
            for i in range((n-1)*L1,(n-1)*L1+L2-lam):
                temp+=(pssm[i][j]-pssm[i+lam][j])**2
            temp_=temp/(L2-lam)
            feature2_n.append(temp_)
    #print('feature2_n......',feature2_n)
    #print('len(feature2_n):',len(feature2_n)) # 40      
    # get all features
    feature=[]
    feature.extend(feature1) # 40 
    feature.extend(feature2) # 40
    feature.extend(feature1_n) # 20
    feature.extend(feature2_n) # 20
    #print('feature......',feature) 
    #print(len(feature)) # 120
    with open("final1225/2_feature_LocalDPP.csv","a+",newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(feature)
3.提取PSSM-DT，基于PSSM的在ACC思想上进行的变种，分为SDT（100维），DDT（1900维）。
  输入为PSSM矩阵文件
    输出为
import scipy.io as sio
import numpy as np
pssm_file = 'PSSM/protein_neg.mat'
load_data = sio.loadmat(pssm_file)
for pssm in load_data['train_pssm'][0]:
    L = len(pssm)
    LG = 5
    SDT=[]
    for lg in range(1,LG+1):
        SDT_lg=[]
        for i in range(0,20):
            temp=0
            for j in range(0,L-lg):
                temp+=pssm[j][i]*pssm[j+lg][i]
            temp=temp/(L-lg)
            SDT_lg.append(temp)
        SDT.extend(SDT_lg)
    with open("final1225/3_feature_PSSMDT_SDT_neg.csv","a+",newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(SDT)
for pssm in load_data['train_pssm'][0]:
    L = len(pssm)
    LG = 5
    DDT=[]
    for lg in range(1,LG+1):
        DDT_lg=[]
        temp=0
        for i1 in range(0,20):
            for i2 in range(0,20):
                if i1==i2:
                    continue
                for j in range(0,L-lg):
                    temp+=pssm[j][i1]*pssm[j+lg][i2]
                temp=temp/(L-lg)
                DDT_lg.append(temp)
        DDT.extend(DDT_lg)
    with open("final1225/3_feature_PSSMDT_DDT_neg.csv","a+",newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(DDT)
4.提取amino_acid_top10，根据论文结果统计的特性占比在前十位的两个氨基酸组合。
     输入文件名，得到相应的特征值。
      输出文件4_feature_amino_acid_top_10.csv，20维的特征向量值。
file="data/temp_test.txt"
amino_acid_10=["KR","LV","RK","GA","VL","AG","AV","EK","GD","AL"]
def load_data(filename):
    returnval = []
    with open(filename) as file1:
        content2 = file1.read().replace("\n", " ").split(">")
        content = [a.split(" ", 1) for a in content2]
        for a in content[1:]:
            if a != "" and a!= [""] and a != []:
                returnval.append([a[0], a[1].replace(" ", "")])
    return returnval
dataset = load_data(file)
import re
def get_amino_acid_frequencies_top_10(seq):
    res = []
    for base in amino_acid_10:
        reg = re.compile("(?="+base+")")
        length = len(reg.findall(seq))
        res.append(length/(len(seq)))
    return res
import csv
for data in dataset:
    frequencies = get_amino_acid_frequencies_top_10(data[1])
    with open("final1225/4_feature_amino_acid_freq_top_10_2.csv","a+",newline='') as file:
        writer = csv.writer(file)
        writer.writerow(frequencies)<jupyter_output><empty_output><jupyter_text>加载特征数据，进行十折交叉验证测试模型，得到准确率较高的为gdbt模型<jupyter_code>import codecs
from itertools import *
import numpy as np
from sklearn import model_selection
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingClassifier
import pylab as pl
from sklearn.externals import joblib
def load_data(filename):
    file = codecs.open(filename,'r','utf-8')
    data = []
    label = []
    for line in islice(file,0,None):
        line = line.strip().split(',')
        #print ("reading data....")
        data.append([float(i) for i in line[1:-1]])
        label.append(line[-1])
    x = np.array(data)
    y = np.array(label)
    return x,y
def gradient_boosting(x_train,y_train):
    print("gradient_boosting...")     
    clf = GradientBoostingClassifier()
    score = model_selection.cross_val_score(clf,x_train,y_train,cv=10,scoring="accuracy")
    joblib.dump(clf,"final1225/gbdt.model")
    # gdbt_model = joblib.load("gdbt.model")
    print ('The accuracy of GradientBoosting:')
    print (np.mean(score))
    x = [int(i) for i in range(1, 11)]
    y = score
    pl.ylabel(u'Accuracy')
    pl.xlabel(u'times')
    pl.plot(x, y,label='GBDT')
    pl.legend()
    pl.savefig("final1225/gradient_boosting.png")
def train_acc(filename):
    x_train,y_train = load_data(filename)
    
    gradient_boosting(x_train,y_train)
    
train_acc("final1225/feature_all.csv")<jupyter_output>gradient_boosting...
The accuracy of GradientBoosting:
0.83792834891<jupyter_text>模型持久化<jupyter_code>x,y = load_data("final1225/feature_all.csv")
gdbt = GradientBoostingClassifier(random_state=2017)
gdbt.fit(x,y)
joblib.dump(gdbt,"final1225/gbdt.model")
y_pred = gdbt.predict(x)
y_preprob = gdbt.predict_proba(x)[:,1]
print (y_pred)  
print (y_preprob)
模型预测
输入test_x为经过上述特征提取之后的2150维的特征值。
输出y_pred为分类结果
gdbt_model = joblib.load("final1225/gbdt.model")
y_pred = gdbt_model.predict(test_x)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1493963782696177, 'fraction_numerical': 0.03898390342052314, 'mean_word_length': 3.012613521695257, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '220643', 'license': 'MIT', 'n_tokens_mistral': 3448, 'n_tokens_neox': 3316, 'n_words': 503, 'path': 'feature_extraction_all_1225.ipynb', 'repo_name': 'pyupcgithub/DNA_binding-protein-prediciton', 'size': '7967'}