<jupyter_start><jupyter_text>Feature Engineering and Selection*****Author:** Jiacheng**Create Time:**  2020-01-06**Update Time:**  2020-01-09*** [0. EDA](0.EDA) [1. 特征工程 (Feature Engineering)](一、特征工程) [2. 特征选择 (Feature Selection)](二、特征选择)<jupyter_code># 导入所需模块
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
from scipy.stats import skew
from sklearn.preprocessing import LabelEncoder,OneHotEncoder
from sklearn.ensemble import IsolationForest

import warnings
warnings.filterwarnings('ignore')
# 数据载入
data_train = pd.read_csv('./Data_Set/train_data.csv')
data_test = pd.read_csv('./Data_Set/test_a.csv')

data_train['Type'] = 'Train'
data_test['Type'] = 'Test'

data_all = pd.concat([data_train, data_test], ignore_index=True)

print ("Train: ",data_train.shape[0],"sales, and ",data_train.shape[1],"features")
print ("Test: ",data_test.shape[0],"sales, and ",data_test.shape[1],"features")<jupyter_output>Train:  41440 sales, and  52 features
Test:  2469 sales, and  51 features<jupyter_text>0.EDA我们将之前EDA的函数整合，用来处理原始数据，随后再进行特征工程<jupyter_code>def preprocessingData(df):
    # 删除无关字段
    df.drop("city", axis=1, inplace=True)
    df.drop("ID", axis=1, inplace=True)
    
    # 转换object
#     columns = ['rentType','communityName','houseType', 'houseFloor', 'houseToward', 'houseDecoration',  'region', 'plate']
#     for feature in columns:
#         df[feature] = LabelEncoder().fit_transform(df[feature])
    
    # pv、uv填充
    df['pv'].fillna(df['pv'].mean(), inplace=True)
    df['uv'].fillna(df['uv'].mean(), inplace=True)
    # 浏览人数为整数
    df['pv'] = df['pv'].astype('int')
    df['uv'] = df['uv'].astype('int')
    
    # rentType转换
    df['rentType'][df['rentType'] == '--'] = '未知方式'
    
    # buildYear处理
    buildYearMode = pd.DataFrame(df[df['buildYear'] != '暂无信息']['buildYear'].mode())
    df.loc[df[df['buildYear'] == '暂无信息'].index, 'buildYear'] = buildYearMode.iloc[0, 0]
    df['buildYear'] = df['buildYear'].astype('int')
    
    # 分割交易时间
    # 年份相同，丢弃
    df['month'] = df['tradeTime'].apply(lambda x: x.split('/')[1]).astype('int')
    df['day'] = df['tradeTime'].apply(lambda x: x.split('/')[2]).astype('int')
    df.drop('tradeTime', axis=1, inplace=True)
    
    return df
def cleanData(df):
    # 孤立森林处理
    def IF_drop(df):
        IForest = IsolationForest(contamination=0.01)
        IForest.fit(df["tradeMoney"].values.reshape(-1,1))
        y_pred = IForest.predict(df["tradeMoney"].values.reshape(-1,1))
        drop_index = df.loc[y_pred==-1].index
        
        df.drop(drop_index,inplace=True)
        return df

    df = IF_drop(df)
    
    # 异常值处理
    df = df[df.area<=200]
    df.drop(df[(df['totalFloor'] == 0)].index, inplace=True)
    df = df[(df.tradeMoney <=16000) & (df.tradeMoney >=500)]
    
    # 深度清洗
    def DeepClean(data):
        data.drop(data[(data['region']=='RG00001') & (data['tradeMoney']<=1000)&(data['area']>=50)].index,inplace=True)
        data.drop(data[(data['region']=='RG00002') & (data['tradeMoney']>60000)&(data['area']<100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00003') & (data['tradeMoney']>30000)&(data['area']<300)].index,inplace=True)
        data.drop(data[(data['region']=='RG00003') & (data['tradeMoney']<500)&(data['area']<50)].index,inplace=True)
        data.drop(data[(data['region']=='RG00003') & (data['tradeMoney']<1500)&(data['area']>100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00003') & (data['tradeMoney']<2000)&(data['area']>300)].index,inplace=True)
        data.drop(data[(data['region']=='RG00003') & (data['tradeMoney']>5000)&(data['area']<20)].index,inplace=True) 
        data.drop(data[(data['region']=='RG00004') & (data['tradeMoney']<1000)&(data['area']>80)].index,inplace=True)
        data.drop(data[(data['region']=='RG00004') & (data['tradeMoney']>20000)&(data['area']>400)].index,inplace=True)
        data.drop(data[(data['region']=='RG00004') & (data['tradeMoney']>8000)&(data['area']<80)].index,inplace=True)
        data.drop(data[(data['region']=='RG00005') & (data['tradeMoney']<2000)&(data['area']>180)].index,inplace=True)
        data.drop(data[(data['region']=='RG00005') & (data['tradeMoney']>50000)&(data['area']<200)].index,inplace=True)
        data.drop(data[(data['region']=='RG00005') & (data['tradeMoney']>30000)&(data['area']<100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00005') & (data['tradeMoney']<50000)&(data['area']>600)].index,inplace=True)
        data.drop(data[(data['region']=='RG00005') & (data['tradeMoney']>50000)&(data['area']>350)].index,inplace=True)
        data.drop(data[(data['region']=='RG00006') & (data['tradeMoney']>4000)&(data['area']<100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00006') & (data['tradeMoney']<600)&(data['area']>100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00006') & (data['area']>165)].index,inplace=True)
        data.drop(data[(data['region']=='RG00006') & (data['tradeMoney']<200)].index,inplace=True)
        data.drop(data[(data['region']=='RG00006') & (data['tradeMoney']<2000)&(data['area']>200)].index,inplace=True)
        data.drop(data[(data['region']=='RG00007') & (data['tradeMoney']<2500)&(data['area']>100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00007') & (data['tradeMoney']<1100)&(data['area']>50)].index,inplace=True)
        data.drop(data[(data['region']=='RG00008') & (data['tradeMoney']<2000)&(data['area']>80)].index,inplace=True)
        data.drop(data[(data['region']=='RG00008') & (data['tradeMoney']>15000)&(data['area']<110)].index,inplace=True)
        data.drop(data[(data['region']=='RG00008') & (data['tradeMoney']>20000)&(data['area']>110)].index,inplace=True)
        data.drop(data[(data['region']=='RG00008') & (data['tradeMoney']<1500)&(data['area']<50)].index,inplace=True)
        data.drop(data[(data['region']=='RG00008') & (data['rentType']=='合租')&(data['area']>50)].index,inplace=True)
        data.drop(data[(data['region']=='RG00009') & (data['tradeMoney']>40000)].index,inplace=True)
        data.drop(data[(data['region']=='RG00009') & (data['area']>300)].index,inplace=True)
        data.drop(data[(data['region']=='RG00009') & (data['tradeMoney']<2000)&(data['area']>100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00010') & (data['tradeMoney']>25000)&(data['area']>200)].index,inplace=True)
        data.drop(data[(data['region']=='RG00010') & (data['tradeMoney']<15000)&(data['area']>400)].index,inplace=True)
        data.drop(data[(data['region']=='RG00010') & (data['tradeMoney']<3000)&(data['area']>200)].index,inplace=True)
        data.drop(data[(data['region']=='RG00010') & (data['tradeMoney']>7000)&(data['area']<75)].index,inplace=True)
        data.drop(data[(data['region']=='RG00010') & (data['tradeMoney']>12500)&(data['area']<100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00011') & (data['tradeMoney']<10000)&(data['area']>390)].index,inplace=True)
        data.drop(data[(data['region']=='RG00012') & (data['tradeMoney']<5000)&(data['area']>120)].index,inplace=True)
        data.drop(data[(data['region']=='RG00012') & (data['tradeMoney']<800)&(data['area']<30)].index,inplace=True)
        data.drop(data[(data['region']=='RG00012') & (data['tradeMoney']<800)&(data['area']<30)].index,inplace=True)   
        data.drop(data[(data['region']=='RG00013') & (data['tradeMoney']>40000)&(data['area']<100)].index,inplace=True)
        data.drop(data[(data['region']=='RG00013') & (data['tradeMoney']>50000)&(data['area']>400)].index,inplace=True)
        data.drop(data[(data['region']=='RG00013') & (data['tradeMoney']<2000)&(data['area']>80)].index,inplace=True)
        data.drop(data[(data['region']=='RG00014') & (data['tradeMoney']>40000)&(data['area']>300)].index,inplace=True)
        data.drop(data[(data['region']=='RG00014') & (data['tradeMoney']<1300)&(data['area']>80)].index,inplace=True)
        data.drop(data[(data['region']=='RG00014') & (data['tradeMoney']<8000)&(data['area']>200)].index,inplace=True)
        data.drop(data[(data['region']=='RG00014') & (data['tradeMoney']<1000)&(data['area']>20)].index,inplace=True)
        data.drop(data[(data['region']=='RG00014') & (data['tradeMoney']>25000)&(data['area']>200)].index,inplace=True)
        data.drop(data[(data['region']=='RG00014') & (data['tradeMoney']<20000)&(data['area']>250)].index,inplace=True)
        data.drop(data[(data['region']=='RG00015') ].index,inplace=True)
        
        data.loc[(data['region']=='RG00002')&(data['area']>50)&(data['rentType']=='合租'),'rentType']='整租'
        data.loc[(data['region']=='RG00014')&(data['rentType']=='合租')&(data['area']>60),'rentType']='整租'
        
        data.reset_index(drop=True, inplace=True)        
        return data
    
    df = DeepClean(df)
    
    return df
data_train = preprocessingData(data_train)
data_test = preprocessingData(data_test)

data_train = cleanData(data_train)

print ("Train: ",data_train.shape[0],"sales, and ",data_train.shape[1],"features")
print ("Test: ",data_test.shape[0],"sales, and ",data_test.shape[1],"features")
data_train.head()
data_test.head()<jupyter_output><empty_output><jupyter_text>一、特征工程数据处理完毕，接下来可以构造一下新的特征  选择部分特征用热图查看一下相关性   [分析判断](分析判断)     [整合函数](整合函数)  分析判断先用简单的语法，生成整理、判断一些特征，最后再整合为函数<jupyter_code># 数据载入
train = data_train.copy()
test = data_test.copy()<jupyter_output><empty_output><jupyter_text>rentType再生成前面我们已经判断 rentType 大多数为‘未知方式’，我们不妨用已知数据加主观判断确定一下部分未知的rentType<jupyter_code>train['Room'] = train['houseType'].apply(lambda x: x.split('室')[0]).astype('int')
train['Hall'] = train['houseType'].apply(lambda x: x.split('室')[1].split('厅')[0]).astype('int') 
train['Bath'] = train['houseType'].apply(lambda x: x.split("室")[1].split("厅")[1].split("卫")[0]).astype('int')
train['Room_Bath'] = (train['Bath']+1) / (train['Room']+1)
train.head()<jupyter_output><empty_output><jupyter_text>画图看看rentType与area、tradeMoney、Room_Bath的关系<jupyter_code>sns.lmplot('area', 'Room_Bath', data=train, hue='rentType', scatter_kws={'alpha':0.2})
sns.lmplot('area', 'tradeMoney', data=train, hue='rentType', scatter_kws={'alpha':0.2})<jupyter_output><empty_output><jupyter_text>这张图一看就想聚类或者SVM啊<jupyter_code>sns.lmplot('area', 'Room', data=train, hue='rentType', scatter_kws={'alpha':0.2})<jupyter_output><empty_output><jupyter_text>emmmm, 有点出乎意料  合租房子的面积基本小于50，（厕所/房间数）基本小于1，价格小于6000  据此可以判断写未知方式 先调用SVM判断可行吗？？？  试一试<jupyter_code>from sklearn.svm import SVC
train['id'] = [i for i in range(train.shape[0])]
SVM_all = pd.DataFrame(train[['id', 'area', 'tradeMoney', 'Room', 'Room_Bath', 'rentType']])
SVM_train = SVM_all[SVM_all['rentType'] != '未知方式']
SVM_test = SVM_all[SVM_all['rentType'] == '未知方式']
SVM_train_x = np.array(SVM_train.iloc[:,1:5])
SVM_train_y = LabelEncoder().fit_transform(np.array(SVM_train.iloc[:,-1])).reshape(-1,1)
SVM_test_x = np.array(SVM_test.iloc[:,1:5])

svm = SVC(kernel='linear')
svm.fit(SVM_train_x, SVM_train_y)
y = svm.predict(SVM_test_x)
y
sns.lmplot('area', 'tradeMoney', data=SVM_train, hue='rentType', scatter_kws={'alpha':0.2})
SVM_test_y = []
n = 0
for i in range(y.size):
    if y[i] == 1:
        SVM_test_y.append("整租")
    else:
        SVM_test_y.append("合租")
        n += 1
        
SVM_test['rentType'] = SVM_test_y
SVM_test.head()
sns.lmplot('area', 'Room_Bath', data=SVM_test, hue='rentType', scatter_kws={'alpha':0.2})
def SVM_RentType(train):
    train['Room'] = train['houseType'].apply(lambda x: x.split('室')[0]).astype('int')
    train['Hall'] = train['houseType'].apply(lambda x: x.split('室')[1].split('厅')[0]).astype('int') 
    train['Bath'] = train['houseType'].apply(lambda x: x.split("室")[1].split("厅")[1].split("卫")[0]).astype('int')
    train['Room_Bath'] = (train['Bath']+1) / (train['Room']+1)
    train['id'] = [i for i in range(train.shape[0])]
    
    SVM_all = pd.DataFrame(train[['id', 'area', 'Room', 'Room_Bath', 'rentType']])
    SVM_train = SVM_all[SVM_all['rentType'] != '未知方式']
    SVM_test = SVM_all[SVM_all['rentType'] == '未知方式']
    SVM_train_x = np.array(SVM_train.iloc[:,1:4])
    SVM_train_y = LabelEncoder().fit_transform(np.array(SVM_train.iloc[:,-1])).reshape(-1,1)
    SVM_test_x = np.array(SVM_test.iloc[:,1:4])
    
    # 预测rentType
    svm = SVC(kernel='rbf')
    svm.fit(SVM_train_x, SVM_train_y)
    
    SVM_test_y = []
    n = 0
    for i in range(y.size):
        if y[i] == 1:
            SVM_test_y.append("整租")
        else:
            SVM_test_y.append("合租")
#     SVM_test['rentType'] = SVM_test_y
    
    train.loc[train['rentType'] == '未知方式', 'rentType'] = SVM_test_y
    
    return train
train = data_train.copy()
train = SVM_RentType(train)
train.head()
train.rentType.value_counts()
test = data_test.copy()
# test = SVM_RentType(test)
test.rentType.value_counts()<jupyter_output><empty_output><jupyter_text>数据再分析<jupyter_code>train.loc[(train['Room'] <= 1), 'rentType'].value_counts()<jupyter_output><empty_output><jupyter_text>Room数量为1的未知方式，可以大致判断为整租<jupyter_code>train.loc[(train['Room_Bath'] >= 1), 'rentType'].value_counts()<jupyter_output><empty_output><jupyter_text>Room_Bath大于等于1的未知方式可以大致判断为整租<jupyter_code>train.loc[(train['Room'] == 2) & (train['area']<=30), 'rentType'].value_counts()<jupyter_output><empty_output><jupyter_text>Room = 2；area<=30 的未知方式可以大致判断为合租<jupyter_code>train.loc[(train['Room'] == 2) & (train['area']>60), 'rentType'].value_counts()<jupyter_output><empty_output><jupyter_text>Room=2;area>60 的未知方式可以大致判断为整租<jupyter_code>train.loc[(train['Room'] == 3) & (train['area']<=60), 'rentType'].value_counts()
train.loc[(train['Room'] == 3) & (train['area']>60), 'rentType'].value_counts()
train.loc[(train['Room'] > 3) & (train['area']>=100), 'rentType'].value_counts()<jupyter_output><empty_output><jupyter_text>整合函数<jupyter_code>def FeatureEngineering(data):
    # houseType抽取信息
    data['Room'] = data['houseType'].apply(lambda x: x.split('室')[0]).astype('int')
    data['Hall'] = data['houseType'].apply(lambda x: x.split('室')[1].split('厅')[0]).astype('int')  
    data['Bath'] = data['houseType'].apply(lambda x: x.split("室")[1].split("厅")[1].split("卫")[0]).astype('int') 
    data['Room_Bath'] = (data['Bath'] + 1) / (data['Room'] + 1)
    
    # 填充租房类型
#     data = SVM_RentType(data)
    data.loc[(data['rentType'] == '未知方式') & (data['Room'] <= 1), 'rentType'] = '整租'
    data.loc[(data['rentType'] == '未知方式') & (data['Room_Bath'] >= 1), 'rentType'] = '整租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] <= 30) & (data['Room'] == 2), 'rentType'] = '合租'  
    data.loc[(data['rentType'] == '未知方式') & (data['area'] > 60) & (data['Room'] == 2), 'rentType'] = '整租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] <= 60) & (data['Room'] == 3), 'rentType'] = '合租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] > 60) & (data['Room'] == 3), 'rentType'] = '整租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] >= 100) & (data['Room'] > 3), 'rentType'] = '整租'
    
    # 合并部分配套设施特征
    data['trainsportNum'] = 5 * data['subwayStationNum'] / data['subwayStationNum'].mean() + data['busStationNum'] /  data['busStationNum'].mean()
                                                                                           
    data['all_SchoolNum'] = 2 * data['interSchoolNum'] / data['interSchoolNum'].mean() + data['schoolNum'] / data['schoolNum'].mean() +  data['privateSchoolNum'] / data['privateSchoolNum'].mean()                    
                            
    data['all_hospitalNum'] = 2 * data['hospitalNum'] / data['hospitalNum'].mean() + data['drugStoreNum'] / data['drugStoreNum'].mean()
                              
    data['all_mall'] = data['mallNum'] / data['mallNum'].mean() + data['superMarketNum'] / data['superMarketNum'].mean()
                       
    data['otherNum'] = data['gymNum'] / data['gymNum'].mean() + data['bankNum'] / data['bankNum'].mean() + \
                       data['shopNum'] / data['shopNum'].mean() + 2 * data['parkNum'] / data['parkNum'].mean()

    data.drop(['subwayStationNum', 'busStationNum',
               'interSchoolNum', 'schoolNum', 'privateSchoolNum',
               'hospitalNum', 'drugStoreNum', 'mallNum', 'superMarketNum', 'gymNum', 'bankNum', 'shopNum', 'parkNum'],
              axis=1, inplace=True)
    
    data.drop('houseType', axis=1, inplace=True)
    
    return data
train = data_train.copy()
train = FeatureEngineering(train)
train.head()
test = data_test.copy()
test = FeatureEngineering(test)
test.head()
print('train shape: ',train.shape,'\n', 'test shape: ',test.shape)<jupyter_output>train shape:  (40160, 46) 
 test shape:  (2469, 45)<jupyter_text>计算统计特征<jupyter_code>def featureCount(train,test):
    train['data_type'] = 0
    test['data_type'] = 1
    data = pd.concat([train, test], axis=0, join='outer')
    
    def feature_count(data, features=[]):
        new_feature = 'count'
        for i in features:
            new_feature += '_' + i
        temp = data.groupby(features).size().reset_index().rename(columns={0: new_feature})
        data = data.merge(temp, 'left', on=features)
        return data

    data = feature_count(data, ['communityName'])
    data = feature_count(data, ['buildYear'])
    data = feature_count(data, ['totalFloor'])
    data = feature_count(data, ['communityName', 'totalFloor'])
    data = feature_count(data, ['communityName', 'newWorkers'])
    data = feature_count(data, ['communityName', 'totalTradeMoney'])
    new_train = data[data['data_type'] == 0]
    new_test = data[data['data_type'] == 1]
    new_train.drop('data_type', axis=1, inplace=True)
    new_test.drop(['data_type'], axis=1, inplace=True)
    return new_train, new_test
train, test = featureCount(train, test)
train.head()
test.head()
print('train shape: ',train.shape,'\n', 'test shape: ',test.shape)<jupyter_output>train shape:  (40160, 52) 
 test shape:  (2469, 52)<jupyter_text>grouby方法生成统计特征<jupyter_code>def groupby(train,test):
    train['data_type'] = 0
    test['data_type'] = 1
    data = pd.concat([train, test], axis=0, join='outer')
    columns = ['rentType', 'houseFloor', 'houseToward', 'houseDecoration', 'communityName', 'region', 'plate']
    for feature in columns:
        data[feature] = LabelEncoder().fit_transform(data[feature])

    temp = data.groupby('communityName')['area'].agg({'com_area_mean': 'mean', 'com_area_std': 'std'})
    temp.fillna(0, inplace=True)
    data = data.merge(temp, on='communityName', how='left')
    
    data['price_per_area'] = data.tradeMeanPrice / data.area * 100
    temp = data.groupby('communityName')['price_per_area'].agg(
        {'comm_price_mean': 'mean', 'comm_price_std': 'std'})
    temp.fillna(0, inplace=True)
    data = data.merge(temp, on='communityName', how='left')
   
    temp = data.groupby('plate')['price_per_area'].agg(
        {'plate_price_mean': 'mean', 'plate_price_std': 'std'})
    temp.fillna(0, inplace=True)
    data = data.merge(temp, on='plate', how='left')
    data.drop('price_per_area', axis=1, inplace=True)

    temp = data.groupby('plate')['area'].agg({'plate_area_mean': 'mean', 'plate_area_std': 'std'})
    temp.fillna(0, inplace=True)
    data = data.merge(temp, on='plate', how='left')
    
    temp = data.groupby(['plate'])['buildYear'].agg({'plate_year_mean': 'mean', 'plate_year_std': 'std'})
    data = data.merge(temp, on='plate', how='left')
    data.plate_year_mean = data.plate_year_mean.astype('int')
    data['comm_plate_year_diff'] = data.buildYear - data.plate_year_mean
    data.drop('plate_year_mean', axis=1, inplace=True)

    temp = data.groupby('plate')['trainsportNum'].agg('sum').reset_index(name='plate_trainsportNum')
    data = data.merge(temp, on='plate', how='left')
    temp = data.groupby(['communityName', 'plate'])['trainsportNum'].agg('sum').reset_index(name='com_trainsportNum')
    data = data.merge(temp, on=['communityName', 'plate'], how='left')
    data['trainsportNum_ratio'] = list(map(lambda x, y: round(x / y, 3) if y != 0 else -1,
                                           data['com_trainsportNum'], data['plate_trainsportNum']))
    data = data.drop(['com_trainsportNum', 'plate_trainsportNum'], axis=1)

    temp = data.groupby('plate')['all_SchoolNum'].agg('sum').reset_index(name='plate_all_SchoolNum')
    data = data.merge(temp, on='plate', how='left')
    temp = data.groupby(['communityName', 'plate'])['all_SchoolNum'].agg('sum').reset_index(name='com_all_SchoolNum')
    data = data.merge(temp, on=['communityName', 'plate'], how='left')
    data = data.drop(['com_all_SchoolNum', 'plate_all_SchoolNum'], axis=1)

    temp = data.groupby(['communityName', 'plate'])['all_mall'].agg('sum').reset_index(name='com_all_mall')
    data = data.merge(temp, on=['communityName', 'plate'], how='left')

    temp = data.groupby('plate')['otherNum'].agg('sum').reset_index(name='plate_otherNum')
    data = data.merge(temp, on='plate', how='left')
    temp = data.groupby(['communityName', 'plate'])['otherNum'].agg('sum').reset_index(name='com_otherNum')
    data = data.merge(temp, on=['communityName', 'plate'], how='left')
    data['other_ratio'] = list(map(lambda x, y: round(x / y, 3) if y != 0 else -1,
                                   data['com_otherNum'], data['plate_otherNum']))
    data = data.drop(['com_otherNum', 'plate_otherNum'], axis=1)

    temp = data.groupby(['month', 'communityName']).size().reset_index(name='communityName_saleNum')
    data = data.merge(temp, on=['month', 'communityName'], how='left')
    temp = data.groupby(['month', 'plate']).size().reset_index(name='plate_saleNum')
    data = data.merge(temp, on=['month', 'plate'], how='left')

    data['sale_ratio'] = round((data.communityName_saleNum + 1) / (data.plate_saleNum + 1), 3)
    data['sale_newworker_differ'] = 3 * data.plate_saleNum - data.newWorkers
    data.drop(['communityName_saleNum', 'plate_saleNum'], axis=1, inplace=True)

    new_train = data[data['data_type'] == 0]
    new_test = data[data['data_type'] == 1]
    new_train.drop('data_type', axis=1, inplace=True)
    new_test.drop(['data_type'], axis=1, inplace=True)
    return new_train, new_test
# train = data_train.copy()
# test = data_test.copy()
# train = FeatureEngineering(train)
# test = FeatureEngineering(test)

train,test = groupby(train, test)
train.head()
test.head()
print('train shape: ',train.shape,'\n', 'test shape: ',test.shape)<jupyter_output>train shape:  (40160, 67) 
 test shape:  (2469, 67)<jupyter_text>聚类方法<jupyter_code>from sklearn.mixture import GaussianMixture
def cluster(train,test):
    train['data_type'] = 0
    test['data_type'] = 1
    data = pd.concat([train, test], axis=0, join='outer')
    col = ['totalFloor',
           'houseDecoration', 'communityName', 'region', 'plate', 'buildYear',

           'tradeMeanPrice', 'tradeSecNum', 'totalNewTradeMoney',
           'totalNewTradeArea', 'tradeNewMeanPrice', 'tradeNewNum', 'remainNewNum',

           'landTotalPrice', 'landMeanPrice', 'totalWorkers',
           'newWorkers', 'residentPopulation', 'lookNum',
           'trainsportNum',
           'all_SchoolNum', 'all_hospitalNum', 'all_mall', 'otherNum']

    # EM
    gmm = GaussianMixture(n_components=3, covariance_type='full', random_state=0)
    data['cluster']= pd.DataFrame(gmm.fit_predict(data[col]))


    col1 = ['totalFloor','houseDecoration', 'communityName', 'region', 'plate', 'buildYear']
    col2 = ['tradeMeanPrice', 'tradeSecNum', 'totalNewTradeMoney',
            'totalNewTradeArea', 'tradeNewMeanPrice', 'tradeNewNum', 'remainNewNum',
            'landTotalPrice', 'landMeanPrice', 'totalWorkers',
            'newWorkers', 'residentPopulation', 'lookNum',
            'trainsportNum',
            'all_SchoolNum', 'all_hospitalNum', 'all_mall', 'otherNum']
    for feature1 in col1:
        for feature2 in col2:
        
            temp = data.groupby(['cluster',feature1])[feature2].agg('mean').reset_index(name=feature2+'_'+feature1+'_cluster_mean')
            temp.fillna(0, inplace=True)
       
            data = data.merge(temp, on=['cluster', feature1], how='left')
    
   
    new_train = data[data['data_type'] == 0]
    new_test = data[data['data_type'] == 1]
    new_train.drop('data_type', axis=1, inplace=True)
    new_test.drop(['data_type'], axis=1, inplace=True)
    
    return new_train, new_test
# train_cluster = data_train.copy()
# test_cluster = data_test.copy()
# train_cluster = FeatureEngineering(train_cluster)
# test_cluster = FeatureEngineering(test_cluster)

# train_cluster, test_cluster = cluster(train_cluster, test_cluster)
train, test = cluster(train, test)
train.head()
test.head()
print('train shape: ',train.shape,'\n', 'test shape: ',test.shape)<jupyter_output>train shape:  (40160, 176) 
 test shape:  (2469, 176)<jupyter_text>log平滑过大量级值取log平滑（针对线性模型有效）<jupyter_code>big_num_cols = ['totalTradeMoney','totalTradeArea','tradeMeanPrice','totalNewTradeMoney', 'totalNewTradeArea',
                'tradeNewMeanPrice','remainNewNum', 'supplyNewNum', 'supplyLandArea',
                'tradeLandArea','landTotalPrice','landMeanPrice','totalWorkers','newWorkers',
                'residentPopulation','pv','uv']
for col in big_num_cols:
        train[col] = train[col].map(lambda x: np.log1p(x))
        test[col] = test[col].map(lambda x: np.log1p(x))
train.head()
test.head()
print('train shape: ',train.shape,'\n', 'test shape: ',test.shape)<jupyter_output>train shape:  (40160, 176) 
 test shape:  (2469, 176)<jupyter_text>结果对比<jupyter_code>train.drop('Type', axis=1, inplace=True)
test.drop('Type', axis=1, inplace=True)
test.drop('tradeMoney', axis=1, inplace=True)
train.shape
test.shape
# 提取目标值
target_train = train.pop('tradeMoney') # train中的tradeMoney也被扔掉
train.shape
# 载入test目标值
target_test = pd.read_csv('./Data_Set/评分文件/sub_a_913.csv')
target_test.shape<jupyter_output><empty_output><jupyter_text>特征工程前<jupyter_code>def Obejct2Num(df):
    # 转换object
    columns = ['rentType','communityName','houseType', 'houseFloor', 'houseToward', 'houseDecoration',  'region', 'plate']
    for feature in columns:
        df[feature] = LabelEncoder().fit_transform(df[feature])
        
    return df
train_before = data_train.copy()
test_before = data_test.copy()

train_before = Obejct2Num(train_before)
test_before = Obejct2Num(test_before)

train_before.drop(['Type','tradeMoney'], axis=1, inplace=True)
test_before.drop('Type', axis=1, inplace=True)
train_before.shape
test_before.shape
#对比特征工程前后线性模型结果情况
test=test.fillna(0)
# Lasso回归
from sklearn.linear_model import Lasso
lasso=Lasso(alpha=0.1)
lasso.fit(train_before, target_train)

#预测测试集和训练集结果
y_pred_train=lasso.predict(train_before)
y_pred_test=lasso.predict(test_before)

# 对比结果
from sklearn.metrics import r2_score
score_train=r2_score(y_pred_train,target_train)
print("训练集结果：",score_train)
score_test=r2_score(y_pred_test, target_test)
print("测试集结果：",score_test)<jupyter_output>训练集结果： 0.6399521880920426
测试集结果： 0.7003568247309877<jupyter_text>特征工程过后<jupyter_code>#对比特征工程前后线性模型结果情况
test=test.fillna(0)
# Lasso回归
from sklearn.linear_model import Lasso
lasso=Lasso(alpha=0.1)
lasso.fit(train,target_train)
#预测测试集和训练集结果
y_pred_train=lasso.predict(train)
y_pred_test=lasso.predict(test)

#对比结果
from sklearn.metrics import r2_score
score_train=r2_score(y_pred_train,target_train)
print("训练集结果：",score_train)
score_test=r2_score(y_pred_test, target_test)
print("测试集结果：",score_test)<jupyter_output>训练集结果： 0.7379464712167035
测试集结果： 0.8285349866247926<jupyter_text>经过多轮特征工程后，明显看见准确率的提升  二、特征选择 [1. 相关系数法](1.相关系数法)      [2. Wrapper](2.Wrapper) [3. Embedded](3.Embedded)  1.相关系数法<jupyter_code># 卡方分布
from sklearn.feature_selection import SelectKBest
sk=SelectKBest(k=150)
new_train=sk.fit_transform(train,target_train)
print(new_train.shape)

# 获取对应列索引
select_columns=sk.get_support(indices = True)
# print(select_columns)

# 获取对应列名
# print(test.columns[select_columns])
select_columns_name=test.columns[select_columns]
new_test=test[select_columns_name]
print(new_test.shape)
# Lasso回归
from sklearn.linear_model import Lasso

lasso=Lasso(alpha=0.1)
lasso.fit(new_train,target_train)
#预测测试集和训练集结果
y_pred_train=lasso.predict(new_train)

y_pred_test=lasso.predict(new_test)

#对比结果
from sklearn.metrics import r2_score
score_train=r2_score(y_pred_train,target_train)
print("训练集结果：",score_train)
score_test=r2_score(y_pred_test, target_test)
print("测试集结果：",score_test)<jupyter_output>(40160, 150)
(2469, 150)
训练集结果： 0.7255924958670174
测试集结果： 0.8119200588240878<jupyter_text>2.Wrapper<jupyter_code>from sklearn.feature_selection import RFE
from sklearn.linear_model import LinearRegression
lr = LinearRegression()
rfe = RFE(lr, n_features_to_select=160)
rfe.fit(train,target_train)

RFE(estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,
                               normalize=False),
    n_features_to_select=40, step=1, verbose=0)




select_columns = [f for f, s in zip(train.columns, rfe.support_) if s]
# print(select_columns)
new_train = train[select_columns]
new_test = test[select_columns]

# Lasso回归
from sklearn.linear_model import Lasso

lasso=Lasso(alpha=0.1)
lasso.fit(new_train,target_train)
#预测测试集和训练集结果
y_pred_train=lasso.predict(new_train)

y_pred_test=lasso.predict(new_test)

#对比结果
from sklearn.metrics import r2_score
score_train=r2_score(y_pred_train,target_train)
print("训练集结果：",score_train)
score_test=r2_score(y_pred_test, target_test)
print("测试集结果：",score_test)<jupyter_output>训练集结果： 0.7355381511913996
测试集结果： 0.8256406226261237<jupyter_text>3.Embedded 3.1基于惩罚项的特征选择法<jupyter_code># Embedded
# 基于惩罚项的特征选择法
# Lasso(l1)和Ridge(l2)

from sklearn.linear_model import Ridge

ridge = Ridge(alpha=5)
ridge.fit(train,target_train)

Ridge(alpha=5, copy_X=True, fit_intercept=True, max_iter=None, normalize=False,
      random_state=None, solver='auto', tol=0.001)

# 特征系数排序
coefSort = ridge.coef_.argsort()
print(coefSort)


# 特征系数
featureCoefSore=ridge.coef_[coefSort]
# print(featureCoefSore)


select_columns = [f for f, s in zip(train.columns, featureCoefSore) if abs(s)> 0.0000005 ] 
# 选择绝对值大于0.0000005的特征

new_train = train[select_columns]
new_test = test[select_columns]
# Lasso回归
from sklearn.linear_model import Lasso

lasso=Lasso(alpha=0.1)
lasso.fit(new_train,target_train)
#预测测试集和训练集结果
y_pred_train=lasso.predict(new_train)

y_pred_test=lasso.predict(new_test)

#对比结果
from sklearn.metrics import r2_score
score_train=r2_score(y_pred_train,target_train)
print("训练集结果：",score_train)
score_test=r2_score(y_pred_test, target_test)
print("测试集结果：",score_test)
new_train.head()<jupyter_output><empty_output><jupyter_text>3.2 基于树模型的特征选择法 随机森林 平均不纯度减少（mean decrease impurity）<jupyter_code># Embedded
# 基于树模型的特征选择法
# 随机森林 平均不纯度减少（mean decrease impurity


from sklearn.ensemble import RandomForestRegressor
rf = RandomForestRegressor()
# 训练随机森林模型，并通过feature_importances_属性获取每个特征的重要性分数。rf = RandomForestRegressor()
rf.fit(train,target_train)
print("Features sorted by their score:")
print(sorted(zip(map(lambda x: round(x, 4), rf.feature_importances_), train.columns),
             reverse=True))

select_columns = [f for f, s in zip(train.columns, rf.feature_importances_) if abs(s)> 0.00005 ] 
# 选择绝对值大于0.00005的特征

new_train = train[select_columns]
new_test = test[select_columns]

# Lasso回归
from sklearn.linear_model import Lasso

lasso=Lasso(alpha=0.1)
lasso.fit(new_train,target_train)
#预测测试集和训练集结果
y_pred_train=lasso.predict(new_train)

y_pred_test=lasso.predict(new_test)

#对比结果
from sklearn.metrics import r2_score
score_train=r2_score(y_pred_train,target_train)
print("训练集结果：",score_train)
score_test=r2_score(y_pred_test, target_test)
print("测试集结果：",score_test)
new_train.head()
new_train.shape
new_test.head()
new_test.shape
target_train.head()
target_train.columns = ['tradeMoney']
target_train<jupyter_output><empty_output><jupyter_text>存储数据存储特征工程后的数据，用于之后的模型选择<jupyter_code>Train_data = new_train
Test_data = new_test
Train_data.to_csv('./Data_Set/X_Train_Data.csv', header=True)
Test_data.to_csv('./Data_Set/X_Test_Data.csv', header=True)
target_train.to_csv('./Data_Set/Y_Train_Data.csv', header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19538725261025403, 'fraction_numerical': 0.03556178899797413, 'mean_word_length': 5.665143331948483, 'pattern_counts': {'":': 0, '<': 141, '<?xml version=': 0, '>': 157, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '636457', 'license': 'MIT', 'n_tokens_mistral': 13534, 'n_tokens_neox': 12731, 'n_words': 1537, 'path': '2. Feature Engineering and Selection.ipynb', 'repo_name': 'Jiachengciel/Futurelab-2019-House-Rent-Forecast', 'size': '32085'}