<jupyter_start><jupyter_text>数据处理  导入相关的包<jupyter_code>import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
%time
# read data
# df_data = pd.read_csv("../raw_data/d_train.csv",encoding="gbk")
#rename clumns's name
rename_dict = {"性别":"gender","年龄":"age","体检日期":"date","血糖":"blood_sugar"}
re_rename_dict = {v:k for k,v in zip(rename_dict.keys(),rename_dict.values())}
df_data.rename(columns=rename_dict,inplace=True)
features = [x for x in list(df_data.columns) if x not in ["gender","age","date","blood_sugar","id"]]
fea_rename = { v:"feature_" + str(k) for k,v in enumerate(features)}
re_fea_rename = {v:k for k, v in zip(fea_rename.keys(), fea_rename.values())}
df_data["gender"] = df_data.gender.apply(lambda x: 1 if x == "男" else 0)
df_data.rename(columns=fea_rename,inplace=True)
df_data.head(10)
df_data.drop(["feature_15","feature_16","feature_17","feature_18","feature_19"],axis=1,inplace=True)
df_data.shape
df_data.drop(df_data[df_data["age"] < 20].index, inplace=True)
df_data.drop(df_data[df_data["age"] > 85].index,inplace=True)
df_data.drop(df_data[df_data["blood_sugar"] > 15].index,inplace=True)
# 3 num's log smoothness
df_data["blood_sugar"] = np.log(df_data["blood_sugar"])
df_data["blood_sugar"] = np.log(df_data["blood_sugar"])
df_data["blood_sugar"] = np.log(df_data["blood_sugar"])
df_data["blood_sugar"] = - df_data["blood_sugar"]
df_data.shape
df_data.blood_sugar.describe()
df_data_des = df_data.describe()
re_rename_dict = {v:k for k,v in zip(rename_dict.keys(),rename_dict.values())}
re_fea_rename = {v:k for k, v in zip(fea_rename.keys(), fea_rename.values())}
re_fea_rename.update(re_rename_dict)
print(re_fea_rename)
# get describe for features
# df_data_des.rename(columns=re_fea_rename,inplace=True)
# df_data_des.to_excel('./feature_describe.xlsx',encoding='gbk')
# del the row that have nan
# df_data_without_nan = df_data.dropna(axis=0,how='any')
# df_data_without_nan.rename(columns=re_fea_rename,inplace=True)
# df_data_without_nan.to_csv('./base_train.csv',encoding='gbk',index=False)
df_data[df_data['feature_28'].isnull().values == True]<jupyter_output><empty_output><jupyter_text>缺失值模型训练以及填充缺失值  导入相关的包<jupyter_code>from sklearn.ensemble import RandomForestRegressor

base_train = pd.read_csv("./base_train.csv")
rf_test = df_data[df_data["feature_8"].isnull().values == True]
cols = list(rf_test.columns)
print(cols)
cols_FT = list(rf_test.isnull().any())
[k for k,v in zip(cols, cols_FT) if v == False]
df_nan_data = pd.read_csv('./train_with_nan.csv',encoding="gbk")
c = "feature_0"
rf_pre_data = df_nan_data[df_nan_data[c].isnull().values == True]
no_index = [x for x in list(df_nan_data.index) if x not in list(rf_pre_data.index)]
rf_no_pre_data = df_nan_data.ix[no_index,:]
all_rf_data = pd.concat([rf_no_pre_data, rf_pre_data],axis=0)
no_nan_cols = [k for k,v in zip(list(rf_pre_data.columns),list(rf_pre_data.isnull().any())) if
               v == False and k not in ['id',"date","blood_sugar"]]
mean = round(df_nan_data[c].mean(),3)
trains = np.array(base_train[no_nan_cols])
labels = np.array(base_train[c])
tests = np.array(rf_pre_data[no_nan_cols])
rf = RandomForestRegressor()
rf.fit(trains, labels)
rf_pre_data[c] = [(2 * x * mean)/ (x + mean) for x in list(rf.predict(tests))]
df_nan_data.ix[rf_pre_data.index,c] = rf_pre_data[c]
df_nan_data = pd.read_csv('./train_with_nan.csv',encoding="gbk")
print(round(df_nan_data[c].mean(),3))
print(df_nan_data.ix[rf_pre_data.index,c])
df_nan_data.ix[rf_pre_data.index,c] = rf_pre_data[c]
print(rf_pre_data[c])
print(df_nan_data.ix[rf_pre_data.index,c])
print(df_nan_data[c])
df_test = pd.read_csv('./d_test_A.csv',encoding='gbk')
rename_dict = {"性别":"gender","年龄":"age","体检日期":"date","血糖":"blood_sugar"}
re_rename_dict = {v:k for k,v in zip(rename_dict.keys(),rename_dict.values())}
df_test.rename(columns=rename_dict,inplace=True)
features = [x for x in list(df_test.columns) if x not in ["gender","age","date","blood_sugar","id"]]
fea_rename = { v:"feature_" + str(k) for k,v in enumerate(features)}
re_fea_rename = {v:k for k, v in zip(fea_rename.keys(), fea_rename.values())}
df_test["gender"] = df_test.gender.apply(lambda x: 1 if x == "男" else 0)
df_test.rename(columns=fea_rename,inplace=True)
df_test.drop(["feature_15","feature_16","feature_17","feature_18","feature_19"],axis=1,inplace=True)
df_test.to_csv('./pre_test.csv',index=False,encoding='gbk')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1763779527559055, 'fraction_numerical': 0.009898762654668167, 'mean_word_length': 8.479744136460555, 'pattern_counts': {'":': 8, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '400792', 'license': 'Apache-2.0', 'n_tokens_mistral': 1880, 'n_tokens_neox': 1884, 'n_words': 269, 'path': 'jupyter/.ipynb_checkpoints/data_processing-checkpoint.ipynb', 'repo_name': 'cyzLoveDream/TNB', 'size': '4445'}