<jupyter_start><jupyter_text>Step 1: load data<jupyter_code>local_path = "/home/jwzhang411898961/Dropbox/DataScience/Kaggle/HousingPrice"

train_df = pd.read_csv(local_path + "/train.csv")
test_df = pd.read_csv(local_path + "/test.csv")<jupyter_output><empty_output><jupyter_text>Step 2: merge data<jupyter_code>prices = pd.DataFrame({'price': train_df['SalePrice'], \
                       'log(price+1)': np.log1p(train_df['SalePrice'])})
prices.head()
y_train = np.log1p(train_df.pop('SalePrice'))
all_df = pd.concat([train_df, test_df], axis=0)<jupyter_output><empty_output><jupyter_text>Step 3：variable transformation<jupyter_code>print(all_df['MSSubClass'].dtypes)
all_df['MSSubClass'] = all_df['MSSubClass'].astype(str)
print(all_df['MSSubClass'].dtypes)
print(all_df['MSSubClass'].value_counts())
MSSubclass_dummies_df = pd.get_dummies(all_df['MSSubClass'], prefix='MSSubClass').head()
all_dummy_df = pd.get_dummies(all_df)
print(all_dummy_df.head())
# print(all_dummy_df.isnull().sum().sort_values(ascending = False).head(11))
mean_cols = all_dummy_df.mean()
all_dummy_df = all_dummy_df.fillna(mean_cols)
print(all_dummy_df.isnull().sum().sum())
numeric_cols = all_df.columns[all_df.dtypes != 'object']
print(numeric_cols)
numeric_col_means = all_dummy_df.loc[:, numeric_cols].mean()
numeric_col_std = all_dummy_df.loc[:, numeric_cols].std()
all_dummy_df.loc[:, numeric_cols] = \
(all_dummy_df.loc[:,numeric_cols] - numeric_col_means) / numeric_col_std<jupyter_output>Index(['Id', 'LotFrontage', 'LotArea', 'OverallQual', 'OverallCond',
       'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2',
       'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF',
       'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath',
       'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces',
       'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF',
       'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal',
       'MoSold', 'YrSold'],
      dtype='object')<jupyter_text>Step 4: 建立模型<jupyter_code># 把数据处理之后，送回训练集和测试集
dummy_train_df = all_dummy_df.iloc[:len(train_df)]
dummy_test_df = all_dummy_df.iloc[len(train_df):]
print(dummy_train_df.shape,dummy_test_df.shape)

# 将DF数据转换成Numpy Array的形式，更好地配合sklearn
X_train = dummy_train_df.values
X_test = dummy_test_df.values
y_train.shape
alphas = np.logspace(-3,2,50)
test_scores = []

for alpha in alphas:
    clf = Ridge(alpha)
    test_score = np.sqrt(-cross_val_score(clf,X_train,y_train,cv = 10,scoring = 'neg_mean_squared_error'))
    test_scores.append(np.mean(test_score))
plt.plot(alphas, test_scores)
plt.title('Alpha VS CV Error')
plt.show()
max_features = [.1,.3,.5,.7,.9,.99]
test_scores = []
for max_feat in max_features:
    clf = RandomForestRegressor(n_estimators = 200,max_features = max_feat)
    test_score = np.sqrt(-cross_val_score(clf,X_train,y_train,cv = 5,scoring = 'neg_mean_squared_error'))
    test_scores.append(np.mean(test_score))
plt.plot(max_features,test_scores)
plt.title('Max Features vs CV Error')
plt.show()<jupyter_output><empty_output><jupyter_text>Step 5: Ensemble<jupyter_code>ridge = Ridge(alpha = 15)
rf = RandomForestRegressor(n_estimators = 500,max_features = .3)
ridge.fit(X_train,y_train)
rf.fit(X_train,y_train)

y_ridge = np.expm1(ridge.predict(X_test))
y_rf = np.expm1(rf.predict(X_test))

y_final = (y_ridge + y_rf) / 2
submission_df = pd.DataFrame(data = {'Id':test_df.index,'SalePrice':y_final})
print(submission_df.head(10))
submission_df.to_csv(local_path + '/submission1.csv',columns = ['Id','SalePrice'],index = False)<jupyter_output>Id      SalePrice
0   0  119511.573363
1   1  150913.726425
2   2  173609.841018
3   3  188962.706188
4   4  195185.230625
5   5  175082.476654
6   6  176337.771857
7   7  168274.264983
8   8  182607.592642
9   9  122382.276311
{'fraction_non_alphanumeric': 0.1536465092135998, 'fraction_numerical': 0.05035037633013236, 'mean_word_length': 6.512670565302145, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '439913', 'license': 'MIT', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1618, 'n_words': 211, 'path': 'HousePrice_model1.ipynb', 'repo_name': 'jwzhang411898961/HousingPrice', 'size': '3853'}