<jupyter_start><jupyter_text>**정규화 및 모델링** 정규화<jupyter_code># from sklearn.preprocessing import StandardScaler
# scaler = StandardScaler()
# df[num_columns] = scaler.fit_transform(df[num_columns])
# df[num_columns]
!pip install category_encoders
from sklearn import linear_model
from sklearn.linear_model import LinearRegression
from sklearn import svm
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.neighbors import KNeighborsRegressor


from sklearn.metrics import mean_absolute_error
from sklearn.pipeline import make_pipeline
from category_encoders import OrdinalEncoder

from sklearn.metrics import mean_squared_error, r2_score, mean_squared_log_error
# df = df.drop("DATE", axis=1)
x = df.drop('DFEDTAR', axis=1)
y = df['DFEDTAR']
from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.4, random_state=20, shuffle=True)
x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.5, random_state=2020,shuffle=True)

x_train.shape, x_test.shape, y_train.shape, x_val.shape<jupyter_output><empty_output><jupyter_text>선형회귀<jupyter_code>lg = linear_model.LinearRegression()

lg.fit(x_train, y_train)      
print('coefficient = ', lg.coef_[0]) # 독립변수의 계수를 출력
print('intercept = ', lg.intercept_)
print('mean_squared_error', mean_squared_error(y_train, lg.predict(x_train))) # 절편을 출력<jupyter_output>coefficient =  0.0005517301416845362
intercept =  -4.986236568654231
mean_squared_error 0.4484365194263439<jupyter_text>SGD<jupyter_code>sgd = linear_model.SGDRegressor()

sgd.fit(x_train, y_train)      
print('coefficient = ', sgd.coef_[0]) # 독립변수의 계수를 출력
print('intercept = ', sgd.intercept_)
print('mean_squared_error', mean_squared_error(y_train, sgd.predict(x_train))) # 절편을 출력<jupyter_output>coefficient =  293286468640203.2
intercept =  [-1.46699896e+11]
mean_squared_error 4.048122866753416e+39<jupyter_text>BayesianRidge<jupyter_code>reg = linear_model.BayesianRidge()

reg.fit(x_train, y_train)
print('coefficient = ', reg.coef_[0]) # 독립변수의 계수를 출력
print('intercept = ', reg.intercept_)
print('mean_squared_error', mean_squared_error(y_train, reg.predict(x_train))) # 절편을 출력<jupyter_output>coefficient =  0.0005535250563370219
intercept =  -4.919033396420122
mean_squared_error 0.4484639702535901<jupyter_text>비교<jupyter_code>#정확도 지표로 어느것을 사용할 수 있나 
sklearn.metrics.SCORERS.keys()
from sklearn.model_selection import KFold #for K-fold cross validation
from sklearn.model_selection import cross_val_score #score evaluation
from sklearn.model_selection import cross_val_predict #prediction

kfold = KFold(n_splits=5, random_state=20, shuffle=True) # k=10, split the data into 10 equal parts

xyz=[]
accuracy=[]
std=[]

Regressors=['Linear Regression',
             'Random Forest',
             'XGB',
             'LGBM',
             'KNeighbors']

models=[LinearRegression(),
        RandomForestRegressor(),
        XGBRegressor(),
        LGBMRegressor(),
        KNeighborsRegressor()]

for i in models:
    model = i
    cv_result = cross_val_score(model , x , y, cv = kfold, scoring = "neg_mean_squared_error")
    cv_result=cv_result

    xyz.append(cv_result.mean())
    std.append(cv_result.std())
    accuracy.append(cv_result)

new_models_dataframe2 = pd.DataFrame(
    {'mean_squared_error_CV Mean':xyz, 'Std':std},
     index=Regressors
     )      
 
new_models_dataframe2.round(4)<jupyter_output>[16:12:50] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
[16:12:50] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
[16:12:51] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
[16:12:51] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
[16:12:52] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.<jupyter_text>**RandomForestRegressor ML** 먼저 xgb부스팅모델<jupyter_code>from xgboost.sklearn import XGBModel
from xgboost import XGBRegressor
pipe = make_pipeline(
    OrdinalEncoder(), 
    XGBRegressor()
)

pipe.fit(x_train, y_train)
print('훈련 R^2: ', pipe.score(x_train, y_train))
print('검증 R^2: ', pipe.score(x_val, y_val))
print('TEST R^2: ', pipe.score(x_test, y_test))

print('\n훈련 MAE: ', mean_absolute_error(pipe.predict(x_train), y_train))
print('검증 MAE: ', mean_absolute_error(pipe.predict(x_val), y_val))
print('TEST MAE: ', mean_absolute_error(pipe.predict(x_test), y_test))<jupyter_output>[16:13:05] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
훈련 R^2:  0.9975120517235978
검증 R^2:  0.9970168414894269
TEST R^2:  0.9969588130282647

훈련 MAE:  0.06075673339009819
검증 MAE:  0.06396850506901304
TEST MAE:  0.06534298454355074<jupyter_text>랜덤 포레스트 .,,,<jupyter_code>from sklearn.ensemble import RandomForestRegressor
from sklearn.pipeline import Pipeline
# encoder, imputer를 preprocessing으로 묶었습니다. 후에 eli5 permutation 계산에 사용합니다
pipe = Pipeline([
    ('preprocessing', make_pipeline(OrdinalEncoder())),
    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) 
])

pipe.fit(x_train, y_train)
print('훈련 R^2: ', pipe.score(x_train, y_train))
print('검증 R^2: ', pipe.score(x_val, y_val))
print('TEST R^2: ', pipe.score(x_test, y_test))

print('\n훈련 MAE: ', mean_absolute_error(pipe.predict(x_train), y_train))
print('검증 MAE: ', mean_absolute_error(pipe.predict(x_val), y_val))
print('TEST MAE: ', mean_absolute_error(pipe.predict(x_test), y_test))<jupyter_output>훈련 R^2:  0.9999400757166952
검증 R^2:  0.9996794620287783
TEST R^2:  0.9992072039293762

훈련 MAE:  0.002900386650386649
검증 MAE:  0.006759768009768006
TEST MAE:  0.010488102501525317<jupyter_text>**Feature importance**<jupyter_code>pipe.named_steps
!pip install eli5
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

import eli5
from eli5.sklearn import PermutationImportance

# permuter 정의
permuter = PermutationImportance(
    pipe.named_steps['rf'], # model
    scoring='r2', # metric
    n_iter=5, # 다른 random seed를 사용하여 5번 반복
    random_state=20
)

# permuter 계산은 preprocessing 된 X_val을 사용합니다.
x_val_transformed = pipe.named_steps['preprocessing'].transform(x_val)

# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다
permuter.fit(x_val_transformed, y_val);
feature_names = x_val.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values()
# 특성별 score 확인
eli5.show_weights(
    permuter, 
    top=None, # top n 지정 가능, None 일 경우 모든 특성 
    feature_names=feature_names # list 형식으로 넣어야 합니다
)<jupyter_output><empty_output><jupyter_text>통화량을 알면 금리를 맞출수 있는 확률이 너무나도 커진다는 것을,,,, 알게 되었습니다. (사실 원래 아는거 였지만 이렇게 크게 작용할 줄은 몰랐습니다.) **다시측정**  M2 통화량을 빼고 측정해 보도록 하겠습니다.<jupyter_code>df_x_m2 = df.drop("M2", axis=1)
df_x_m2
x_ = df_x_m2.drop('DFEDTAR', axis=1)
y_ = df_x_m2['DFEDTAR']
x_, y_
from sklearn.model_selection import train_test_split

x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)
x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)

x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape
x_train_.head(),x_train_.shape
pipe_ = Pipeline([
    ('preprocessing', make_pipeline(OrdinalEncoder())),
    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) 
])

pipe_.fit(x_train_, y_train_)
print('훈련 R^2: ', pipe_.score(x_train_, y_train_))
print('검증 R^2: ', pipe_.score(x_val_, y_val_))
print('TEST R^2: ', pipe_.score(x_test_, y_test_))

print('\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))
print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))
print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))<jupyter_output>훈련 R^2:  0.9999034834500641
검증 R^2:  0.9991185714437719
TEST R^2:  0.9987776192958645

훈련 MAE:  0.004187525437525434
검증 MAE:  0.01199481074481074
TEST MAE:  0.013145210494203781<jupyter_text>거의 변화없음 ---<jupyter_code># permuter 정의
permuter = PermutationImportance(
    pipe_.named_steps['rf'], # model
    scoring='r2', # metric
    n_iter=5, # 다른 random seed를 사용하여 5번 반복
    random_state=20
)

# permuter 계산은 preprocessing 된 X_val을 사용합니다.
x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)

# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다
permuter.fit(x_val_transformed, y_val_);
feature_names = x_val_.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values()

# 특성별 score 확인
eli5.show_weights(
    permuter, 
    top=None, # top n 지정 가능, None 일 경우 모든 특성 
    feature_names=feature_names # list 형식으로 넣어야 합니다
)<jupyter_output><empty_output><jupyter_text>실업률이랑 달러/위완화 만으로도 금리 예측이 어느정도 가능하다 ?---<jupyter_code>df_x_m2 = df_x_m2.drop("UNRATE", axis=1)
df_x_m2
x_ = df_x_m2.drop('DFEDTAR', axis=1)
y_ = df_x_m2['DFEDTAR']
x_, y_

from sklearn.model_selection import train_test_split

x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)
x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)

x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape

pipe_ = Pipeline([
    ('preprocessing', make_pipeline(OrdinalEncoder())),
    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) 
])

pipe_.fit(x_train_, y_train_)
print('훈련 R^2: ', pipe_.score(x_train_, y_train_))
print('검증 R^2: ', pipe_.score(x_val_, y_val_))
print('TEST R^2: ', pipe_.score(x_test_, y_test_))

print('\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))
print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))
print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))
# permuter 정의
permuter = PermutationImportance(
    pipe_.named_steps['rf'], # model
    scoring='r2', # metric
    n_iter=5, # 다른 random seed를 사용하여 5번 반복
    random_state=20
)

# permuter 계산은 preprocessing 된 X_val을 사용합니다.
x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)

# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다
permuter.fit(x_val_transformed, y_val_);

feature_names = x_val_.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values()

# 특성별 score 확인
eli5.show_weights(
    permuter, 
    top=None, # top n 지정 가능, None 일 경우 모든 특성 
    feature_names=feature_names # list 형식으로 넣어야 합니다
)<jupyter_output><empty_output><jupyter_text>---<jupyter_code>df_x_m2 = df_x_m2.drop("USD/CNY", axis=1)
df_x_m2
#타겟설정
x_ = df_x_m2.drop('DFEDTAR', axis=1)
y_ = df_x_m2['DFEDTAR']
x_, y_
#데이터 분할 
from sklearn.model_selection import train_test_split

x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)
x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)

x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape
#파이프라인구축
pipe_ = Pipeline([
    ('preprocessing', make_pipeline(OrdinalEncoder())),
    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) 
])
#훈련
pipe_.fit(x_train_, y_train_)
#검증
print('훈련 R^2: ', pipe_.score(x_train_, y_train_))
print('검증 R^2: ', pipe_.score(x_val_, y_val_))
print('TEST R^2: ', pipe_.score(x_test_, y_test_))

print('\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))
print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))
print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))
# permuter 정의
permuter = PermutationImportance(
    pipe_.named_steps['rf'], # model
    scoring='r2', # metric
    n_iter=5, # 다른 random seed를 사용하여 5번 반복
    random_state=20
)

# permuter 계산은 preprocessing 된 X_val을 사용합니다.
x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)

# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다
permuter.fit(x_val_transformed, y_val_);

feature_names = x_val_.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values()

# 특성별 score 확인
eli5.show_weights(
    permuter, 
    top=None, # top n 지정 가능, None 일 경우 모든 특성 
    feature_names=feature_names # list 형식으로 넣어야 합니다
)
pd.Series(permuter.feature_importances_, x_test_.columns).sort_values()
import seaborn as sns
zipp  = []
for zipper in zip(x_train_.columns, permuter.feature_importances_):
  zipp.append(zipper)
zipp = pd.DataFrame(zipp,columns=['feature','importance']).sort_values('importance',ascending=False)
plt.figure(figsize=(5,5))
sns.barplot(y = zipp.feature[:10], x= zipp.importance[:10], palette='Blues_r')
plt.title("Permutation Importance",fontsize=20);<jupyter_output><empty_output><jupyter_text>zg는 금 가격인데 왜 ?  금리 예측에 도움이 되는걸까 ,,, ZG를 빼도 결과가 너무 좋다<jupyter_code>df_x_m2 = df_x_m2.drop("ZI", axis=1)

x_ = df_x_m2.drop('DFEDTAR', axis=1)
y_ = df_x_m2['DFEDTAR']
x_, y_

from sklearn.model_selection import train_test_split

x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)
x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)

x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape

pipe_ = Pipeline([
    ('preprocessing', make_pipeline(OrdinalEncoder())),
    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) 
])

pipe_.fit(x_train_, y_train_)
print('훈련 R^2: ', pipe_.score(x_train_, y_train_))
print('검증 R^2: ', pipe_.score(x_val_, y_val_))
print('TEST R^2: ', pipe_.score(x_test_, y_test_))

print('\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))
print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))
print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))
# permuter 정의
permuter = PermutationImportance(
    pipe_.named_steps['rf'], # model
    scoring='r2', # metric
    n_iter=5, # 다른 random seed를 사용하여 5번 반복
    random_state=20
)

# permuter 계산은 preprocessing 된 X_val을 사용합니다.
x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)

# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다
permuter.fit(x_val_transformed, y_val_);

feature_names = x_val_.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values()

# 특성별 score 확인
eli5.show_weights(
    permuter, 
    top=None, # top n 지정 가능, None 일 경우 모든 특성 
    feature_names=feature_names # list 형식으로 넣어야 합니다
)
df_x_m2 = df_x_m2.drop("HSN1F", axis=1)


x_ = df_x_m2.drop('DFEDTAR', axis=1)
y_ = df_x_m2['DFEDTAR']
x_, y_

from sklearn.model_selection import train_test_split

x_train_, x_test_, y_train_, y_test_ = train_test_split(x_, y_, test_size=0.4, random_state=20, shuffle=True)
x_val_, x_test_, y_val_, y_test_ = train_test_split(x_test_, y_test_, test_size=0.5, random_state=20,shuffle=True)

x_train_.shape, x_test_.shape, x_val_.shape, y_train_.shape, y_test_.shape, y_val_.shape

pipe_ = Pipeline([
    ('preprocessing', make_pipeline(OrdinalEncoder())),
    ('rf', RandomForestRegressor(n_estimators=100, random_state=20, n_jobs=-1)) 
])

pipe_.fit(x_train_, y_train_)
print('훈련 R^2: ', pipe_.score(x_train_, y_train_))
print('검증 R^2: ', pipe_.score(x_val_, y_val_))
print('TEST R^2: ', pipe_.score(x_test_, y_test_))

print('\n훈련 MAE: ', mean_absolute_error(pipe_.predict(x_train_), y_train_))
print('검증 MAE: ', mean_absolute_error(pipe_.predict(x_val_), y_val_))
print('TEST MAE: ', mean_absolute_error(pipe_.predict(x_test_), y_test_))

print('\n훈련 MAE: ', mean_squared_error(pipe_.predict(x_train_), y_train_))
print('검증 MAE: ', mean_squared_error(pipe_.predict(x_val_), y_val_))
print('TEST MAE: ', mean_squared_error(pipe_.predict(x_test_), y_test_))
# permuter 정의
permuter = PermutationImportance(
    pipe_.named_steps['rf'], # model
    scoring='neg_mean_absolute_error', # metric
    n_iter=5, # 다른 random seed를 사용하여 5번 반복
    random_state=20
)

# permuter 계산은 preprocessing 된 X_val을 사용합니다.
x_val_transformed = pipe_.named_steps['preprocessing'].transform(x_val_)

# 실제로 fit 의미보다는 스코어를 다시 계산하는 작업입니다
permuter.fit(x_val_transformed, y_val_);

feature_names = x_val_.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values()

# 특성별 score 확인
eli5.show_weights(
    permuter, 
    top=None, # top n 지정 가능, None 일 경우 모든 특성 
    feature_names=feature_names # list 형식으로 넣어야 합니다
)<jupyter_output><empty_output><jupyter_text>원자재 가격이 이렇게 큰 영향을 미친다고 ?? ---<jupyter_code># dpi(dots per inch) 수치를 조정해 이미지 화질을 조정 
import matplotlib.pyplot as plt
plt.rcParams['figure.dpi'] = 144
!pip install pdpbox
from pdpbox.pdp import pdp_isolate, pdp_plot
feature = 'DFEDTAR'
!pip install shap
row = x_test_.iloc[[1]]  # 중첩 brackets을 사용하면 결과물이 DataFrame입니다
row
y_test_.iloc[[1]]
# 모델 예측값
pipe_.predict(row)
explainer<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12654592345114588, 'fraction_numerical': 0.04360822929488307, 'mean_word_length': 6.254251700680272, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '95529', 'license': 'MIT', 'n_tokens_mistral': 7906, 'n_tokens_neox': 7997, 'n_words': 1289, 'path': '1. ML_baseline.ipynb', 'repo_name': 'syh0397/Benchmark-interest-rate-prediction', 'size': '17061'}