<jupyter_start><jupyter_text>7.1 투표 기반 분류기  1. hard_voting<jupyter_code>from sklearn.datasets import make_moons

X, y = make_moons(n_samples = 1000, noise = 0.15)

from sklearn.model_selection import train_test_split

X_train, X_val, y_train, y_val = train_test_split(X, y, random_state = 42)
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC

log_clf = LogisticRegression()
rnd_clf = RandomForestClassifier()
svm_clf = SVC()

voting_clf = VotingClassifier(
    estimators = [('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
    voting = 'hard'
)
voting_clf.fit(X_train,y_train)
from sklearn.metrics import accuracy_score
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
  clf.fit(X_train, y_train)
  y_pred = clf.predict(X_val)
  print(clf.__class__.__name__, accuracy_score(y_val, y_pred))<jupyter_output>LogisticRegression 0.864
RandomForestClassifier 0.98
SVC 0.98
VotingClassifier 0.98<jupyter_text>2. soft_voting<jupyter_code>log_clf = LogisticRegression()
rnd_clf = RandomForestClassifier()
svm_clf = SVC(probability=True)

voting_clf = VotingClassifier(
    estimators = [('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
    voting = 'soft'
)
voting_clf.fit(X_train,y_train)
from sklearn.metrics import accuracy_score
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
  clf.fit(X_train, y_train)
  y_pred = clf.predict(X_val)
  print(clf.__class__.__name__, accuracy_score(y_val, y_pred))<jupyter_output>LogisticRegression 0.864
RandomForestClassifier 0.98
SVC 0.98
VotingClassifier 0.98<jupyter_text>7.2 배깅과 페이스팅  7.2.1 사이킷런의 배깅과 페이스팅<jupyter_code>from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier

bag_clf = BaggingClassifier(
    DecisionTreeClassifier(), n_estimators = 500,
    max_samples = 100, bootstrap = True, n_jobs = -1
)
bag_clf.fit(X_train, y_train)
y_pred = bag_clf.predict(X_val)<jupyter_output><empty_output><jupyter_text>7.2.1-1 결정 경계 시각화 코드<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

def plot_decision_boundaries(X, y, model_class, **model_params):
    """
    Function to plot the decision boundaries of a classification model.
    This uses just the first two columns of the data for fitting 
    the model as we need to find the predicted value for every point in 
    scatter plot.
    Arguments:
            X: Feature data as a NumPy-type array.
            y: Label data as a NumPy-type array.
            model_class: A Scikit-learn ML estimator class 
            e.g. GaussianNB (imported from sklearn.naive_bayes) or
            LogisticRegression (imported from sklearn.linear_model)
            **model_params: Model parameters to be passed on to the ML estimator
    
    Typical code example:
            plt.figure()
            plt.title("KNN decision boundary with neighbros: 5",fontsize=16)
            plot_decision_boundaries(X_train,y_train,KNeighborsClassifier,n_neighbors=5)
            plt.show()
    """
    try:
        X = np.array(X)
        y = np.array(y).flatten()
    except:
        print("Coercing input data to NumPy arrays failed")
    # Reduces to the first two columns of data
    reduced_data = X[:, :2]
    # Instantiate the model object
    model = model_class(**model_params)
    # Fits the model with the reduced data
    model.fit(reduced_data, y)

    # Step size of the mesh. Decrease to increase the quality of the VQ.
    h = .02     # point in the mesh [x_min, m_max]x[y_min, y_max].    

    # Plot the decision boundary. For that, we will assign a color to each
    x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
    y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
    # Meshgrid creation
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

    # Obtain labels for each point in mesh using the model.
    Z = model.predict(np.c_[xx.ravel(), yy.ravel()])    

    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                         np.arange(y_min, y_max, 0.1))

    # Predictions to obtain the classification results
    Z = model.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)

    # Plotting
    plt.contourf(xx, yy, Z, alpha=0.4)
    plt.scatter(X[:, 0], X[:, 1], c=y, alpha=0.8)
    plt.xlabel("Feature-1",fontsize=15)
    plt.ylabel("Feature-2",fontsize=15)
    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)
    return plt
plt.figure()
plt.title("Decision_tree_classifier",fontsize = 16)
plot_decision_boundaries(X_train, y_train, DecisionTreeClassifier)
plt.show()
plt.figure()
plt.title("RandomForestClassifier",fontsize = 16)
plot_decision_boundaries(X_train, y_train, RandomForestClassifier)
plt.show()
plt.figure()
plt.title("LogisticRegression",fontsize = 16)
plot_decision_boundaries(X_train, y_train, LogisticRegression)
plt.show()
from sklearn.neighbors import KNeighborsClassifier
n_neighbors = [1,3,5,7]
for i in n_neighbors:
  plt.figure()
  plt.title(f"KNeighborClassifier n_neighbors{i}",fontsize = 16)
  plot_decision_boundaries(X_train, y_train, KNeighborsClassifier,
                           n_neighbors = i)
  plt.show()
# 배깅도 시각화 가능하게 코드 수정 

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

def plot_decision_boundaries(X, y, model):
    """
    Function to plot the decision boundaries of a classification model.
    This uses just the first two columns of the data for fitting 
    the model as we need to find the predicted value for every point in 
    scatter plot.
    Arguments:
            X: Feature data as a NumPy-type array.
            y: Label data as a NumPy-type array.
            model_class: A Scikit-learn ML estimator class 
            e.g. GaussianNB (imported from sklearn.naive_bayes) or
            LogisticRegression (imported from sklearn.linear_model)
            **model_params: Model parameters to be passed on to the ML estimator
    
    Typical code example:
            plt.figure()
            plt.title("KNN decision boundary with neighbros: 5",fontsize=16)
            plot_decision_boundaries(X_train,y_train,KNeighborsClassifier,n_neighbors=5)
            plt.show()
    """
    try:
        X = np.array(X)
        y = np.array(y).flatten()
    except:
        print("Coercing input data to NumPy arrays failed")
    # Reduces to the first two columns of data
    reduced_data = X[:, :2]
    # Instantiate the model object
    model = model
    # Fits the model with the reduced data

    # Step size of the mesh. Decrease to increase the quality of the VQ.
    h = .02     # point in the mesh [x_min, m_max]x[y_min, y_max].    

    # Plot the decision boundary. For that, we will assign a color to each
    x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
    y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
    # Meshgrid creation
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

    # Obtain labels for each point in mesh using the model.
    Z = model.predict(np.c_[xx.ravel(), yy.ravel()])    

    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                         np.arange(y_min, y_max, 0.1))

    # Predictions to obtain the classification results
    Z = model.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)

    # Plotting
    plt.contourf(xx, yy, Z, alpha=0.4)
    plt.scatter(X[:, 0], X[:, 1], c=y, alpha=0.8)
    plt.xlabel("Feature-1",fontsize=15)
    plt.ylabel("Feature-2",fontsize=15)
    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)
    return plt
# 배깅도 시각화 가능하게 코드 수정 
from sklearn.datasets import make_moons

X, y = make_moons(n_samples = 1000, noise = 0.15)

from sklearn.model_selection import train_test_split

X_train, X_val, y_train, y_val = train_test_split(X, y, random_state = 42)


bag_clf = BaggingClassifier(
    DecisionTreeClassifier(), n_estimators = 500,
    max_samples = 100, bootstrap = True, n_jobs = -1
)
bag_clf.fit(X_train, y_train)

plt.figure()
plt.title("LogisticRegression",fontsize = 16)
plot_decision_boundaries(X_train, y_train, bag_clf)
plt.show()<jupyter_output><empty_output><jupyter_text>7.2.2 oob 평가<jupyter_code>bag_clf = BaggingClassifier(
    DecisionTreeClassifier(), n_estimators = 500,
    bootstrap = True, n_jobs = -1, oob_score = True
)
bag_clf.fit(X_train, y_train)
bag_clf.oob_score_
from sklearn.metrics import accuracy_score
y_pred = bag_clf.predict(X_val)
accuracy_score(y_val, y_pred)
bag_clf.oob_decision_function_<jupyter_output><empty_output><jupyter_text>7.3 랜덤 패치와 랜덤 서브스페이스<jupyter_code># 랜덤패치 
bag_clf = BaggingClassifier(
    DecisionTreeClassifier(), n_estimators = 500,
    bootstrap = False, max_features = 1, n_jobs = -1
)
bag_clf.fit(X_train, y_train)
y_pred = bag_clf.predict(X_val)
accuracy_score(y_val, y_pred)
# 서브스페이스
bag_clf = BaggingClassifier(
    DecisionTreeClassifier(), n_estimators = 500,
    bootstrap = True, max_features = 0.8, n_jobs = -1
)
bag_clf.fit(X_train, y_train)
y_pred = bag_clf.predict(X_val)
accuracy_score(y_val, y_pred)<jupyter_output><empty_output><jupyter_text>7.4 랜덤 포레스트<jupyter_code>from sklearn.ensemble import RandomForestClassifier

rnd_clf = RandomForestClassifier(n_estimators = 500, max_leaf_nodes=16, n_jobs=-1)
rnd_clf.fit(X_train, y_train)

y_pred_rf = rnd_clf.predict(X_val)
bag_clf = BaggingClassifier(
    DecisionTreeClassifier(max_features='sqrt', max_leaf_nodes=16), n_estimators = 500)
rnd_clf.fit(X_train, y_train)
y_pred = rnd_clf.predict(X_val)
accuracy_score(y_val, y_pred)
bag_clf.fit(X_train, y_train)
y_pred = bag_clf.predict(X_val)
accuracy_score(y_val, y_pred)<jupyter_output><empty_output><jupyter_text>7.4.1 엑스트라트리<jupyter_code>from sklearn.ensemble import ExtraTreesClassifier

etc_clf = ExtraTreesClassifier()
etc_clf.fit(X_train, y_train)
y_pred = etc_clf.predict(X_val)
accuracy_score(y_val, y_pred)<jupyter_output><empty_output><jupyter_text>7.4.2 특성 중요도 ---------------- *참고*> - 트리 기반 모델의 피쳐 중요도 계산 -> https://soohee410.github.io/iml_tree_importance2> - 트리 기반 피쳐 중요도는 high cardinality가 높을 수록 중요한 변수로 인식한다. 특히 모델 과적합시에 더 심하다. -> https://soohee410.github.io/iml_tree_importance2> - model-specific한 방법 말고 model-agnostic한 방법인 permutation feature importance를 쓰자. robust 한 방법이다. 하지만 단점도 있다. -> https://soohee410.github.io/iml_permutation_importance<jupyter_code>from sklearn.datasets import load_iris
iris = load_iris()
rnd_clf = RandomForestClassifier(n_estimators=500, n_jobs=-1)
rnd_clf.fit(iris['data'], iris['target'])
for name, score in zip(iris['feature_names'], rnd_clf.feature_importances_):
  print(name, score)<jupyter_output>sepal length (cm) 0.09112643426932107
sepal width (cm) 0.02376438585556858
petal length (cm) 0.43340008866671026
petal width (cm) 0.45170909120839997<jupyter_text>7.5 부스팅  7.5.1 에이다부스트- Adaboost 작동원리 : https://bkshin.tistory.com/entry/%EB%A8%B8%EC%8B%A0%EB%9F%AC%EB%8B%9D-14-AdaBoost- 연속형 변수 임계점 잡기 : https://leedakyeong.tistory.com/entry/%EC%9D%98%EC%82%AC%EA%B2%B0%EC%A0%95%EB%82%98%EB%AC%B4Decision-Tree-%EB%8F%85%EB%A6%BD%EB%B3%80%EC%88%98%EA%B0%80-%EC%97%B0%EC%86%8D%ED%98%95-%EC%9D%BC-%EB%95%8C<jupyter_code>from sklearn.ensemble import AdaBoostClassifier


ada_clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth = 1), n_estimators = 200,
                             algorithm = 'SAMME.R', learning_rate = 0.5)
ada_clf.fit(X_train, y_train)
y_pred = ada_clf.predict(X_val)
accuracy_score(y_val, y_pred)<jupyter_output><empty_output><jupyter_text>7.5.2 그레이디언트 부스팅<jupyter_code>X = np.random.randn(100,1)
y = 3 * (X**2) + np.random.rand(100,1)
plt.scatter(X, y)
X_new = np.array([[0],[2]])
from sklearn.tree import DecisionTreeRegressor
y = y.ravel()

tree_reg1 = DecisionTreeRegressor(max_depth=2)
tree_reg1.fit(X, y)
y2 = y - tree_reg1.predict(X)
tree_reg2 = DecisionTreeRegressor(max_depth=2)
tree_reg2.fit(X,y2)
y3 = y2 - tree_reg2.predict(X)
tree_reg3 = DecisionTreeRegressor(max_depth=2)
tree_reg3.fit(X, y3)

y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
y_pred
from sklearn.ensemble import GradientBoostingRegressor

gbrt = GradientBoostingRegressor(max_depth = 2, n_estimators = 3, learning_rate = 1.0)
gbrt.fit(X, y)
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

X_train, X_val, y_train, y_val = train_test_split(X, y, random_state = 42)

gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=500)
gbrt.fit(X_train, y_train)

errors = [mean_squared_error(y_val, y_pred) \
          for y_pred in gbrt.staged_predict(X_val)
          ]
bst_n_estimators = np.argmin(errors) + 1

gbrt_best = GradientBoostingRegressor(max_depth=2, n_estimators=bst_n_estimators)
gbrt_best.fit(X_train, y_train)
bst_n_estimators
errors[245:250]
gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True)

y = y.tolist()

min_val_error = float('inf')
error_going_up = 0
for n_estimators in range(1,100):
  gbrt.n_estimators = n_estimators
  gbrt.fit(X_train, y_train)
  y_pred = gbrt.predict(X_val)
  val_error = mean_squared_error(y_val, y_pred)
  if val_error < min_val_error:
    min_val_error = min_val_error
    error_going_up = 0
  else:
    error_going_up += 1
    if error_going_up == 5:
      break # 조기종료 
import xgboost

xgb_reg = xgboost.XGBRegressor()
xgb_reg.fit(X_train, y_train)
y_pred = xgb_reg.predict(X_val)
xgb_reg.fit(X_train, y_train, eval_set = [(X_val, y_val)], early_stopping_rounds=2)
y_pred = xgb_reg.predict(X_val)<jupyter_output>[14:59:57] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.
[0]	validation_0-rmse:4.21241
Will train until validation_0-rmse hasn't improved in 2 rounds.
[1]	validation_0-rmse:3.90905
[2]	validation_0-rmse:3.63634
[3]	validation_0-rmse:3.3922
[4]	validation_0-rmse:3.17458
[5]	validation_0-rmse:2.9659
[6]	validation_0-rmse:2.7669
[7]	validation_0-rmse:2.59939
[8]	validation_0-rmse:2.43387
[9]	validation_0-rmse:2.28781
[10]	validation_0-rmse:2.15613
[11]	validation_0-rmse:2.04012
[12]	validation_0-rmse:1.92637
[13]	validation_0-rmse:1.82504
[14]	validation_0-rmse:1.73657
[15]	validation_0-rmse:1.65603
[16]	validation_0-rmse:1.57915
[17]	validation_0-rmse:1.51217
[18]	validation_0-rmse:1.45598
[19]	validation_0-rmse:1.40168
[20]	validation_0-rmse:1.3525
[21]	validation_0-rmse:1.30204
[22]	validation_0-rmse:1.25621
[23]	validation_0-rmse:1.20836
[24]	validation_0-rmse:1.16724
[25]	validation_0-rmse:1.12642
[26]	validation_0[...]<jupyter_text>7.6 스태킹-  참고 : https://github.com/rickiepark/handson-ml2/blob/master/07_ensemble_learning_and_random_forests.ipynb<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.datasets import fetch_openml

mnist = fetch_openml('mnist_784', version=1)
mnist.target = mnist.target.astype(np.uint8)

X_train_val, X_test, y_train_val, y_test = train_test_split(
    mnist.data, mnist.target, test_size=10000, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(
    X_train_val, y_train_val, test_size=10000, random_state=42)
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from sklearn.svm import LinearSVC
from sklearn.neural_network import MLPClassifier
random_forest_clf = RandomForestClassifier(n_estimators=100, random_state=42)
extra_trees_clf = ExtraTreesClassifier(n_estimators=100, random_state=42)
svm_clf = LinearSVC(max_iter=100, tol=20, random_state=42)
mlp_clf = MLPClassifier(random_state=42)
estimators = [random_forest_clf, extra_trees_clf, svm_clf, mlp_clf]
for estimator in estimators:
    print("Training the", estimator)
    estimator.fit(X_train, y_train)
[estimator.score(X_val, y_val) for estimator in estimators]
from sklearn.ensemble import VotingClassifier
named_estimators = [
    ("random_forest_clf", random_forest_clf),
    ("extra_trees_clf", extra_trees_clf),
    ("svm_clf", svm_clf),
    ("mlp_clf", mlp_clf),
]
voting_clf = VotingClassifier(named_estimators)
voting_clf.fit(X_train, y_train)
voting_clf.score(X_val, y_val)
[estimator.score(X_val, y_val) for estimator in voting_clf.estimators_]
voting_clf.set_params(svm_clf=None)
voting_clf.estimators
voting_clf.estimators_
del voting_clf.estimators_[2]
voting_clf.score(X_val, y_val)
voting_clf.voting = "hard"
voting_clf.score(X_test, y_test)
[estimator.score(X_test, y_test) for estimator in voting_clf.estimators_]
X_val_predictions = np.empty((len(X_val), len(estimators)), dtype=np.float32)

for index, estimator in enumerate(estimators):
    X_val_predictions[:, index] = estimator.predict(X_val)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11667554608417688, 'fraction_numerical': 0.04155567394778902, 'mean_word_length': 4.646390374331551, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 51, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '386592', 'license': 'MIT', 'n_tokens_mistral': 7364, 'n_tokens_neox': 7058, 'n_words': 1363, 'path': '_ipynb/Chapter7.ipynb', 'repo_name': 'Yeongjin-Jo/minimal-mistakes', 'size': '16893'}