<jupyter_start><jupyter_text>カテゴリ変数 * ワンホットエンコーディング（ダミー変数）<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import mglearn
from IPython.display import display
import os
#このファイルにはコラム名を含んだヘッダがないので、header=Noneを指定し、コラム名を"names"で明示的に指定。
adult_path = os.path.join(mglearn.datasets.DATA_PATH, "adult.data")
data = pd.read_csv(
    adult_path, header=None, index_col=False, 
    names=["age","workclass","fnlwgt","education","education-num","marital-status","occupation","relationship",
           "race","gender","capital-gain","capital-loss","hours-per-week","native-country","income"])
#解説のために、いくつかのカラムだけ選択
data = data[["age","workclass","education","gender","hours-per-week","occupation","income"]]
# IPython.displayを使うとJupyter notebookできれいな出力が得られる
display(data.head())<jupyter_output><empty_output><jupyter_text>* 文字列で表されているカテゴリデータのチェック<jupyter_code># pandasのDataFrame中の行を表すSeriesクラスの関数value_countsを用いて、チェック
print(data.gender.value_counts())
print(data.education.value_counts())
#pandasのget_dummies関数を使って簡単にデータをワンホットエンコーディングできる
print("Original features:\n", list(data.columns), "\n")
data_dummies = pd.get_dummies(data)
print("Features after get_dummies: \n", list(data_dummies.columns))
print(data_dummies)
data_dummies
# ageからoccupation_ Transport-movingまでを抜き出す
features = data_dummies.loc[:, "age":"occupation_ Transport-moving"]
# Numpy配列を取り出す
X = features.values
y = data_dummies["income_ >50K"].values
print("X.shape: {} y.shape: {}".format(X.shape, y.shape))
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
print("Test score: {:2f}".format(logreg.score(X_test, y_test)))<jupyter_output>C:\ProgramData\Anaconda3.2\lib\site-packages\sklearn\linear_model\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.
  FutureWarning)<jupyter_text>* 数値でエンコードされているカテゴリ<jupyter_code># 整数特徴量とカテゴリ文字列特徴量からなるDataFrameを作る
demo_df = pd.DataFrame({"Integer Feature": [0, 1, 2, 1],
                        "Categorical Feature": ["socks", "fox", "socks", "box"]})
display(demo_df)
pd.get_dummies(demo_df)
demo_df["Integer Feature"] = demo_df["Integer Feature"].astype(str)
pd.get_dummies(demo_df, columns=["Integer Feature", "Categorical Feature"])<jupyter_output><empty_output><jupyter_text>ビニング、離散型、線型モデル、決定木<jupyter_code>from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor

X,y = mglearn.datasets.make_wave(n_samples=100)
line = np.linspace(-3, 3, 1000, endpoint=False).reshape(-1, 1)

#決定木
reg = DecisionTreeRegressor(min_samples_split=3).fit(X,y)
plt.plot(line, reg.predict(line), label="decision tree")
#線形モデル
reg = LinearRegression().fit(X,y)
plt.plot(line, reg.predict(line), label="linear regression")

plt.plot(X[:, 0], y, "o", c="k")
plt.ylabel("Regressin output")
plt.xlabel("Input feature")
plt.legend(loc="best")
bins = np.linspace(-3, 3, 11)
print("bins: {}".format(bins))
which_bin = np.digitize(X, bins=bins)
print("\nData points:\n", X[:5])
print("\nBin membership for data points:\n", which_bin[:5])
from sklearn.preprocessing import OneHotEncoder
# OneHotEncoderで変換する
encoder = OneHotEncoder(sparse=False)
# encoder.fitでwhich_binに現れる整数値のバリエーションを確認
encoder.fit(which_bin)
# transformでワンホットエンコーディングを行う
X_binned = encoder.transform(which_bin)
print(X_binned[:5])
print("X_binned.shape: {}".format(X_binned.shape))
line_binned = encoder.transform(np.digitize(line, bins=bins))

reg = LinearRegression().fit(X_binned, y)
plt.plot(line, reg.predict(line_binned), label="linear regression binned")

reg = DecisionTreeRegressor(min_samples_split=3).fit(X_binned, y)
plt.plot(line, reg.predict(line_binned), label="decision tree binned")
plt.plot(X[:, 0], y, "o", c="k")
plt.vlines(bins, -3, 3, linewidth=1, alpha=.2)
plt.legend(loc="best")
plt.ylabel("Regression output")
plt.xlabel("Input feature")<jupyter_output><empty_output><jupyter_text>交互作用と多項式<jupyter_code>X_combined = np.hstack([X, X_binned])
print(X_combined.shape)
reg = LinearRegression().fit(X_combined, y)

line_combined = np.hstack([line, line_binned])
plt.plot(line, reg.predict(line_combined), label="linear regression combined")

for bin in bins:
    plt.plot([bin, bin], [-3, 3], ":", c="k")#垂直点線
plt.legend(loc=2)
plt.ylabel("Regression output")
plt.xlabel("Input feature")
plt.plot(X[:, 0], y, "o", c="k")
X_product = np.hstack([X_binned, X*X_binned])
print(X_product.shape)
reg = LinearRegression().fit(X_product, y)

line_product = np.hstack([line_binned, line*line_binned])
plt.plot(line, reg.predict(line_product), label="linear regression product")

for bin in bins:
    plt.plot([bin, bin], [-3, 3], ":", c="k")
    
plt.plot(X[:, 0], y, "o", c="k")
plt.ylabel("Regression output")
plt.xlabel("Input feature")
plt.legend(loc=2)
from sklearn.preprocessing import PolynomialFeatures

# x**10までの多項式を加える
# デフォルトの"include_bias=True"だと、常に１となる特徴量を加える
poly = PolynomialFeatures(degree=10, include_bias=False)
poly.fit(X)
X_poly = poly.transform(X)
print("X_poly.shape: {}".format(X_poly.shape))
print("Entries of X:\n{}".format(X[:5]))
print("Entries of X_poly:\n{}".format(X_poly[:5]))
# get_feature_namesで個々の特徴量の意味が分かる
print("Polynomial feature names:\n{}".format(poly.get_feature_names()))
reg = LinearRegression().fit(X_poly, y)

line_poly = poly.transform(line)
plt.plot(line, reg.predict(line_poly), label="polynomical linear regression")
plt.plot(X[:, 0], y, "o", c="k")
plt.ylabel("Regression output")
plt.xlabel("Input feature")
plt.legend(loc="best")
#比較としてカーネル法を用いたSVMモデルを、変換していないオリジナルデータに適用する
from sklearn.svm import SVR

for gamma in [1, 10]:
    svr = SVR(gamma=gamma).fit(X,y)
    plt.plot(line, svr.predict(line), label="SVR gamma={}".format(gamma))
    plt.plot(X[:, 0], y, "o", c="k")
    plt.ylabel("Regression output")
    plt.xlabel("Input feature")
    plt.legend(loc="best")
#交互作用特徴量と多項式特徴量を、より現実的なboston_housingデータセットに適用してみる。
#最初に、データをロードしてMinMaxScalerを用いて0から1の間になるようにスケール変換する

from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler

boston = load_boston()
X_train, X_test, y_train, y_test = train_test_split(boston.data, boston.target, random_state=0)

#データのスケール変換
scaler = MinMaxScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.fit_transform(X_test)

# 2次までの多項式特徴量と交互作用を抽出する
poly = PolynomialFeatures(degree=2).fit(X_train_scaled)
X_train_poly = poly.transform(X_train_scaled)
X_test_poly = poly.transform(X_test_scaled)
print("X_train.shape: {}".format(X_train.shape))
print("X_train_poly.shape: {}".format(X_train_poly.shape))
print("Polynomial feature names:\n{}".format(poly.get_feature_names()))
# Ridgeを使って、交互作用特徴量を入れた場合と入れない場合を比較してみる
from sklearn.linear_model import Ridge
ridge = Ridge().fit(X_train_scaled, y_train)
print("Score without interactions: {:.3f}".format(ridge.score(X_test_scaled, y_test)))
ridge = Ridge().fit(X_train_poly, y_train)
print("Score with interections: {:.3f}".format(ridge.score(X_test_poly, y_test)))
# 次はランダムフォレストのような複雑なモデルを使う
from sklearn.ensemble import RandomForestRegressor
rf = RandomForestRegressor(n_estimators=100).fit(X_train_scaled, y_train )
print("Score without interactions: {:.3f}".format(rf.score(X_test_scaled, y_test)))
rf = RandomForestRegressor(n_estimators=100).fit(X_train_poly, y_train)
print("Score with interactions: {:.3f}".format(rf.score(X_test_poly, y_test)))<jupyter_output>Score without interactions: 0.786
Score with interactions: 0.747<jupyter_text>単変量非線形変換<jupyter_code>rnd = np.random.RandomState(0)
X_org = rnd.normal(size=(1000, 3))
w = rnd.normal(size=3)

X = rnd.poisson(10 * np.exp(X_org))
y = np.dot(X_org, w)
print("Number of feature appearances:\n{}".format(np.bincount(X[:, 0])))
# bincountを可視化する
bins = np.bincount(X[:, 0])
plt.bar(range(len(bins)), bins, color="b")
plt.ylabel("Number of appearances")
plt.xlabel("Value")
bins = np.bincount(X[:, 1])
plt.bar(range(len(bins)), bins, color="b")
plt.ylabel("Number of appearances")
plt.xlabel("Value")
from sklearn.linear_model import Ridge
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
score = Ridge().fit(X_train, y_train).score(X_test, y_test)
print("Test score: {:.3f}".format(score))
# log0に対して定義できないため、log(x+1)
X_train_log = np.log(X_train+1)
X_test_log = np.log(X_test+1)
plt.hist(X_train_log[:, 0], bins=25, color="gray")
plt.ylabel("Number of appearances")
plt.xlabel("Value")
#変換後のデータでRidgeモデルを作る
score = Ridge().fit(X_train_log, y_train).score(X_test_log, y_test)
print("Test score: {:.3f}".format(score))<jupyter_output>Test score: 0.875<jupyter_text>自動特徴量選択 * 単変量統計<jupyter_code>from sklearn.datasets import load_breast_cancer
from sklearn.feature_selection import SelectPercentile
from sklearn.model_selection import train_test_split

cancer = load_breast_cancer()

#シードを指定して乱数を決定
rng = np.random.RandomState(42)
noise = rng.normal(size=(len(cancer.data), 50))
#ノイズ特徴量をデータに加える
#最初の30特徴量はデータセット。あとの50特徴量はノイズ
X_w_noise = np.hstack([cancer.data, noise])

X_train, X_test, y_train, y_test = train_test_split(X_w_noise, cancer.target, random_state=0, test_size=.5)
# f_classif(デフォルト)とSelectPercentileを使って50%の特徴量を選択
select = SelectPercentile(percentile=50)
select.fit(X_train, y_train)
#訓練セットを変換
X_train_selected = select.transform(X_train)

print("X_train.shape: {}".format(X_train.shape))
print("X_train_selected.shape: {}".format(X_train_selected.shape))
mask = select.get_support()
print(mask)
#マスクを可視化する
plt.matshow(mask.reshape(1, -1), cmap="gray_r")
plt.xlabel("Sample index")
#ロジスティック回帰で、すべての特徴量を使った場合と、選択された特徴量だけを使った場合で比較する
from sklearn.linear_model import LogisticRegression

#テストデータの変換
X_test_selected = select.transform(X_test)

#すべての特徴量
lr = LogisticRegression()
lr.fit(X_train, y_train)
print("Score with all features: {:.3f}".format(lr.score(X_test, y_test)))
#選択された特徴量
lr.fit(X_train_selected, y_train)
print("Score with only selected features: {:.3f}".format(lr.score(X_test_selected, y_test)))<jupyter_output>Score with all features: 0.930
Score with only selected features: 0.940<jupyter_text>* モデルベース特徴量選択<jupyter_code>from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import RandomForestClassifier
select = SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold="median")
#単変量統計とモデルベース特徴量選択を比較する
select.fit(X_train, y_train)
X_train_l1 = select.transform(X_train)
print("X_train.shape: {}".format(X_train.shape))
print("X_train_l1.shape: {}".format(X_train_l1.shape))
mask = select.get_support()
#マスクを可視化　--黒が真、白が偽
plt.matshow(mask.reshape(1, -1), cmap="gray_r")
plt.xlabel("Sample index")
X_test_l1 = select.transform(X_test)
score = LogisticRegression().fit(X_train_l1, y_train).score(X_test_l1, y_test)
print("Test score: {:.3f}".format(score))<jupyter_output>Test score: 0.951<jupyter_text>* 反復特徴量選択<jupyter_code>from sklearn.feature_selection import RFE
select = RFE(RandomForestClassifier(n_estimators=100, random_state=42), n_features_to_select=40)

select.fit(X_train, y_train)
#選択された特徴量を可視化する
mask = select.get_support()
plt.matshow(mask.reshape(1, -1), cmap="gray_r")
plt.xlabel("Sample index")
X_train_rfe = select.transform(X_train)
X_test_rfe = select.transform(X_test)
score = LogisticRegression().fit(X_train_rfe, y_train).score(X_test_rfe, y_test)
print("Test score: {:.3f}".format(score))
# RFE内部で用いられたモデルを使って予測を行うこともできる
print("Test score: {:.3f}".format(select.score(X_test, y_test)))<jupyter_output>Test score: 0.951<jupyter_text>専門家知識の利用<jupyter_code>#レンタル自転車におけるデータ予測を行う
citibike = mglearn.datasets.load_citibike()
print("Citi Bike data:\n{}".format(citibike.head()))
plt.figure(figsize=(10, 3))
xticks = pd.date_range(start=citibike.index.min(), end=citibike.index.max(), freq="D")
plt.xticks(xticks, xticks.strftime("%a %m-%d"), rotation=90, ha="left")
plt.plot(citibike, linewidth=1)
plt.xlabel("Date")
plt.ylabel("Rentals")
#ターゲット値（レンタル数）を抽出
y=citibike.values
# 10**9で割ってPOSIX時刻に変換
X = citibike.index.astype("int64").to_numpy().reshape(-1, 1) // 10**9
#最初の184データポイントを訓練に、残りのテストに使う
n_train = 184

#与えられた特徴量セットで、回帰器を評価しプロットする関数
def eval_on_features(features, target, regressor):
    #与えられた特徴量を訓練セットとテストセットに分割
    X_train, X_test = features[:n_train], features[n_train:]
    #ターゲットの配列も分割する
    y_train, y_test = target[:n_train], target[n_train:]
    regressor.fit(X_train, y_train)
    print("Test-set R^2: {:.2f}".format(regressor.score(X_test, y_test)))
    y_pred = regressor.predict(X_test)
    y_pred_train = regressor.predict(X_train)
    plt.figure(figsize=(10, 3))
    
    plt.xticks(range(0, len(X), 8), xticks.strftime("%a %m-%d"),rotation=90, ha="left")
    
    plt.plot(range(n_train), y_train, label="train")
    plt.plot(range(n_train, len(y_test) + n_train), y_test, "-", label="test")
    plt.plot(range(n_train), y_pred_train, "--", label="prediction train")
    
    plt.plot(range(n_train, len(y_test) + n_train), y_pred, "--", label="prediction test")
    plt.legend(loc=(1.01, 0))
    plt.xlabel("Date")
    plt.ylabel("Rental")
from sklearn.ensemble import RandomForestRegressor
regressor = RandomForestRegressor(n_estimators=100, random_state=0)
plt.figure()
eval_on_features(X, y, regressor)
X_hour = citibike.index.hour.to_numpy().reshape(-1, 1)
eval_on_features(X_hour, y, regressor)
X_hour_week = np.hstack([citibike.index.dayofweek.to_numpy().reshape(-1, 1),
                         citibike.index.hour.to_numpy().reshape(-1, 1)])
eval_on_features(X_hour_week, y, regressor)
#時刻をカテゴリ変数として解釈させることで、時刻が遅いほどレンタル数多くなるという線形モデルの解釈をただす。
#整数をOneHotEncoderを用いて、変換する
enc = OneHotEncoder()
X_hour_week_onehot = enc.fit_transform(X_hour_week).toarray()
eval_on_features(X_hour_week_onehot, y, Ridge())
#交互作用を用いて、曜日と時刻の組み合わせに対して係数を学習させる
poly_transformer = PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)
X_hour_week_onehot_poly = poly_transformer.fit_transform(X_hour_week_onehot)
lr = Ridge()
eval_on_features(X_hour_week_onehot_poly, y, lr)
#このモデルで学習した係数をプロットすることができる。ランダムフォレストではできない。
hour = ["%02d:00" % i for i in range(0, 24, 3)]
day = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
features = day + hour
features_poly = poly_transformer.get_feature_names(features)
features_nonzero = np.array(features_poly)[lr.coef_ != 0]
coef_nonzero = lr.coef_[lr.coef_ != 0]
plt.figure(figsize=(15, 2))
plt.plot(coef_nonzero, "o")
plt.xticks(np.arange(len(coef_nonzero)), features_nonzero, rotation=90)
plt.xlabel("Feature name")
plt.ylabel("Feature magnitude")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15400901516186313, 'fraction_numerical': 0.01577653326048354, 'mean_word_length': 7.950488997555012, 'pattern_counts': {'":': 5, '<': 35, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '574107', 'license': 'MIT', 'n_tokens_mistral': 6677, 'n_tokens_neox': 6339, 'n_words': 895, 'path': '4.feature engineering.ipynb', 'repo_name': 'kaaikatsunori/kaaikatsunori.github.io', 'size': '14642'}