<jupyter_start><jupyter_text>Sequential API<jupyter_code># 销毁当前的 TF 图并创建一个新图,避免旧模型/网络层混乱
tf.keras.backend.clear_session()
tf.random.set_seed(42)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Input

# 构建输入层、隐藏层、输出层,建立神经网络模型
model_1 = Sequential()
model_1.add(Input(shape=X_train.shape[1:]))
model_1.add(Dense(30, activation="relu"))
model_1.add(Dense(1))
# 设定神经网络的损失函数，优化器
model_1.compile(loss="mean_squared_error",
                optimizer="sgd")
# 看各层的参数个数
model_1.summary()
# 训练神经网络
history_1 = model_1.fit(X_train, y_train, epochs=30,
                        validation_data=(X_valid, y_valid))
# 看测试集的评分
mse_test_1 = model_1.evaluate(X_test, y_test)
# 查看测试集数据的预测值
y_pred_1 = model_1.predict(X_test)
y_pred_1
# 保存model_1模型到keras_model_1.h5中
model_1.save("keras_model_1.h5")
# 加载keras_model_1.h5的模型
# model_1 = keras.models.load_model("keras_model.h5")
def loss_compared(history):
    """
    画训练集、测试集中损失函数的值变化
    
    :param history: 神经网络的训练过程
    """

    labels = list(history.history.keys())
    plt.figure(figsize=(8, 4))
    for i in labels:
        plt.plot(history.history[i], label=i)
    plt.legend()
    plt.grid(True)
    plt.show()

loss_compared(history_1)   
save_fig("learning_curves_1")<jupyter_output><empty_output><jupyter_text>Function API ![wide_deep_model](./learning_images/wide_deep_model.png)<jupyter_code># 销毁当前的 TF 图并创建一个新图,避免旧模型/网络层混乱
tf.keras.backend.clear_session()
tf.random.set_seed(42)
from tensorflow.keras.layers import Input, Dense, concatenate
from tensorflow.keras.models import Model

# 构建输入层、隐藏层、输出层，形成 Wide&Deep neural network，具体参照上图
input_ = Input(shape=X_train.shape[1:])
hidden1 = Dense(30, activation="relu")(input_)
hidden2 = Dense(30, activation="relu")(hidden1)
concat = concatenate([input_, hidden2])
output = Dense(1)(concat)
model_2 = Model(inputs=[input_], outputs=[output])
# 图神经网络的各层参数信息
model_2.summary()
# 设置损失函数、优化器
model_2.compile(loss="mean_squared_error",
              optimizer=keras.optimizers.Adam())
# 保存迭代过程中，使得在验证集中损失函数值最小的模型到keras_model_2.h5中
checkpoint_cb = keras.callbacks.ModelCheckpoint("keras_model_2.h5", save_best_only=True)
# 训练模型
history_2 = model_2.fit(X_train, y_train, epochs=30,
                        validation_data=(X_valid, y_valid),
                        callbacks=[checkpoint_cb])
loss_compared(history_2)
save_fig("learning_curves_2")
# 通过测试集获得评分
mse_test_2 = model_2.evaluate(X_test, y_test)
# 模型对测试集的预测值
y_pred_2 = model_2.predict(X_test)
y_pred_2
# 从keras_model_2.h5中加载迭代过程中使得损失函数最小的模型
model_2_best = keras.models.load_model("keras_model_2.h5")
mse_test_best = model_2_best.evaluate(X_test, y_test)<jupyter_output>78/78 [==============================] - 0s 611us/sample - loss: 0.4907<jupyter_text>![overlapping_feature](./learning_images/overlapping_feature.png)<jupyter_code># 销毁当前的 TF 图并创建一个新图,避免旧模型/网络层混乱
tf.keras.backend.clear_session()
tf.random.set_seed(42)
# 构建两个输入层，建立二输入的神经网络，具体参照上图
input_A = Input(shape=[8], name="wide_input")
input_B = Input(shape=[12], name="deep_input")
hidden1 = Dense(30, activation="relu")(input_B)
hidden2 = Dense(30, activation="relu")(hidden1)
concat = concatenate([input_A, hidden2])
output = Dense(1, name="output")(concat)
model_3 = Model(inputs=[input_A, input_B], outputs=[output])
# 设定模型的损失函数和优化器
model_3.compile(loss="mse", optimizer="adam")
# 保存迭代过程中，使得在验证集中损失函数值最小的模型到keras_model_3.h5中
checkpoint_cb = keras.callbacks.ModelCheckpoint("keras_model_3.h5", save_best_only=True)
# 使用早期停止法，防止过拟合
early_stopping_cb = keras.callbacks.EarlyStopping(patience=5,
                                                  restore_best_weights=True)
# 将训练集、测试集、验证集的特征值分成两类
X_train_A, X_train_B = X_train.iloc[:, :8], X_train.iloc[:, 4:]
X_valid_A, X_valid_B = X_valid.iloc[:, :8], X_valid.iloc[:, 4:]
X_test_A, X_test_B = X_test.iloc[:, :8], X_test.iloc[:, 4:]
history_3 = model_3.fit((X_train_A, X_train_B), y_train, epochs=100,
                         validation_data=((X_valid_A, X_valid_B), y_valid),
                         callbacks=[checkpoint_cb, early_stopping_cb])
loss_compared(history_3)
save_fig("learning_curves_3")
# 使用验证集获取模型的评分
mse_test_3 = model_3.evaluate((X_test_A, X_test_B), y_test)
# 用模型来预测测试集的数据
y_pred_3 = model_3.predict((X_test_A, X_test_B))
y_pred_3<jupyter_output><empty_output><jupyter_text>![regularization](./learning_images/regularization.png)<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)
# 构建两个输入层，两个输出层，建立正则化的Wide & Deep neural network 
input_A = Input(shape=[8], name="wide_input")
input_B = Input(shape=[12], name="deep_input")
hidden1 = Dense(30, activation="relu")(input_B)
hidden2 = Dense(30, activation="relu")(hidden1)
concat = concatenate([input_A, hidden2])
output = Dense(1, name="main_output")(concat)
aux_output = Dense(1, name="aux_output")(hidden2)
model_4 = Model(inputs=[input_A, input_B],
              outputs=[output, aux_output])
# 构建模型的损失函数，优化器，各输出层权重
model_4.compile(loss=["mse", "mse"], loss_weights=[0.9, 0.1],
              optimizer="adam")
# 训练模型
history_4 = model_4.fit([X_train_A, X_train_B], [y_train, y_train], epochs=30,
                    validation_data=([X_valid_A, X_valid_B], [y_valid, y_valid]))
# 获取整个神经网络的损失函数值及单个输出层的损失函数值
total_loss, main_loss, aux_loss = model_4.evaluate([X_test_A, X_test_B], [y_test, y_test])
loss_compared(history_4)
save_fig("learning_curves_4")
# 获取两个输出层对测试集数据的预测值
y_pred_main, y_pred_aux = model_4.predict([X_test_A, X_test_B])
y_pred_main, y_pred_aux<jupyter_output><empty_output><jupyter_text>Subclassing API<jupyter_code>class WideAndDeepModel(keras.models.Model):
    """建立动态模型"""
    def __init__(self, units=30, activation="relu", **kwargs):
        super().__init__(**kwargs)
        self.hidden1 = keras.layers.Dense(units, activation=activation)
        self.hidden2 = keras.layers.Dense(units, activation=activation)
        self.main_output = keras.layers.Dense(1)
        self.aux_output = keras.layers.Dense(1)
        
    def call(self, inputs):
        input_A, input_B = inputs
        hidden1 = self.hidden1(input_B)
        hidden2 = self.hidden2(hidden1)
        concat = keras.layers.concatenate([input_A, hidden2])
        main_output = self.main_output(concat)
        aux_output = self.aux_output(hidden2)
        return main_output, aux_output
# 构建二输入、二输出的正则化神经网络模型
model_5 = WideAndDeepModel(30, activation="relu")
# 选择模型的损失函数、优化器、各输出层的权重
model_5.compile(loss="mse",
                loss_weights=[0.9, 0.1],
                optimizer=keras.optimizers.Adam())
# 训练模型
history_5 = model_5.fit((np.array(X_train_A), np.array(X_train_B)), (np.array(y_train), np.array(y_train)),
                        epochs=30, validation_data=((np.array(X_valid_A), np.array(X_valid_B)),(np.array(y_valid), np.array(y_valid))))
# # 获取整个神经网络的损失函数值及单个输出层的损失函数值
total_loss, main_loss, aux_loss = model_5.evaluate((X_test_A, X_test_B), (y_test, y_test))
loss_compared(history_5)
save_fig("learning_curves_5")<jupyter_output><empty_output><jupyter_text>Fine-Tuning Hyparameters<jupyter_code># 摧毁掉当前的tf图
keras.backend.clear_session()
np.random.seed(42)
tf.random.set_seed(42)
def build_model(n_hidden=1, n_neurons=30, learning_rate=3e-3, input_shape=[16]):
    """
    搭建简单全连接的DNN
    
    :param n_hidden: 隐藏层的层数
    :param n_neurons: 单层神经元个数
    :param learning_rate: 学习率
    :param input_shape: 训练集的特征维度
    """
    model = keras.models.Sequential()
    model.add(keras.layers.InputLayer(input_shape=input_shape))
    for layer in range(n_hidden):
        model.add(keras.layers.Dense(n_neurons, activation="relu"))
    model.add(keras.layers.Dense(1))
    optimizer = keras.optimizers.SGD(lr=learning_rate)
    model.compile(loss="mse", optimizer=optimizer)
    return model
# 调用构建好的模型，同scikit_learn库结合
keras_reg = keras.wrappers.scikit_learn.KerasRegressor(build_model)
# 训练模型，使用早停法
keras_reg.fit(X_train, y_train, epochs=100,
              validation_data=(X_valid, y_valid),
              callbacks=[keras.callbacks.EarlyStopping(patience=10)])
# 获取模型评分
mse_test = keras_reg.score(X_test, y_test)
# 模型对测试集的预测值
y_pred = keras_reg.predict(X_test)
from scipy.stats import reciprocal
from sklearn.model_selection import RandomizedSearchCV

# 超参数范围
param_distribs = {
    "n_hidden": [0, 1, 2, 3],
    "n_neurons": np.arange(1, 100),
    "learning_rate": reciprocal(3e-4, 3e-2),
}

# 进行超参数查找，设置交叉验证次数为3
rnd_search_cv = RandomizedSearchCV(keras_reg, param_distribs, n_iter=10, cv=3, n_jobs=-1, verbose=2)
rnd_search_cv.fit(X_train, y_train, epochs=100,
                  validation_data=(X_valid, y_valid),
                  callbacks=[keras.callbacks.EarlyStopping(patience=10)])
# 获取最优超参数
rnd_search_cv.best_params_
# 获取最优模型评分，scikit_learn库使用loss函数的负数形式，即为效用函数
rnd_search_cv.best_score_
# 获取最优评分的模型
rnd_search_cv.best_estimator_
# 获取最优模型的对测试集的评分
rnd_search_cv.score(X_test, y_test)<jupyter_output>78/78 [==============================] - 0s 54us/sample - loss: 0.1182
{'fraction_non_alphanumeric': 0.13890457368718237, 'fraction_numerical': 0.026538678712591756, 'mean_word_length': 5.298719772403983, 'pattern_counts': {'":': 3, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '451031', 'license': 'MIT', 'n_tokens_mistral': 3989, 'n_tokens_neox': 4166, 'n_words': 460, 'path': 'dnn_keras/dnn_keras_regression.ipynb', 'repo_name': 'Geeksongs/new-algorithm', 'size': '8855'}