<jupyter_start><jupyter_text>第4章: Theta-Misspecification<jupyter_code>import matplotlib.pyplot as plt
plt.style.use('ggplot')
import numpy as np
import torch
from torch.optim import Adam
from pytorchltr.datasets import MSLR30K

from model import MLPScoreFunc
from train import train_ranker
# 実験設定
batch_size = 32
hidden_layer_sizes = (10, 10)
learning_rate = 0.0001
n_epochs = 100
# MSLR30Kデータセットを読み込む（初回だけ時間がかかる）
train = MSLR30K(split="train")
test = MSLR30K(split="test")<jupyter_output><empty_output><jupyter_text>pow_true=1.0のデータに対して、pow_used=0.0として、MLPRankerを学習（ナイーブ推定量と同じ）<jupyter_code>torch.manual_seed(12345)
score_fn = MLPScoreFunc(
    input_size=train[0].features.shape[1],
    hidden_layer_sizes=hidden_layer_sizes,
)
optimizer = Adam(score_fn.parameters(), lr=learning_rate)

ndcg_score_list_naive = train_ranker(
    score_fn=score_fn,
    optimizer=optimizer,
    estimator="ips",
    train=train,
    test=test,
    batch_size=batch_size,
    pow_true=1.0,
    pow_used=0.0,
    n_epochs=n_epochs,
)<jupyter_output>100%|██████████| 100/100 [13:23<00:00,  8.04s/it]<jupyter_text>pow_true=1.0のデータに対して、pow_used=0.5として、MLPRankerを学習（バイアスの過小見積もり）<jupyter_code>torch.manual_seed(12345)
score_fn = MLPScoreFunc(
    input_size=train[0].features.shape[1],
    hidden_layer_sizes=hidden_layer_sizes,
)
optimizer = Adam(score_fn.parameters(), lr=learning_rate)

ndcg_score_list_ips_small_pow = train_ranker(
    score_fn=score_fn,
    optimizer=optimizer,
    estimator="ips",
    train=train,
    test=test,
    batch_size=batch_size,
    pow_true=1.0,
    pow_used=0.5,
    n_epochs=n_epochs,
)<jupyter_output>100%|██████████| 100/100 [13:25<00:00,  8.05s/it]<jupyter_text>pow_true=1.0のデータに対して、pow_used=1.0として、MLPRankerを学習<jupyter_code>torch.manual_seed(12345)
score_fn = MLPScoreFunc(
    input_size=train[0].features.shape[1],
    hidden_layer_sizes=hidden_layer_sizes,
)
optimizer = Adam(score_fn.parameters(), lr=learning_rate)

ndcg_score_list_ips = train_ranker(
    score_fn=score_fn,
    optimizer=optimizer,
    estimator="ips",
    train=train,
    test=test,
    batch_size=batch_size,
    pow_true=1.0,
    pow_used=1.0,
    n_epochs=n_epochs,
)<jupyter_output>100%|██████████| 100/100 [13:11<00:00,  7.91s/it]<jupyter_text>pow_true=1.0のデータに対して、pow_used=1.5として、MLPRankerを学習（バイアスの過大見積もり）<jupyter_code>torch.manual_seed(12345)
score_fn = MLPScoreFunc(
    input_size=train[0].features.shape[1],
    hidden_layer_sizes=hidden_layer_sizes,
)
optimizer = Adam(score_fn.parameters(), lr=learning_rate)

ndcg_score_list_ips_large_pow = train_ranker(
    score_fn=score_fn,
    optimizer=optimizer,
    estimator="ips",
    train=train,
    test=test,
    batch_size=batch_size,
    pow_true=1.0,
    pow_used=1.5,
    n_epochs=n_epochs,
)<jupyter_output>100%|██████████| 100/100 [13:05<00:00,  7.85s/it]<jupyter_text>学習曲線の描画（図4.18）<jupyter_code>plt.subplots(1, figsize=(8,6))
plt.plot(range(n_epochs), ndcg_score_list_naive, label="pow_used=0.0", linewidth=3, linestyle="dotted")
plt.plot(range(n_epochs), ndcg_score_list_ips_small_pow, label="pow_used=0.5", linewidth=3, linestyle="dashed")
plt.plot(range(n_epochs), ndcg_score_list_ips, label="pow_used=1.0", linewidth=3)
plt.plot(range(n_epochs), ndcg_score_list_ips_large_pow, label="pow_used=1.5", linewidth=3, linestyle="dashdot")
   
plt.title(f"Test nDCG@10 Curve Of IPS Estimator with Different Levels of Misspecification", fontdict=dict(size=15))
plt.xlabel("Number of Epochs", fontdict=dict(size=20))
plt.ylabel("Test nDCG@10", fontdict=dict(size=20))
plt.tight_layout()
plt.legend(loc="best", fontsize=20)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1425414364640884, 'fraction_numerical': 0.05331491712707182, 'mean_word_length': 6.975770925110132, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '269073', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1753, 'n_tokens_neox': 1640, 'n_words': 174, 'path': 'ch04/theta-misspecification.ipynb', 'repo_name': 'masakiaota/ml_design_book', 'size': '3620'}