<jupyter_start><jupyter_text>generate data according to experiment 3<jupyter_code>from scipy.stats import norm
def gen3(x,q):
  z=torch.from_numpy(norm.ppf(q)).float()
  muintermediate=torch.cos(x*2*2*3.14159)
  sdintermediate=(.2+.3*(muintermediate+1))
  y=muintermediate+sdintermediate*z
  return muintermediate,sdintermediate,y
experiment=3
if experiment==0:
  n=100
  torch.manual_seed(42)
  x=torch.linspace(-.5,.5,n).reshape(-1,1)
  y=5*x+.5*torch.randn(n,1)
  figure=plt.figure(figsize=[8,6])
  plt.plot(x,y,'.',ms=15)
  plt.xlabel('$x$')
  plt.ylabel('$y$')
  plt.show()
elif experiment==1:
  n=100
  torch.manual_seed(42)
  x=torch.linspace(-.5,.5,n).reshape(-1,1)
  y=-1+2*(x>0)+.5*torch.randn(n,1)
  figure=plt.figure(figsize=[8,6])
  plt.plot(x,y,'.',ms=15)
  plt.xlabel('$x$')
  plt.ylabel('$y$')
  plt.show()
elif experiment==2:
  n=100
  torch.manual_seed(42)
  x=torch.linspace(-.5,.5,n).reshape(-1,1)
  y=torch.cos(x*2*2*3.14159)+.5*torch.randn(n,1)
  figure=plt.figure(figsize=[8,6])
  plt.plot(x,y,'.',ms=15)
  plt.xlabel('$x$')
  plt.ylabel('$y$')
  plt.show()
elif experiment==3:
  n=100
  torch.manual_seed(42)
  x=torch.linspace(-.5,.5,n).reshape(-1,1)
  np.random.seed(42)
  q=np.random.rand(n,1)
  mu3,sd3,y=gen3(x,q)
  figure=plt.figure(figsize=[8,6])
  plt.plot(x,y,'.',ms=15) 
  plt.xlabel('$x$')
  plt.ylabel('$y$')
  plt.show()
### save result for other comparisons
xymusd=np.c_[x.detach().numpy(),y.detach().numpy(),mu3.detach().numpy(),sd3.detach().numpy()]
np.save(r'/content/drive/My Drive/overfit_discover/xymusd',xymusd)
class my_dataset(Dataset):
    def __init__(self,data,label):
        self.data=x
        self.label=y          
    def __getitem__(self, index):
        return self.data[index],self.label[index]
    def __len__(self):
        return len(self.data)<jupyter_output><empty_output><jupyter_text>point estimate model<jupyter_code>class mlp_regressor(nn.Module):
    def __init__(self):
        super().__init__()
        self.k1=1000
        self.k2=1000
        self.fc1 = nn.Linear(1, self.k1)
        self.bn1 = nn.BatchNorm1d(self.k1)
        self.fc2 = nn.Linear(self.k1, self.k2)
        self.bn2 = nn.BatchNorm1d(self.k2)
        self.fc3 = nn.Linear(self.k2, 1)

    def forward(self, x):
        h1 = F.relu(self.fc1(x))
        h1 = self.bn1(h1)
        h2 = F.relu(self.fc2(h1))
        h2 = self.bn2(h2)
        h3 = self.fc3(h2)
        return h3<jupyter_output><empty_output><jupyter_text>mlp linear regression overfit<jupyter_code>dataloader = DataLoader(dataset=my_dataset(x,y),
                            batch_size=n,
                            shuffle=True)

epochs = 40000
model = mlp_regressor().to(device)
criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(),lr=1e-5)
mlp_loss = []

for epoch in range(epochs):
    for instances, labels in dataloader:
        instances, labels = instances.to(device), labels.to(device)
        optimizer.zero_grad()
        
        output = model(instances)
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()
        
    mlp_loss.append(loss.item())
plt.plot(mlp_loss)
y_mlp=model(x.to(device)).to(cpu)
plt.plot(y_mlp.detach())
plt.plot(y)<jupyter_output><empty_output><jupyter_text>quantile regression overfit: a percentile at a time<jupyter_code>def tilted_loss(q=.5):
    def qloss(output,labels):
        higher=output>labels
        higher=higher.float()
        intermediate=(q*higher+(1-q)*(1-higher))*torch.abs(output-labels)
        loss=torch.mean(intermediate)
        return loss
    return qloss
dataloader = DataLoader(dataset=my_dataset(x,y),
                            batch_size=n,
                            shuffle=True)

epochs = 50000
model_tl_25 = mlp_regressor().to(device)
model_tl_50 = mlp_regressor().to(device)
model_tl_75 = mlp_regressor().to(device)

criterion25 = tilted_loss(q=.25)
criterion50 = tilted_loss(q=.5)
criterion75 = tilted_loss(q=.75)


optimizer25 = optim.Adam(model_tl_25.parameters(), lr=1e-5)
optimizer50 = optim.Adam(model_tl_50.parameters(), lr=1e-5)
optimizer75 = optim.Adam(model_tl_75.parameters(), lr=1e-5)

ql25_loss = []
ql50_loss = []
ql75_loss = []

for epoch in range(epochs):
    for instances, labels in dataloader:
        instances, labels = instances.to(device), labels.to(device)

        optimizer25.zero_grad() 
        output25 = model_tl_25(instances)
        loss25 = criterion25(output25, labels)
        loss25.backward()
        optimizer25.step()


        optimizer50.zero_grad() 
        output50 = model_tl_50(instances)
        loss50 = criterion50(output50, labels)
        loss50.backward()
        optimizer50.step()


        optimizer75.zero_grad() 
        output75 = model_tl_75(instances)
        loss75 = criterion75(output75, labels)
        loss75.backward()
        optimizer75.step()

    ql25_loss.append(loss25.item())
    ql50_loss.append(loss50.item())
    ql75_loss.append(loss75.item())



plt.plot(ql25_loss)
plt.plot(ql50_loss)
plt.plot(ql75_loss)
y_25=model_tl_25(x.to(device)).to(cpu)
y_50=model_tl_50(x.to(device)).to(cpu)
y_75=model_tl_75(x.to(device)).to(cpu)
### save result for point estimate
mse_qr25_50_75=np.c_[y_mlp.detach().numpy(),y_25.detach().numpy(),y_50.detach().numpy(),y_75.detach().numpy()]
np.save(r'/content/drive/My Drive/overfit_discover/mse_qr25_50_75',mse_qr25_50_75)
from scipy import stats
dt=np.load(r'/content/drive/My Drive/overfit_discover/xymusd.npy')
x=dt[:,0]
y=dt[:,1]
med=dt[:,2].ravel()
std=dt[:,3].ravel()
up=med+stats.norm.ppf(0.75)*std
low=med+stats.norm.ppf(0.25)*std
mse_qr25_50_75=np.load(r'/content/drive/My Drive/overfit_discover/mse_qr25_50_75.npy')

fig = plt.gcf()
fig.set_size_inches(6,4.5)


plt.rcParams["font.weight"] = "bold"
plt.rcParams["axes.labelweight"] = "bold"
plt.rcParams["axes.labelpad"] = 0


plt.rcParams['axes.linewidth'] = 0.5



plt.rcParams['axes.labelsize'] = 10

plt.rcParams['axes.titlesize'] = 10

plt.rcParams['xtick.labelsize'] = 8

plt.rcParams['ytick.labelsize'] = 8
plt.rcParams['legend.fontsize'] = 9.5
plt.plot(x,y,'*',ms=8,color='red')
plt.plot(x,mse_qr25_50_75[:,0],'-.',lw=2,color='green')
plt.plot(x,mse_qr25_50_75[:,2],'-',lw=2,alpha=0.9,color='orange')
plt.plot(x,mse_qr25_50_75[:,1],'--',lw=2,alpha=0.8,color='brown')
plt.plot(x,mse_qr25_50_75[:,3],'-',lw=2,alpha=0.4,color='blue')
plt.legend(['Observations','MSE','QR_0.5','QR_0.25','QR_0.75'],frameon=False)
plt.xlabel('$x$',fontsize=14)
plt.gca().xaxis.set_label_coords(1.0, -0.02)
plt.ylabel('$y$',fontsize=14)
plt.gca().yaxis.set_label_coords(-0.03, 0.5)

plt.ylim([-1.7,3.1])
plt.xlim([-0.55,0.55])
fig.savefig(r'/content/drive/My Drive/overfit_discover/pt_mseqr.pdf',transparent=True,bbox_inches = 'tight',
    pad_inches = 0.2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18050221565731167, 'fraction_numerical': 0.059527326440177254, 'mean_word_length': 4.02299703264095, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '645279', 'license': 'MIT', 'n_tokens_mistral': 3073, 'n_tokens_neox': 2832, 'n_words': 320, 'path': 'property_of_learning_f/overfit_discover/mlp_qr.ipynb', 'repo_name': 'thuizhou/Collaborating-Networks', 'size': '6770'}