<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd
from tqdm import trange
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data as data 

train= pd.read_csv('/content/train.csv')
test= pd.read_csv('/content/test.csv')
test_ids= test['id'].to_numpy()
from apex import amp
def preprocess(df):
    #some new
    dfr= pd.get_dummies(df['R'], prefix= "R_")
    df= pd.concat([df, dfr], axis= 1)
    dfc= pd.get_dummies(df['C'], prefix= "C_")
    df= pd.concat([df, dfc], axis= 1)
    df= df.drop(['R', 'C'], axis= 1)

    df['u_in_cumsum']= df['u_in'].groupby(df['breath_id']).cumsum()
    df['time_step_cumsum']= df['time_step'].groupby(df['breath_id']).cumsum()
    
    df['u_in_min']= df['u_in'].groupby(df['breath_id']).transform('min')
    df['u_in_max']= df['u_in'].groupby(df['breath_id']).transform('max')
    df['u_in_mean']= df['u_in'].groupby(df['breath_id']).transform('mean')
   
    df['u_in_lag2']= df['u_in'].groupby(df['breath_id']).shift(2)
    df['u_in_lag1']= df['u_in'].groupby(df['breath_id']).shift(1)
    df['u_in_lag-1']= df['u_in'].groupby(df['breath_id']).shift(-1)
    df['u_in_lag-2']= df['u_in'].groupby(df['breath_id']).shift(-2)
    df= df.fillna(0)

    df['u_in_diff1']= df['u_in']- df['u_in_lag1']
    df['u_in_diff2']= df['u_in']- df['u_in_lag2']
    df['u_in_diff3']= df['u_in_max']- df['u_in']
    df['u_in_diff4']= df['u_in_mean']- df['u_in']

    df1= df[df['u_out'] == 0]
    df['mean_inspiratory_uin']= df1['u_in'].groupby(df['breath_id']).transform('mean')

    df2= df[df['u_out'] == 1]
    df['mean_expiratory_uin']= df2['u_in'].groupby(df['breath_id']).transform('mean')
    
    df['u_in_diff5']= df['mean_inspiratory_uin']- df['u_in']
    df['u_in_diff6']= df['mean_expiratory_uin']- df['u_in']
    
    df= df.fillna(0)
    
    df['delta_t']= df.groupby('breath_id')['time_step'].diff().fillna(0)
    df['delta_uin']= df.groupby('breath_id')['u_in'].diff().fillna(0)
    
    df['area']= df['u_in']*df['delta_t']
    df['area']= df.groupby('breath_id')['area'].cumsum()
    df['slope']= (df['delta_uin']/df['delta_t']).fillna(0)



    return df
groups= train.breath_id.values.reshape(-1, 80)[:, 0]
groups.shape

train= preprocess(train)
targets= train['pressure'].to_numpy().reshape(-1, 80)
train.drop(['id','pressure', "breath_id"], axis= 1, inplace= True)

test= preprocess(test)
test.drop(['id', "breath_id"], axis= 1, inplace= True)
y_test= np.zeros(test.shape[0]).reshape(-1, 80)

from sklearn.preprocessing import RobustScaler
RS = RobustScaler()
train = RS.fit_transform(train)
test  = RS.transform(test)

num_features= train.shape[-1]
train= train.reshape(-1, 80, num_features)
test= test.reshape(-1, 80, num_features)
class CustomDataset:
    def __init__(self, data, target):
        self.data= data
        self.target= target
        
    def __len__(self):
        return len(self.data)
    
    def __getitem__(self, idx):
        current_sample= self.data[idx, :, :]
        current_target= self.target[idx, :]
        
        return torch.tensor(current_sample, dtype= torch.float), torch.tensor(current_target, dtype= torch.float)
     
class RNNModel(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(RNNModel, self).__init__()
        
        hidden_dim= [400, 300, 200, 100]
        self.bilstm1= nn.LSTM(input_dim, hidden_dim[0], batch_first= True, bidirectional= True)
        self.norm1= nn.LayerNorm(hidden_dim[0]*2)
        
        self.bilstm2= nn.LSTM(hidden_dim[0]*2, hidden_dim[1], batch_first= True, bidirectional= True)
        self.norm2= nn.LayerNorm(hidden_dim[1]*2)
        
        self.bilstm3= nn.LSTM(hidden_dim[1]*2, hidden_dim[2], batch_first= True, bidirectional= True)
        self.norm3= nn.LayerNorm(hidden_dim[2]*2)
        
        self.bilstm4= nn.LSTM(hidden_dim[2]*2, hidden_dim[3], batch_first= True, bidirectional= True)
        self.norm4= nn.LayerNorm(hidden_dim[3]*2)
        
        self.d= nn.Dropout(p= 0.002)
        
        self.fc1= nn.Linear(hidden_dim[3]*2, 100)
        self.fc2= nn.Linear(100, output_dim)
#         self.fc3= nn.Linear(32, output_dim)

        
    def forward(self, X):
        pred, _= self.bilstm1(X)
        pred= self.norm1(pred)
        
        pred, _= self.bilstm2(pred)
        pred= self.norm2(pred)
        
        pred, _= self.bilstm3(pred)
        pred= self.norm3(pred)
        
        pred, _= self.bilstm4(pred)
        pred= self.norm4(pred)
        
        pred= self.d(pred)
        
        pred= self.fc1(pred)
        pred= F.selu(pred)
        
        pred= self.fc2(pred)
        
#         pred= F.selu(pred)
#         pred= self.fc3(pred)

        pred= pred.squeeze(dim= 2)
        
        return pred
def initialize_parameters(m):
    if isinstance(m, nn.LSTM):
        nn.init.orthogonal_(m.weight_ih_l0.data, gain= nn.init.calculate_gain('tanh'))
        nn.init.orthogonal_(m.weight_hh_l0.data, gain= nn.init.calculate_gain('tanh'))
        nn.init.orthogonal_(m.weight_ih_l0_reverse.data, gain= nn.init.calculate_gain('tanh'))
        nn.init.orthogonal_(m.weight_hh_l0_reverse.data, gain= nn.init.calculate_gain('tanh'))
        
        nn.init.constant_(m.bias_ih_l0.data, 0)
        nn.init.constant_(m.bias_hh_l0.data, 0)
        nn.init.constant_(m.bias_ih_l0_reverse.data, 0)
        nn.init.constant_(m.bias_hh_l0_reverse.data, 0)
        
    if isinstance(m, nn.Linear):
        nn.init.xavier_normal_(m.weight.data)
        nn.init.constant_(m.bias.data, 0)
device= "cuda" if torch.cuda.is_available() else 'cpu'
INPUT_DIM= num_features
OUTPUT_DIM= 1
BATCH_SIZE= 1024

model= RNNModel(input_dim= INPUT_DIM, output_dim= OUTPUT_DIM).to(device)
model.apply(initialize_parameters)

criterion= nn.L1Loss()
criterion.to(device)

optimizer= optim.Adam(model.parameters(), lr= 0.001)

model, optimizer = amp.initialize(model, optimizer, opt_level="O1") # 这里是“欧一”，不是“零一”

scheduler= optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor= 0.5, patience= 10, verbose= True)

def train_model(dataloader, model, criterion, optimizer):
    size= len(dataloader.dataset)
    model.train()
    batches= len(dataloader)
    train_loss= 0
    
    for batch_idx, (X, y) in enumerate(dataloader):
        X, y= X.to(device), y.to(device)

        scores= model(X)
        loss= criterion(scores, y)

        optimizer.zero_grad()
        with amp.scale_loss(loss, optimizer) as scaled_loss:
          scaled_loss.backward()
        optimizer.step()
        
        loss= loss.item()
        train_loss += loss
        
    train_loss_avg= train_loss/batches
    print(f"avg. train loss: {train_loss_avg}")
    
    return train_loss_avg
def val_model(dataloader, model, criterion):
    
    size= len(dataloader.dataset)
    batches= len(dataloader)
    model.eval()
    test_loss= 0

    with torch.no_grad():
        for X, y in (dataloader):
            X, y= X.to(device), y.to(device)
      
            scores= model(X)
            test_loss += criterion(scores, y)

    test_loss /= batches

    print(f"avg test loss : {test_loss}")
    
    return test_loss
def predict_model(dataloader, model):
    model.eval()
    y_pred= np.array([])
    
    with torch.no_grad():
        for X , y in dataloader:
            X, y= X.to(device), y.to(device)
            
            preds= model(X)
            preds= preds.flatten().cpu().numpy()
            
            y_pred= np.concatenate((y_pred, preds))
            
    return y_pred
from sklearn.model_selection import GroupKFold

kfold= GroupKFold(n_splits= 5)
EPOCHS= 200
cv_scores= []
predictions= np.zeros(test_ids.shape[0])


for fold, (train_idx, val_idx) in enumerate(kfold.split(train, targets, groups= groups)):
    X_train, X_val= train[train_idx], train[val_idx]
    y_train, y_val= targets[train_idx], targets[val_idx]
    
    train_dataset= CustomDataset(data= X_train, target= y_train)
    val_dataset= CustomDataset(data= X_val, target= y_val)

    train_loader= data.DataLoader(train_dataset, batch_size= BATCH_SIZE)
    val_loader= data.DataLoader(val_dataset, batch_size= BATCH_SIZE)
    
    best_valid_loss= float('inf')
    
    avg_train_losses= []
    avg_val_losses= []
    
    for t in trange(EPOCHS):
        print(f"Epoch: {t+1}")
        train_loss= train_model(train_loader, model, criterion, optimizer)
        val_loss= val_model(val_loader, model, criterion)
        
        avg_train_losses.append(train_loss)
        avg_val_losses.append(val_loss)
        
        if (val_loss< best_valid_loss):
            best_valid_loss= val_loss
            ofilename = 'ventilator%d.pth' % fold
            torch.save(model.state_dict(),  ofilename)
        
        scheduler.step(val_loss)
    
    cv_scores.append(best_valid_loss)
    
    test_dataset= CustomDataset(data= test, target= y_test)
    test_loader= data.DataLoader(test_dataset, batch_size= BATCH_SIZE)
                       
    model.load_state_dict(torch.load('ventilator%d.pth' % fold, map_location=device))
    predictions += (predict_model(test_loader, model))
    
    break
sub= pd.DataFrame({'id': test_ids, 'pressure': predictions})
sub.to_csv('submission.csv',index = False)
import matplotlib.pyplot as plt
plt.plot(avg_train_losses)
plt.plot(avg_val_losses)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15242988653393277, 'fraction_numerical': 0.016377649325626204, 'mean_word_length': 3.2009892086330933, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '63549', 'license': 'MIT', 'n_tokens_mistral': 3672, 'n_tokens_neox': 3537, 'n_words': 588, 'path': 'Apex_Improved.ipynb', 'repo_name': 'Vaincookie/FundamentalAnalysis', 'size': '9342'}