<jupyter_start><jupyter_text>Path<jupyter_code>path = Path('/kaggle/osic_pulmonary')
assert path.exists()<jupyter_output><empty_output><jupyter_text>Read Data<jupyter_code>train_df, test_df, submission_df = common.read_data(path)<jupyter_output><empty_output><jupyter_text>Feature generation<jupyter_code>len(train_df)
submission_df['Patient'] = submission_df['Patient_Week'].apply(lambda x:x.split('_')[0])
submission_df['Weeks'] = submission_df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission_df = submission_df[['Patient','Weeks','Confidence','Patient_Week']]
submission_df = submission_df.merge(test_df.drop('Weeks', axis=1).copy(), on=['Patient'])
test_df[test_df['Patient'] == 'ID00419637202311204720264']
train_df[train_df['Patient'] == 'ID00419637202311204720264']
submission_df[submission_df['Patient'] == 'ID00419637202311204720264'].head(10)<jupyter_output><empty_output><jupyter_text>Adding missing values<jupyter_code>train_df['WHERE'] = 'train'
test_df['WHERE'] = 'val'
submission_df['WHERE'] = 'test'
data = train_df.append([test_df, submission_df])
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
base = base[base.nb==1]
base[base['Patient'] == 'ID00419637202311204720264']
data = data.merge(base, on='Patient', how='left')
data['base_week'] = data['Weeks'] - data['min_week']
data['base_week'] = data['base_week']
del base
data[data['Patient'] == 'ID00419637202311204720264']
cat_feats = ['SmokingStatus', 'Sex', 'Age']
label_encoders = {}
for cat_col in cat_feats:
    label_encoders[cat_col] = LabelEncoder()
    data[cat_col] = label_encoders[cat_col].fit_transform(data[cat_col])
def normalize(df:pd.DataFrame, cont_names):
    "Compute the means and stds of `self.cont_names` columns to normalize them."
    means, stds = {},{}
    for n in cont_names:
        means[n], stds[n] = df[n].mean(), df[n].std()
        df[n] = (df[n]-means[n]) / (1e-7 + stds[n])
data['BASE'] = data['min_FVC']
data['week'] = data['base_week']
data['percent'] = data['Percent']
normalize(data, ['BASE', 'week', 'percent'])
data
# data['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )
# data['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )
# data['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )
# data['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )
FE = ['SmokingStatus', 'Sex', 'Age','percent','week','BASE']
train_df = data.loc[data.WHERE=='train']
test_df = data.loc[data.WHERE=='val']
submission_df = data.loc[data.WHERE=='test']
del data
train_df['Sex'].unique()
train_df = train_df[FE + ['FVC']]
train_df<jupyter_output><empty_output><jupyter_text>Seed<jupyter_code>def seed_everything(seed=2020):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(42)
    torch.cuda.manual_seed(42)
    
seed_everything(42)<jupyter_output><empty_output><jupyter_text>Create Dataset<jupyter_code>class TabularDataset(Dataset):
    def __init__(self, data, cat_cols=None, output_col=None):
        self.n = len(data)
        if output_col:
            self.y = data[output_col].astype(np.float32).values.reshape(-1, 1)
        else:
            self.y = np.zeros((self.n, 1))
        self.cat_cols = cat_cols if cat_cols else []
        
        self.cont_cols = [
            col for col in data.columns if col not in self.cat_cols + [output_col]
        ]
        
        self.cont_X = data[self.cont_cols].astype(np.float32).values
        self.cat_X = data[cat_cols].astype(np.int64).values
        
    def __len__(self):
        """
        Denotes the total number of samples.
        """
        return self.n
    
    def __getitem__(self, idx):
        """
        Generates one sample of data.
        """
        return [self.y[idx], self.cont_X[idx], self.cat_X[idx]]
    
    def __repr__(self):
        return f'cat_cols: {self.cat_cols} cont_cols: {self.cont_cols} cont_X: {self.cont_X.shape} cat_X: {self.cat_X.shape} y: {self.y.shape}'
    
    def cont_cols_len(self):
        return len(self.cont_cols)
# Test the dataset
sample_ds = TabularDataset(train_df, cat_cols=['SmokingStatus', 'Sex',  'Age'], output_col='FVC')
def create_dl(df, batch_size=128, num_workers=10):
    ds = TabularDataset(df, cat_cols=cat_feats, output_col='FVC')
    return DataLoader(ds, batch_size, shuffle=True, num_workers=num_workers)
sample_dl = create_dl(train_df)
y_sample, x_cont_sample, x_cat_sample = next(iter(sample_dl))
x_cont_sample.shape, x_cat_sample.shape, y_sample.shape<jupyter_output><empty_output><jupyter_text>Prepare neural network<jupyter_code>cat_dims = [int(train_df[col].nunique()) for col in cat_feats]
emb_dims = [(x, min(50, (x + 1) // 2)) for x in cat_dims]
emb_dims
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = "cpu"

def move_to_dev(*args):
    return [x.to(device) for x in args]
C1, C2 = torch.tensor(70, dtype=torch.float32), torch.tensor(1000, dtype=torch.float32)
C1, C2 = move_to_dev(C1, C2)
Q = torch.tensor([0.2, 0.50, 0.8]).float().to(device)

def score(y_true, y_pred):
    sigma = y_pred[:, 2] - y_pred[:, 0]
    fvc_pred = y_pred[:, 1]
    
    #sigma_clip = sigma + C1
    sigma_clip = torch.max(sigma, C1)
    delta = torch.abs(y_true[:, 0] - fvc_pred)
    delta = torch.min(delta, C2)
    sq2 = torch.sqrt(torch.tensor(2.))
    metric = (delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)
    return torch.mean(metric)

def qloss(y_true, y_pred):
    # Pinball loss for multiple quantiles
    e = y_true - y_pred
    v = torch.max(Q*e, (Q-1)*e)
    return torch.mean(v)

def mloss(_lambda):
    def loss(y_true, y_pred):
        y_true = y_true.unsqueeze(1)
        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)
    return loss
class OsicModel(nn.Module):
    def __init__(self, emb_dims, no_of_cont, lin_layer_sizes, output_size=3, emb_dropout=0.0, lin_layer_dropouts=[0.001,0.001], y_range=[0, 5000]):
        super().__init__()
        
        # Embedding layers
        self.emb_layers = nn.ModuleList([nn.Embedding(x, y) for x, y in emb_dims])
        self.no_of_embs = sum([y for _, y in emb_dims])
        self.no_of_cont = no_of_cont
        
        # Linear Layers
        first_lin_layer = nn.Linear(self.no_of_embs + self.no_of_cont, lin_layer_sizes[0])
        self.lin_layers = nn.ModuleList(
            [first_lin_layer] + [nn.Linear(lin_layer_sizes[i], lin_layer_sizes[i + 1]) for i in range(len(lin_layer_sizes) - 1)]
        )
        
        for lin_layer in self.lin_layers:
            nn.init.kaiming_normal_(lin_layer.weight.data)
            
        # Output Layers
        self.p1 = nn.Linear(lin_layer_sizes[-1], 3)
        self.p2 = nn.Linear(lin_layer_sizes[-1], 3)
        for output_layer in [self.p1, self.p2]:
            nn.init.kaiming_normal_(output_layer.weight.data)
            
        # Batch Norm Layers
        self.first_bn_layer = nn.BatchNorm1d(self.no_of_cont, momentum=0.1)
        self.bn_layers = nn.ModuleList([nn.BatchNorm1d(size) for size in lin_layer_sizes])
        
        # Dropout Layers
        self.emb_dropout_layer = nn.Dropout(emb_dropout) if emb_dropout > 0.0 else None
        self.droput_layers = nn.ModuleList(
            [nn.Dropout(size) for size in lin_layer_dropouts]
        )
        
        self.y_range = y_range
        
    def forward(self, cont_data, cat_data):
        
        x = [emb_layer(cat_data[:, i]) for i, emb_layer in enumerate(self.emb_layers)]
        x = torch.cat(x, 1)
        if self.emb_dropout_layer is not None:
            x = self.emb_dropout_layer(x)
        
        normalized_cont_data = self.first_bn_layer(cont_data)
        x = torch.cat([x, normalized_cont_data], 1)
        
        for i, (lin_layer, dropout_layer, bn_layer) in enumerate(zip(self.lin_layers, self.droput_layers, self.bn_layers)):
            x = F.relu(lin_layer(x))
            x = bn_layer(x)
#             x = dropout_layer(x)

        p1 = self.p1(x)
        p2 = F.relu(self.p2(x))
        preds = p1 + torch.cumsum(p2, axis=1)
        if self.y_range is not None:
            preds = (self.y_range[1]-self.y_range[0]) * torch.sigmoid(preds) + self.y_range[0]
        return preds
def create_model(nh1=100, nh2=100):
    model = OsicModel(emb_dims, x_cont_sample.shape[1], [nh1, nh2])
    model = model.to(device)
    return model
sample_model = create_model(nh1=100, nh2=100)
sample_model
criterion=mloss(0.8)
y_sample, x_cont_sample, x_cat_sample = move_to_dev(y_sample, x_cont_sample, x_cat_sample)
y_sample.shape, x_cont_sample.shape, x_cat_sample.shape
# Test model

output = sample_model(x_cont_sample, x_cat_sample)
criterion(y_sample, output)<jupyter_output><empty_output><jupyter_text>Training functions<jupyter_code>def get_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']
def eval_loop(valid_dl, model):
    model.eval()
    total_eval_loss = 0
    total_eval_score = 0
    for i, (y, x_cont, x_cat) in enumerate(valid_dl):
        y, x_cont, x_cat = move_to_dev(y, x_cont, x_cat)
        output = model(x_cont, x_cat)
        loss = criterion(y, output)
        total_eval_loss += loss.item()
        total_eval_score += score(y.unsqueeze(1), output)
        
    avg_val_loss = total_eval_loss / len(valid_dl)
    avg_val_score = total_eval_score / len(valid_dl) * -1
    return {
        'avg_val_loss': avg_val_loss,
        'avg_val_score': avg_val_score
    }
def train_loop(epochs, train_dl, valid_dl, model, lr = 1e-3):
    steps = len(train_dl) * epochs
    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
    scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, steps_per_epoch=len(train_dl), epochs=epochs)
    avg_train_losses = []
    avg_val_losses = []
    avg_val_scores = []
    lr = []
    for epoch in tqdm(range(epochs), total=epochs):
        model.train()
        total_train_loss = 0.0
        for i, (y, x_cont, x_cat) in enumerate(train_dl):
            y, x_cont, x_cat = move_to_dev(y, x_cont, x_cat)
            model.zero_grad()
            output = model(x_cont, x_cat)
            loss = criterion(y, output)
            total_train_loss += loss.item()
            
            # Backward Pass and Optimization
            loss.backward()
            optimizer.step()
            scheduler.step()
            lr.append(get_lr(optimizer))
        
        avg_train_loss = total_train_loss / len(train_dl)
        avg_train_losses.append(avg_train_loss)
        eval_res = eval_loop(valid_dl, model)
        avg_val_loss = eval_res['avg_val_loss']
        avg_val_score = eval_res['avg_val_score']
        avg_val_losses.append(avg_val_loss)
        avg_val_scores.append(avg_val_score.item())
#         print(f'{epoch}: avg_val_score: {avg_val_score}')
    return pd.DataFrame({'avg_train_losses': avg_train_losses, 'avg_val_losses': avg_val_losses, 'avg_val_scores': avg_val_scores}), pd.DataFrame({'lr': lr})
res_df, lr_df = train_loop(500, sample_dl, sample_dl, sample_model, lr=1e-3)
res_df[['avg_train_losses', 'avg_val_losses']].plot()
res_df[['avg_val_scores']].plot()
lr_df.plot()
res_df[['avg_val_scores']].max()<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>NFOLD = 5
kf = KFold(n_splits=NFOLD)
EPOCHS=500
def convert_to_tensor(df, dtype):
    return torch.tensor(df.values, dtype=dtype).to(device)
cont_cols = sample_ds.cont_cols
cat_values = convert_to_tensor(submission_df[cat_feats], torch.int64)
cont_values = convert_to_tensor(submission_df[cont_cols], torch.float32)
cat_values.shape, cont_values.shape, cat_values.dtype
pe = np.zeros((cat_values.shape[0], 3))
pred = np.zeros((train_df.shape[0], 3))
pred.shape
def predict(cont_values, cat_values, model):
    return model(cont_values, cat_values).detach().cpu().numpy()
%%time

res_dfs = []
for cnt, (tr_idx, val_idx) in tqdm(enumerate(kf.split(train_df)), total=NFOLD):
    X_y_train = train_df.loc[tr_idx]
    X_y_valid = train_df.loc[val_idx]
    print(f"FOLD {cnt}", X_y_train.shape, X_y_valid.shape)
    model = create_model()
    train_dl = create_dl(X_y_train)
    valid_dl = create_dl(X_y_valid)
    res_df, _ = train_loop(EPOCHS, train_dl, valid_dl, model)
    res_dfs.append(res_df)
    pe += predict(cont_values, cat_values, model)
    pred[val_idx] = predict(
        convert_to_tensor(X_y_valid[cont_cols], torch.float32), 
        convert_to_tensor(X_y_valid[cat_feats], torch.int64), 
        model
    )
pe = pe / NFOLD
from matplotlib.pyplot import figure

nrows = len(res_dfs) // 2 + 1
ncols = 2
fig, axes = plt.subplots(nrows, ncols, figsize=(20, 10))
figure(num=None, figsize=(10, 6), dpi=80, facecolor='w', edgecolor='k')
for r in range(nrows):
    for c in range(ncols):
        index = r * 2 + c
        if index < len(res_dfs):
            res_dfs[r * 2 + c].plot(ax=axes[r,c])
print("Mean validation score", np.mean([res_dfs[i]['avg_val_scores'][len(res_dfs[0]) - 1] for i in range(NFOLD)]))<jupyter_output>Mean validation score -8.15509090423584<jupyter_text>Prediction<jupyter_code>sigma_opt = mean_absolute_error(train_df['FVC'], pred[:, 1])
unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
sigma_opt, sigma_mean
submission_df['FVC1'] = pe[:,1]
submission_df['Confidence1'] = pe[:, 2] - pe[:, 0]
submission_df.head(15)
subm = submission_df[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull()].shape, subm.shape
subm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']
if sigma_mean<70:
    subm['Confidence'] = sigma_opt
else:
    subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']
subm.describe().T
def replace_with_existing(df):
    for i in range(len(df)):
        patient_week_filter = subm['Patient_Week']==df.Patient[i]+'_'+str(df.Weeks[i])
        subm.loc[patient_week_filter, 'FVC'] = df.FVC[i]
        subm.loc[patient_week_filter, 'Confidence'] = 0.1

train_df = pd.read_csv(path/'train.csv', dtype = common.TRAIN_TYPES)
test_df = pd.read_csv(path/'test.csv', dtype = common.TRAIN_TYPES)
replace_with_existing(train_df)
replace_with_existing(test_df)
subm[subm['Patient_Week'].str.find('ID00419637202311204720264') > -1].head(30)
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False)
pd.read_csv('submission.csv')
!cat submission.csv<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1532600633807565, 'fraction_numerical': 0.024543186568673724, 'mean_word_length': 4.004048582995951, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '299567', 'license': 'Apache-2.0', 'n_tokens_mistral': 5906, 'n_tokens_neox': 5655, 'n_words': 926, 'path': 'nbs_gil/osic_pulmonary/03_training_pytorch_embeddings.ipynb', 'repo_name': 'gilfernandes/fastai-exercises', 'size': '14831'}