<jupyter_start><jupyter_text>读取数据<jupyter_code>df = pd.read_pickle(os.path.join(data_path, 'processed_data_numerical.pkl'))
df['age'] = df['age'] - 1
df['gender'] = df['gender'] - 1
df.head(1)<jupyter_output><empty_output><jupyter_text>读取预训练好的Word Embedding<jupyter_code>os.listdir(embedding_path)
embedding = np.load(os.path.join(embedding_path, 'embedding_w2v_sg1_hs0_win10_size300.npz'))
creative = embedding['creative_w2v']
ad= embedding['ad_w2v']
advertiser = embedding['advertiser_w2v']
product = embedding['product_w2v']
industry = embedding['industry_w2v']
product_cate = embedding['product_cate_w2v']
del embedding
gc.collect()<jupyter_output><empty_output><jupyter_text>需要使用的embedding特征以及对应的序列编号<jupyter_code># 这里将需要使用到的特征列直接拼接成一个向量，后面直接split即可
data_seq = df[['creative_id', 'ad_id', 'advertiser_id', 'product_id', 'industry', 'click_times']].progress_apply(lambda s: np.hstack(s.values), axis=1).values

# embedding_list = [creative_embed, ad_embed, advertiser_embed, product_embed]
# embedding_list = [creative_glove, ad_glove, advertiser_glove, product_glove]
embedding_list = [creative, ad, advertiser, product, industry]<jupyter_output>19%|█▉        | 754606/4000000 [01:33<06:34, 8230.65it/s]<jupyter_text>建立PyTorch Dataset 和 Dataloader<jupyter_code>class CustomDataset(Dataset):
    def __init__(self, seqs, labels, input_num, shuffle=False):
        self.seqs = seqs
        self.labels = labels
        self.input_num = input_num
        self.shuffle = shuffle
    
    def __len__(self):
        return len(self.seqs)
    
    def __getitem__(self, idx):
        length = int(self.seqs[idx].shape[0]/self.input_num)
        seq_list = list(torch.LongTensor(self.seqs[idx]).split(length, dim=0))          
        label = torch.LongTensor(self.labels[idx])
        # 对数据进行随机shuffle
        if self.shuffle and torch.rand(1) < 0.5:
            random_pos = torch.randperm(length)
            for i in range(len(seq_list)):
                seq_list[i] = seq_list[i][random_pos]
        return seq_list + [length, label]

    
def pad_truncate(Batch):
    *seqs, lengths, labels = list(zip(*Batch))
    # 长度截取到99%的大小，可以缩短pad长度，大大节省显存
    trun_len = torch.topk(torch.tensor(lengths), max(int(0.01*len(lengths)), 1))[0][-1]
    # 保险起见，再设置一个最大长度
    max_len = min(trun_len, 150)
    seq_list = list(pad_sequence(seq, batch_first=True)[:, :max_len] for seq in seqs)
    return seq_list, torch.tensor(lengths).clamp_max(max_len), torch.stack(labels)
input_num = 6
BATCH_SIZE_TRAIN = 1024
BATCH_SIZE_VAL = 2048
BATCH_SIZE_TEST = 2048
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
data_folds = []
valid_indexs = [] # 用于后面保存五折的验证集结果时，按照1到900000对应顺序
for idx, (train_index, valid_index) in enumerate(kf.split(X=df.iloc[:3000000], y=df.iloc[:3000000]['age'])):
    valid_indexs.append(valid_index)
    X_train, X_val, X_test = data_seq[train_index], data_seq[valid_index], data_seq[3000000:]
    y_train, y_val =  np.array(df.iloc[train_index, -2:]), np.array(df.iloc[valid_index, -2:])
    y_test = np.random.rand(X_test.shape[0], 2)
    
    train_dataset = CustomDataset(X_train, y_train, input_num, shuffle=True)
    val_dataset = CustomDataset(X_val, y_val, input_num, shuffle=False)
    test_dataset = CustomDataset(X_test, y_test, input_num, shuffle=False)

    train_dataloader = DataLoader(train_dataset, 
                                  batch_size=BATCH_SIZE_TRAIN, 
                                  shuffle=True, 
                                  collate_fn=pad_truncate, 
                                  num_workers=0, 
                                  worker_init_fn=worker_init_fn)
    
    valid_dataloader = DataLoader(val_dataset, 
                                  batch_size=BATCH_SIZE_VAL, 
                                  sampler=SequentialSampler(val_dataset), 
                                  shuffle=False, 
                                  collate_fn=pad_truncate, 
                                  num_workers=0, 
                                  worker_init_fn=worker_init_fn)
    
    test_dataloader = DataLoader(test_dataset, 
                                 batch_size=BATCH_SIZE_TEST, 
                                 sampler=SequentialSampler(test_dataset), 
                                 shuffle=False, 
                                 collate_fn=pad_truncate, 
                                 num_workers=0, 
                                 worker_init_fn=worker_init_fn)
    data_folds.append((train_dataloader, valid_dataloader, test_dataloader))
del data_seq, creative, ad, advertiser, product, industry, product_cate
gc.collect()<jupyter_output><empty_output><jupyter_text>搭建模型<jupyter_code>class BiLSTM(nn.Module):
    def __init__(self, embedding_list, embedding_freeze, lstm_size, fc1, fc2, num_layers=1, rnn_dropout=0.2, embedding_dropout=0.2, fc_dropout=0.2):
        super().__init__()
        self.embedding_layers = nn.ModuleList([nn.Embedding.from_pretrained(torch.HalfTensor(embedding).cuda(), freeze=freeze) for embedding, freeze in zip(embedding_list, embedding_freeze)])
        self.input_dim = int(np.sum([embedding.shape[1] for embedding in embedding_list]))
        self.lstm = nn.LSTM(input_size = self.input_dim, 
                                      hidden_size = lstm_size, 
                                      num_layers = num_layers,
                                      bidirectional = True, 
                                      batch_first = True, 
                                      dropout = rnn_dropout) 
                                                  
        
        
        self.fc1 = nn.Linear(2*lstm_size, fc1)
        self.fc2 = nn.Linear(fc1, fc2)
        self.fc3 = nn.Linear(fc2, 12)
        
        self.rnn_dropout = nn.Dropout(rnn_dropout)
        self.embedding_dropout = nn.Dropout(embedding_dropout)
        self.fc_dropout = nn.Dropout(fc_dropout)
    
    def forward(self, seq_list, lengths):
        batch_size, total_length= seq_list[0].size()
        lstm_outputs = []
        click_time = seq_list[-1]
        embeddings = []
        for idx, seq in enumerate(seq_list[:-1]):
            embedding = self.embedding_layers[idx](seq).to(torch.float32)
            embedding = self.embedding_dropout(embedding)
            embeddings.append(embedding)
        packed = pack_padded_sequence(torch.cat(embeddings, dim=-1), lengths, batch_first=True, enforce_sorted=False)
        packed_output, (h_n, c_n) = self.lstm(packed)
        lstm_output, _ = pad_packed_sequence(packed_output, batch_first=True, total_length=total_length, padding_value=-float('inf'))
        lstm_output = self.rnn_dropout(lstm_output)
        # lstm_output shape: (batchsize, total_length, 2*lstm_size)
        max_output = F.max_pool2d(lstm_output, (total_length, 1), stride=(1, 1)).squeeze()
        # output shape: (batchsize, 2*lstm_size)
        fc_out = F.relu(self.fc1(max_output))
        fc_out = self.fc_dropout(fc_out)
        fc_out = F.relu(self.fc2(fc_out))
        pred = self.fc3(fc_out)
        age_pred = pred[:, :10]
        gender_pred = pred[:, -2:]
        return age_pred, gender_pred<jupyter_output><empty_output><jupyter_text>训练模型<jupyter_code>def validate(model, val_dataloader, criterion, history, n_iters):
    model.eval()
    global best_acc, best_model, validate_history
    costs = []
    age_accs = []
    gender_accs = []
    with torch.no_grad():
        for idx, batch in enumerate(val_dataloader):
            seq_list, lengths, labels = batch
            seq_list_device = [seq.cuda() for seq in seq_list]
            lengths_device = lengths.cuda()
            labels = labels.cuda()
            age_output, gender_output = model(seq_list_device, lengths_device)    
            loss = criterion(age_output, gender_output, labels)
            costs.append(loss.item())
            _, age_preds = torch.max(age_output, 1)
            _, gender_preds = torch.max(gender_output, 1)
            age_accs.append((age_preds == labels[:, 0]).float().mean().item())
            gender_accs.append((gender_preds == labels[:, 1]).float().mean().item())
            torch.cuda.empty_cache()
    mean_accs = np.mean(age_accs) + np.mean(gender_accs)
    mean_costs = np.mean(costs)
    writer.add_scalar('gender/validate_accuracy', np.mean(gender_accs), n_iters)
    writer.add_scalar('gender/validate_loss', mean_costs, n_iters)
    writer.add_scalar('age/validate_accuracy',np.mean(age_accs), n_iters)
    writer.add_scalar('age/validate_loss', mean_costs, n_iters)
    if mean_accs > history['best_model'][0][0]:  
        save_dict = copy.deepcopy(model.state_dict())
        embedding_keys = []
        for key in save_dict.keys():
            if key.startswith('embedding'):
                embedding_keys.append(key)
        for key in embedding_keys:
            save_dict.pop(key)
        heapq.heapify(history['best_model'])
        checkpoint_pth = history['best_model'][0][1]
        heapq.heappushpop(history['best_model'], (mean_accs, checkpoint_pth))
        torch.save(save_dict, checkpoint_pth)
        del save_dict
        gc.collect()
        torch.cuda.empty_cache()
    return mean_costs, mean_accs


def train(model, train_dataloader, val_dataloader, criterion, optimizer, epoch, history, validate_points, scheduler, step=True):
    model.train()
    costs = []
    age_accs = []
    gender_accs = []
    val_loss, val_acc = 0, 0
    with tqdm(total=len(train_dataloader.dataset), desc='Epoch{}'.format(epoch)) as pbar:
        for idx, batch in enumerate(train_dataloader):
            seq_list, lengths, labels = batch
            seq_list_device = [seq.cuda() for seq in seq_list]
            lengths_device = lengths.cuda()
            labels = labels.cuda()
            age_output, gender_output = model(seq_list_device, lengths_device)    
            loss = criterion(age_output, gender_output, labels)
            
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if step:
                scheduler.step()
            with torch.no_grad():
                costs.append(loss.item())
                _, age_preds = torch.max(age_output, 1)
                _, gender_preds = torch.max(gender_output, 1)
                age_accs.append((age_preds == labels[:, 0]).float().mean().item())
                gender_accs.append((gender_preds == labels[:, 1]).float().mean().item())
                pbar.update(labels.size(0))
            n_iters = idx + len(train_dataloader)*(epoch-1)
            if idx in validate_points:
                val_loss, val_acc = validate(model, val_dataloader, criterion, history, n_iters)
                model.train()
            
            writer.add_scalar('gender/train_accuracy', gender_accs[-1], n_iters)
            writer.add_scalar('gender/train_loss', costs[-1], n_iters)
            writer.add_scalar('age/train_accuracy', age_accs[-1], n_iters)
            writer.add_scalar('age/train_loss', costs[-1], n_iters)
            writer.add_scalar('age/learning_rate', scheduler.get_lr()[0], n_iters)
            pbar.set_postfix_str('loss:{:.4f}, acc:{:.4f}, val-loss:{:.4f}, val-acc:{:.4f}'.format(np.mean(costs[-10:]), np.mean(age_accs[-10:])+np.mean(gender_accs[-10:]), val_loss, val_acc))
            torch.cuda.empty_cache()

    
def test(oof_train_test, model, test_dataloader, val_dataloader, valid_index, weight=1):
    # 这里测试的时候对验证集也进行计算，以便于后续模型融合和search weight等提高
    model.eval()
    y_val = []
    age_pred = []
    gender_pred = []
    age_pred_val = []
    gender_pred_val = []
    with torch.no_grad():
        for idx, batch in enumerate(test_dataloader):
            seq_list, lengths, labels = batch
            seq_list_device = [seq.cuda() for seq in seq_list]
            lengths_device = lengths.cuda()
            age_output, gender_output = model(seq_list_device, lengths_device)    
            age_pred.append(age_output.cpu())
            gender_pred.append(gender_output.cpu())
            torch.cuda.empty_cache()
            
        for idx, batch in enumerate(val_dataloader):
            seq_list, lengths, labels = batch
            seq_list_device = [seq.cuda() for seq in seq_list]
            lengths_device = lengths.cuda()
            age_output, gender_output = model(seq_list_device, lengths_device)
            age_pred_val.append(age_output.cpu())
            gender_pred_val.append(gender_output.cpu())
            y_val.append(labels)
            torch.cuda.empty_cache()
            
    # 0到9列存储age的预测概率分布，10列到11列存储gender的预测概率分布，12、13列分别存储age和gender的真实标签        
    oof_train_test[valid_index, :10] += F.softmax(torch.cat(age_pred_val)).numpy() * weight
    oof_train_test[valid_index, 10:12] += F.softmax(torch.cat(gender_pred_val)).numpy() * weight
    oof_train_test[valid_index, 12:] = torch.cat(y_val).numpy()
    oof_train_test[3000000:, :10] += F.softmax(torch.cat(age_pred)).numpy() * (1/5) * weight
    oof_train_test[3000000:, 10:12] += F.softmax(torch.cat(gender_pred)).numpy() * (1/5) * weight
# 定义联合损失函数
def criterion(age_output, gender_output, labels):
    age_loss = nn.CrossEntropyLoss()(age_output, labels[:, 0])
    gender_loss = nn.CrossEntropyLoss()(gender_output, labels[:, 1])
    return age_loss*0.6 + gender_loss*0.4

# 0到9列存储age的预测概率分布，10列到11列存储gender的预测概率分布，12、13列分别存储age和gender的真实标签
oof_train_test = np.zeros((4000000, 14))
# oof_train_test = np.load(os.path.join(model_save, "lstm_v2_300size_fold_2.npy"))

acc_folds = []
model_name = 'lstm_v1_300size_win10'
best_checkpoint_num = 3
for idx, (train_dataloader, val_dataloader, test_dataloader) in enumerate(data_folds):
#     if idx in [0, 1, 2]:
#         continue
    history = {'best_model': []}
    for i in range(best_checkpoint_num):
        history['best_model'].append((0, os.path.join(model_save, '{}_checkpoint_{}.pth'.format(model_name, i))))
    # 对应顺序: creative_w2v, ad_w2v, advertiser_w2v, product_w2v, industry_w2v
    embedding_freeze = [True, True, True, True, True]
    validate_points = list(np.linspace(0, len(train_dataloader)-1, 3).astype(int))[1:]
    
    model = BiLSTM(embedding_list, embedding_freeze, lstm_size=1024, fc1=1024, fc2=512,  num_layers=2, rnn_dropout=0.0, fc_dropout=0.0, embedding_dropout=0.0)    
    model = model.cuda()
    model = nn.parallel.DistributedDataParallel(model, find_unused_parameters=True)
    optimizer = torch.optim.Adam(model.parameters(), betas=(0.9, 0.999), lr=1e-3)
    epochs = 7
#     scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.7)
    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-4, max_lr=2e-3, step_size_up=int(len(train_dataloader)/2), cycle_momentum=False, mode='triangular')
#     scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=3e-3, epochs=epochs, steps_per_epoch=len(train_dataloader), pct_start=0.2, anneal_strategy='linear', div_factor=30, final_div_factor=1e4)
    for epoch in range(1, epochs+1):
        writer = SummaryWriter(log_dir='./runs/{}_fold_{}'.format(model_name, idx))
        train(model, train_dataloader, val_dataloader, criterion, optimizer, epoch, history, validate_points, scheduler, step=True)
#         scheduler.step()
        gc.collect()
    for (acc, checkpoint_pth), weight in zip(sorted(history['best_model'], reverse=True), [0.5, 0.3, 0.2]):
        model.load_state_dict(torch.load(checkpoint_pth, map_location=torch.device('cpu')), strict=False)
        test(oof_train_test, model, test_dataloader, val_dataloader, valid_indexs[idx], weight=weight)
    acc_folds.append(sorted(history['best_model'], reverse=True)[0][0])
    np.save(os.path.join(model_save, "{}_fold_{}.npy".format(model_name, idx)), oof_train_test)
    del model, history
    gc.collect()
    torch.cuda.empty_cache()
acc_folds
np.save(os.path.join(res_path, "{}_5folds_{:.4f}.npy".format(model_name, np.mean(acc_folds))), oof_train_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12295700276590395, 'fraction_numerical': 0.020870002514458134, 'mean_word_length': 2.973276723276723, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '171668', 'license': 'MIT', 'n_tokens_mistral': 5830, 'n_tokens_neox': 5680, 'n_words': 819, 'path': '2020-Tencent-Advertisement-Algorithm-Competition-Rank19/3_LSTM_v1_win10_300size_1.4721.ipynb', 'repo_name': 'PerpetualSmile/Code-Repository', 'size': '15908'}