<jupyter_start><jupyter_text>Combining Word Embeddings with More Features<jupyter_code>def get_embedding(data_loader, model, device):
    from tqdm.notebook import tqdm
    # Put the model in eval mode
    model.to(device)
    model.eval()
    # List for store final predictions
    final_predictions = []
    with torch.no_grad():
        tk0 = tqdm(data_loader, total=len(data_loader))
        for b_idx, data in enumerate(tk0):
            for key,value in data.items():
                data[key] = value.to(device)
            predictions = model._embeddings(data['ids'],data['mask'])
            predictions = predictions.cpu()
            final_predictions.append(predictions)
    return np.vstack(final_predictions)
col_unique = generate_col_unique(df_train,config.COLUMNS_ENCODE)
tokenizer     = transformers.RobertaTokenizer.from_pretrained(config.PRETRAINED)
train_dataset = BNPParibasText(df_train,config.MAX_LENGTH,tokenizer,col_unique)
train_loader = torch.utils.data.DataLoader(
            train_dataset,
            batch_size  = 32,
            pin_memory  = True,
            num_workers = 72
        )
embedding_all = 0
for fold in np.sort(df_train['fold'].unique()):
    model         = Roberta_Model(pretrained_model=config.PRETRAINED)
    model.load_state_dict(torch.load(f'../03.Models/BNP_PARIBAS_ROBERTA_FOLD_{fold}'))
    embedding_all += get_embedding(train_loader, model, 'cuda')
    del model
    torch.cuda.empty_cache()
embedding_all = embedding_all/len(df_train['fold'].unique())
df_train[[f'emb_{col_unique}_{i}' for i in range(embedding_all.shape[1])]] = embedding_all
df_test      = pd.read_csv(os.path.join(DATA_PATH,'test_preprocessed.csv'))
df_test['target'] = -1

col_unique = generate_col_unique(df_test,config.COLUMNS_ENCODE)
tokenizer     = transformers.RobertaTokenizer.from_pretrained(config.PRETRAINED)
test_dataset = BNPParibasText(df_test,config.MAX_LENGTH,tokenizer,col_unique)
test_loader = torch.utils.data.DataLoader(
            test_dataset,
            batch_size  = 32,
            pin_memory  = True,
            num_workers = 72
        )
embedding_all = 0
for fold in np.sort(df_train['fold'].unique()):
    model         = Roberta_Model(pretrained_model=config.PRETRAINED)
    model.load_state_dict(torch.load(f'../03.Models/BNP_PARIBAS_ROBERTA_FOLD_{fold}'))
    embedding_all += get_embedding(test_loader, model, 'cuda')
    del model
    torch.cuda.empty_cache()
embedding_all = embedding_all/len(df_train['fold'].unique())
df_test[[f'emb_{col_unique}_{i}' for i in range(embedding_all.shape[1])]] = embedding_all
columns_modeling = ['additives_n','ingredients_from_palm_oil_n',
                    'ingredients_that_may_be_from_palm_oil_n','target',
                    'states_en_brands','states_en_categories','states_en_characteristics','states_en_expiration date',
                    'states_en_general_complete','states_en_ingredients','pnns_groups_1','pnns_groups_2',
                    'states_en_packaging','states_en_packaging-code-','states_en_photo_upload',
                    'states_en_photo_validate','states_en_product name','states_en_quantity','diff_t'] + [f'emb_{col_unique}_{i}' for i in range(embedding_all.shape[1])]
columns_label = df_train[columns_modeling].select_dtypes(include=['object']).columns.to_list()
print(columns_label)
df_train,dict_le = label_encoding(df_train,label_cols = columns_label, drop_original = True, missing_new_cat = True)
df_test = apply_label_encoder(df_test,dict_le,drop_original = True, missing_new_cat = True)
params = {
        'task': 'train',
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'metric': {'rmse'},
        'num_leaves':12,
        'learning_rate': 0.001,
        "min_child_samples": 150,
        "max_depth" : 5,
        'feature_fraction':  0.5,
        "bagging_freq": 1,
        'bagging_fraction': 0.75,
        "is_unbalance" : False,
        'force_col_wise':True,
        'num_threads':18,
        #"scale_pos_weight":5 -> Generally  is the ratio of number of negative class to the positive class.
        'bagging_seed':42,
        'lambda_l1':1.5,
        'lambda_l2':1,
        'verbose': 1

}
cat_columns = [i for i in df_train.columns.to_list() if i.startswith('label_')]
columns_modeling_last = list(set(columns_modeling)-set(columns_label)) + ['fold'] + cat_columns 
results,models,importances,oof,feature_list = Training_Lightgbm(df_train[columns_modeling_last],params,fold_column = 'fold',target_column = 'target',cat_vars = cat_columns ,metric = 'RMSE',early_stopping = 200,max_boost_round = 8000)
probs = 0
for i in models:
    probs = probs + (i.predict(df_test[feature_list]))
    print('fin_predict')
y_test_pred = probs/5.0
print(f'Real: ',math.sqrt(mean_squared_error(y_test_pred,df_test['Target'].values)))<jupyter_output>fin_predict
fin_predict
fin_predict
fin_predict
fin_predict
Real:  4.4556653551634025
{'fraction_non_alphanumeric': 0.1205311542390194, 'fraction_numerical': 0.015526046986721144, 'mean_word_length': 4.1, 'pattern_counts': {'":': 3, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '491900', 'license': 'MIT', 'n_tokens_mistral': 1712, 'n_tokens_neox': 1656, 'n_words': 247, 'path': '02.Code/03.Roberta Model.ipynb', 'repo_name': 'CristianLazoQuispe/BNP-LATAM-Datathon', 'size': '4895'}