<jupyter_start><jupyter_text>src = '/media/w/1c392724-ecf3-4615-8f3c-79368ec36380/DS Projects/Kaggle/Quora/scripts/features/'X_test = get_test()new_feats = get_new_feats()networks_tony = pd.read_pickle(src + 'test_fullnetworkfeatsTony.pkl')networks_weighted = pd.read_pickle(src + 'test_networkfeats_weighted.pkl')col_dict = {    'q1_cluster': 'q1_cluster_tony',    'q1_degrees': 'q1_degrees_tony',    'q1_squared_cluster': 'q1_squared_cluster_tony',    'q1_triangles_cluster': 'q1_triangles_cluster_tony',    'q2_cluster': 'q2_cluster_tony',    'q2_degrees': 'q2_degrees_tony',    'q2_squared_cluster': 'q2_squared_cluster_tony',    'q2_triangles_cluster': 'q2_triangles_cluster_tony'}networks_tony.rename(columns = col_dict, inplace = True)cv_svd50_dist = pd.read_csv(src + 'test_SVD_CV1gram_50dim.csv')cv_lsa50_dist = pd.read_csv(src + 'test_LSA_CV1gram_50dim.csv')tfidf_svd50_dist = pd.read_csv(src + 'test_SVD_TFIDF_3grams_words_50dim.csv')tfidf_lsa50_dist = pd.read_csv(src + 'test_LSA_TFIDF_3grams_words_50dim.csv')d2v_pre = pd.read_csv(src + 'test_doc2vec_pretrained_distances.csv')d2v_quora = pd.read_csv(src + 'test_doc2vec_quoratrain_distances.csv')transforms = pd.read_csv(src + 'test_SVDLSA_CV1gram_distances.csv')X_test = pd.concat([X_test, new_feats, networks_tony, networks_weighted,                     cv_svd50_dist, cv_lsa50_dist, tfidf_svd50_dist, tfidf_lsa50_dist,                     d2v_pre, d2v_quora, transforms], axis = 1)cols_to_drop = ['counts_max_network_weighted', 'counts_min_network_weighted', 'diff_counts_network_weighted', 'diff_degrees_network_weighted', 'diff_triangles_cluster_network_weighted', 'exactly_same', 'jaccard_distance_test_LSA_TFIDF_3grams_words_50dim', 'max_degrees_network_weighted', 'max_triangles_cluster_network_weighted', 'min_degrees_network_weighted', 'min_triangles_cluster_network_weighted', 'mult_counts_network_weighted', 'q1_counts_network_weighted', 'q1_degrees_network_weighted', 'q1_triangles_cluster_network_weighted', 'q2_counts_network_weighted', 'q2_degrees_network_weighted', 'q2_triangles_cluster_network_weighted', 'question1_100', 'question1_101', 'question1_102', 'question1_103', 'question1_104', 'question1_105', 'question1_106', 'question1_107', 'question1_108', 'question1_109', 'question1_110', 'question1_111', 'question1_112', 'question1_113', 'question1_114', 'question1_115', 'question1_116', 'question1_117', 'question1_118', 'question1_119', 'question1_120', 'question1_121', 'question1_122', 'question1_123', 'question1_124', 'question1_125', 'question1_126', 'question1_127', 'question1_68', 'question1_69', 'question1_70', 'question1_71', 'question1_72', 'question1_73', 'question1_74', 'question1_75', 'question1_76', 'question1_77', 'question1_78', 'question1_79', 'question1_80', 'question1_81', 'question1_82', 'question1_83', 'question1_84', 'question1_85', 'question1_86', 'question1_87', 'question1_88', 'question1_89', 'question1_90', 'question1_91', 'question1_92', 'question1_93', 'question1_94', 'question1_95', 'question1_96', 'question1_97', 'question1_98', 'question1_99', 'question2_103', 'question2_104', 'question2_106', 'question2_108', 'question2_109', 'question2_110', 'question2_112', 'question2_113', 'question2_114', 'question2_115', 'question2_116', 'question2_118', 'question2_119', 'question2_120', 'question2_121', 'question2_122', 'question2_123', 'question2_126', 'question2_127', 'question2_81', 'question2_83', 'question2_87', 'question2_88', 'question2_89', 'question2_96', 'sum_counts_network_weighted', 'sum_degrees_network_weighted', 'sum_triangles_cluster_network_weighted']X_test.drop(cols_to_drop, axis = 1, inplace = True)X_test = X_test.astype('float32')print('Final shape:', X_test.shape)X_test.to_pickle('Xtest_916cols.pkl')del new_feats, networks_tony, networks_weighted, cv_svd50_dist, cv_lsa50_dist, \tfidf_svd50_dist, tfidf_lsa50_dist, d2v_pre, d2v_quora, transformsgc.collect()<jupyter_code>X_test = pd.read_pickle('Xtest_500bestCols.pkl')
predict_test(X_test, 'XGB_new_NetworkFeats_experiments_500feats')<jupyter_output>Predicting on test set.
{'fraction_non_alphanumeric': 0.13431372549019607, 'fraction_numerical': 0.10024509803921569, 'mean_word_length': 11.634674922600619, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '68768', 'license': 'MIT', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1581, 'n_words': 184, 'path': 'models/.ipynb_checkpoints/XGB - Test Predictions 13.05-checkpoint.ipynb', 'repo_name': 'Ravirajadrangi/Quora-Questions-Pairs-', 'size': '4080'}