<jupyter_start><jupyter_text>计算每个Feature的WOE/IV<jupyter_code>iv_list = {}
woe_table = {}
type_list = {}

def mt_iv(feature):
    return atecml.data.calc_iv(df=train_df,feature=feature,target=target,category_num=64)

with ProcessPoolExecutor(max_workers=40) as pool:
    for pool_result in pool.map(mt_iv,predictors):
        iv = pool_result[0]
        woe_df = pool_result[1]
        feature_type = pool_result[2]
        iv_list.update(iv)
        woe_table.update(woe_df)
        type_list.update(feature_type)
iv_df = pd.DataFrame()
iv_df['IV'] = pd.DataFrame.from_dict(iv_list,orient='index')[0]
iv_df['Type'] = pd.DataFrame.from_dict(type_list,orient='index')[0]
iv_df.sort_values('IV',ascending=False).head(100)
iv_df.loc['NaN_LIST']
woe_table['NaN_LIST'][woe_table['NaN_LIST']['Bad Rate'] > 0.01]['Value']
woe_table['NaN_LIST'].loc[25,'Value']
important_feature = list(iv_df[ (iv_df.IV > 0.1) & (iv_df.IV < 3 )].index)
filter_feature = important_feature 
#list(set(important_feature) -set(atecml.data.NOT_IMP_FEATURE))
print(len(filter_feature))
rename_dict ={}
for item in filter_feature:
    old_feature_name = item
    new_feature_name = 'n_' + item
    rename_dict[old_feature_name] = new_feature_name<jupyter_output>257<jupyter_text>使用WOE/IV结果对原有训练集特征纬度提升<jupyter_code>test_df = atecml.data.load_train()
test_df['f5'] =test_df['f5']//100

a = train_df['f5'].unique()
test_df['f5'][~test_df['f5'].isin(a)] =np.nan

def mt_conv_test(feature):
    return atecml.data.woe_convert(df=test_df,feature=feature,woe_table=woe_table,type_list=type_list,category_num=64)

new_feature_tt =[]
with ProcessPoolExecutor(max_workers=60) as pool:
    for pool_result in pool.map(mt_conv_test,predictors):
        new_feature_tt.append(pool_result)
        
_t_new_tt_df = pd.concat(new_feature_tt,axis=1)

_t_new_tt_df = _t_new_tt_df[filter_feature]

mean = _t_new_tt_df.mean(axis=1)
variance = _t_new_tt_df.var(axis=1)
skew = _t_new_tt_df.skew(axis=1)
_t_new_tt_df['mean'] = mean
#_t_new_tt_df['variance'] = variance
#_t_new_tt_df['skew'] = skew
_t_new_tt_df.rename(columns=rename_dict,inplace=True)
ext_tt_df = pd.concat([test_df[['id','date','label']],_t_new_tt_df],axis=1)
ext_tt_df.to_pickle('./01_train.dat')
test_df = atecml.data.load_test()
test_df['f5'] =test_df['f5']//100
test_df['f5'][~test_df['f5'].isin(a)] =np.nan

def mt_conv_test(feature):
    return atecml.data.woe_convert(df=test_df,feature=feature,woe_table=woe_table,type_list=type_list,category_num=64)

new_feature_tt =[]
with ProcessPoolExecutor(max_workers=60) as pool:
    for pool_result in pool.map(mt_conv_test,predictors):
        new_feature_tt.append(pool_result)
        
_t_new_tt_df = pd.concat(new_feature_tt,axis=1)
_t_new_tt_df = _t_new_tt_df[filter_feature]


mean = _t_new_tt_df.mean(axis=1)
variance = _t_new_tt_df.var(axis=1)
skew = _t_new_tt_df.skew(axis=1)
_t_new_tt_df['mean'] = mean
#_t_new_tt_df['variance'] = variance
#_t_new_tt_df['skew'] = skew

_t_new_tt_df.rename(columns=rename_dict,inplace=True)
ext_tt_df = pd.concat([test_df[['id','date']],_t_new_tt_df],axis=1)
ext_tt_df.to_pickle('./01_test.dat')
list(ext_tt_df.columns)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1508716323296355, 'fraction_numerical': 0.01901743264659271, 'mean_word_length': 7.176165803108808, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '353670', 'license': 'MIT', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1376, 'n_words': 142, 'path': 'model/WOE_as_Matrix/01_WOE_IV_for_KGB.ipynb', 'repo_name': 'ogotaiking/anti_ml', 'size': '3155'}