<jupyter_start><jupyter_text>prev  test = 1:1<jupyter_code>!cat /home/kesci/data/competition_A/train_set.csv | head -n 2
!cat /home/kesci/data/competition_A/test_set.csv | head -n 1
!./kesci_submit -token ***************** -file /home/kesci/work/sub.csv<jupyter_output>Kesci Submit Tool 4.0.0

> 已验证Token
> 提交文件 /home/kesci/work/sub.csv (18.72 KiB), Target Qiniu
> 已上传 100 %
> 文件已上传        
> 服务器响应: 200 提交成功，请等待评审完成
> 提交完成<jupyter_text>data_pre  train<jupyter_code>import pandas as pd
import gc
train = pd.read_csv('/home/kesci/data/competition_A/train_set.csv').replace(' ', -1).fillna(-1)
# print(train.columns)
# print(train.isnull().values.any())
# print(set(train['Gender\n性别'].values.tolist()))
train_group = train.groupby('护理来源')
del train
gc.collect()
# from sklearn.preprocessing import OneHotEncoder
# enc = OneHotEncoder(sparse = False)
# result = enc.fit_transform(train[['Source of Care\n护理来源']])   
# enc.fit([[0], [1], [2], [3], [4], [5], [6], [7], [8], [9]])
# result = enc.transform(data[[41]])
dic_1 = {'Private Hospital':1, -1:2, 'Governament Hospital':3, 'Never Counsulted':4, 'clinic':5}
buf_1 = pd.DataFrame()
for name,group in train_group:
    group['护理来源'] = dic_1[name]
    buf_1 = pd.concat([buf_1,group],ignore_index=True)

dic_2 = {'F':1, 'M':2}
buf_1_group = buf_1.groupby('性别')
buf_2 = pd.DataFrame()
for name,group in buf_1_group:
    group['性别'] = dic_2[name]
    buf_2 = pd.concat([buf_2,group],ignore_index=True)
    
dic_3 = {'north':1, 'east':2, 'south':3, 'west':4}    
buf_3 = pd.DataFrame()
buf_2_group = buf_2.groupby('区域')
for name,group in buf_2_group:
    group['区域'] = dic_3[name]
    buf_3 = pd.concat([buf_3,group],ignore_index=True)

buf_3 = buf_3.astype(float)

cat_list = ['肥胖腰围',
            '教育', '未婚',
            '护理来源', '视力不佳',
            '饮酒', '高血压',
            '家庭高血压', '糖尿病',
            '家族糖尿病', '肝炎', '家族肝炎',
            '慢性疲劳']
for i in cat_list:
    buf_3[i] = buf_3[i].astype(int)
buf_3.to_csv('train_.csv',index=0)<jupyter_output><empty_output><jupyter_text>test<jupyter_code>import pandas as pd
import gc
train = pd.read_csv('/home/kesci/data/competition_A/test_set.csv').replace(' ', -1).fillna(-1)
# print(train.columns)
# print(train.isnull().values.any())
# print(set(train['Gender\n性别'].values.tolist()))
train_group = train.groupby('护理来源')
del train
gc.collect()
# from sklearn.preprocessing import OneHotEncoder
# enc = OneHotEncoder(sparse = False)
# result = enc.fit_transform(train[['Source of Care\n护理来源']])   
# enc.fit([[0], [1], [2], [3], [4], [5], [6], [7], [8], [9]])
# result = enc.transform(data[[41]])
dic_1 = {'Private Hospital':1, -1:2, 'Governament Hospital':3, 'Never Counsulted':4, 'clinic':5}
buf_1 = pd.DataFrame()
for name,group in train_group:
    group['护理来源'] = dic_1[name]
    buf_1 = pd.concat([buf_1,group],ignore_index=True)

dic_2 = {'F':1, 'M':2}
buf_1_group = buf_1.groupby('性别')
buf_2 = pd.DataFrame()
for name,group in buf_1_group:
    group['性别'] = dic_2[name]
    buf_2 = pd.concat([buf_2,group],ignore_index=True)
    
dic_3 = {'north':1, 'east':2, 'south':3, 'west':4}    
buf_3 = pd.DataFrame()
buf_2_group = buf_2.groupby('区域')
for name,group in buf_2_group:
    group['区域'] = dic_3[name]
    buf_3 = pd.concat([buf_3,group],ignore_index=True)

buf_3 = buf_3.astype(float)

cat_list = ['肥胖腰围',
            '教育', '未婚',
            '护理来源', '视力不佳',
            '饮酒', '高血压',
            '家庭高血压', '糖尿病',
            '家族糖尿病', '家族肝炎',
            '慢性疲劳']
for i in cat_list:
    buf_3[i] = buf_3[i].astype(int)
buf_3.to_csv('test_.csv',index=0)<jupyter_output><empty_output><jupyter_text>ctb<jupyter_code>from catboost import CatBoostClassifier,CatBoostRegressor
import pandas as pd
from sklearn.model_selection import train_test_split
import numpy as np

data = pd.read_csv('train_.csv').sort_values(by=['肝炎'])

train_ = data.drop(columns=['ID','肝炎'])
label_ = data['肝炎']

train, val, label, val_label = train_test_split(train_,label_,test_size=0.1, random_state=1234)
categorical_features_indices = np.where(train.dtypes != np.float)[0]
categorical_features_indices
model = CatBoostClassifier(iterations=1, 
                            depth=10,
                            one_hot_max_size=10,
                            cat_features=categorical_features_indices,
                            loss_function='Logloss',
                            eval_metric='AUC',
                            logging_level='Verbose',)
model.fit(train,label,eval_set=(val, val_label),plot=True)
import matplotlib.pyplot as plt 
from matplotlib import cm
%matplotlib inline
score = pd.DataFrame()
score['fea_name'] = model.feature_names_
score['fea']=model.feature_importances_
score = score.sort_values(['fea'], ascending=False)
temp = pd.DataFrame()
temp = score[:60]
color = cm.jet(temp['fea']/temp['fea'].max())
plt.figure(figsize=(10, 15))
plt.barh(temp['fea_name'],temp['fea'],height =0.8,color=color,alpha=0.8)
plt.show()
test_ = pd.read_csv('test_.csv')
ans = model.predict(test_.drop(columns=['ID']))
ans.mean()
ans.shape
ans_sub = pd.DataFrame({'ID':test_['ID'].astype(int),'hepatitis':ans.astype(int)})
set(ans.tolist())
ans_sub.to_csv('sub.csv',index=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18344907407407407, 'fraction_numerical': 0.02816358024691358, 'mean_word_length': 4.253292806484295, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '26026', 'license': 'Apache-2.0', 'n_tokens_mistral': 2226, 'n_tokens_neox': 2224, 'n_words': 339, 'path': 'ctb&data_pre.ipynb', 'repo_name': 'yrqUni/IAS_stc', 'size': '5184'}