<jupyter_start><jupyter_text>查看数据概况<jupyter_code>data.describe()
data.info()
#查看每个字段不重复的变量有多少个，为挑选目标变量做准备
col_num = {}
for col in data.columns:
    col_num[col] = len(data[col].unique())
col_num<jupyter_output><empty_output><jupyter_text>Arrest作为目标变量<jupyter_code>data['Arrest'] = data['Arrest'].astype(int)
data.head()<jupyter_output><empty_output><jupyter_text>查看Arrest与Primary Type的关系（可视化）<jupyter_code>fbi_arrest_count = data.groupby(['Primary Type','Arrest']).agg({'Arrest':'count'}).unstack()
fbi_arrest_count.columns = [0,1]
fbi_arrest_count
from pyecharts.charts import Bar
from pyecharts import options as opts
import random
%matplotlib inline
attr = fbi_arrest_count.index.tolist()
v1 = fbi_arrest_count[0].tolist()
v2 = fbi_arrest_count[1].tolist()
#bar = Bar()
#bar.add("Arrest 0", attr, v1, is_label_show = True, xaxis_label_textsize = 6, xaxis_rotate = 90, is_stack=True)
#bar.add("Arrest 1", attr, v2, is_label_show = True, xaxis_label_textsize = 6,xaxis_rotate = 90,is_stack=True)
#bar.render("bar.html")
bar = (
        Bar()
        .add_xaxis(attr)
        .add_yaxis("Not Arrest", v1, stack="stack1",color="black")
        .add_yaxis("Arrest", v2,stack="stack1",color="red")
        .set_series_opts(label_opts=opts.LabelOpts(is_show=False))
        .set_global_opts(title_opts=opts.TitleOpts(title="Primary Type Count By FBI Code"),
                         xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=90,font_size=10))
                         
        )
    )
bar.render("bar.html")<jupyter_output><empty_output><jupyter_text>数据处理、缺失值填充<jupyter_code>from sklearn.preprocessing import LabelEncoder
feat_col = [col for col in data.columns if col not in ['Case Number','Arrest']]
obj_col = data[feat_col].dtypes[data[feat_col].dtypes == 'object'].index.tolist()
for col in obj_col:
    le = LabelEncoder()
    data[col] = le.fit_transform(data[col].astype(str))
    
    
for col in feat_col:
    data[col] = data[col].fillna(data[col].mode()[0])
    
data.info()
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
train_x,val_x,train_y,val_y = train_test_split(data[feat_col],data['Arrest'],test_size=0.8,random_state=2019)
'done'
clf = GaussianNB()
clf.fit(train_x,train_y)
pre_val = clf.predict_proba(val_x)[::,1]
print('roc is',roc_auc_score(val_y,pre_val))<jupyter_output>roc is 0.7514373067936143
{'fraction_non_alphanumeric': 0.14513422818791946, 'fraction_numerical': 0.020553691275167787, 'mean_word_length': 5.794871794871795, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '390534', 'license': 'MIT', 'n_tokens_mistral': 976, 'n_tokens_neox': 959, 'n_words': 133, 'path': 'naivebayes/naivebayes.ipynb', 'repo_name': 'DiracSea/CS235', 'size': '2384'}