<jupyter_start><jupyter_text>对100w的数据进行分析(表现比较好，JMT小于6的数据)<jupyter_code>import pandas as pd
df = pd.read_csv('../data/SAP_MEETINGJMT_20170810-20170825/all_jmt.csv')

df_all6 = df[df.USERJMT<=6]
print df_all6.columns.values
print df_all6.shape
# df_sample.head()
print df_all6.USEROS.value_counts()
print df_all6.USERBROWSER.value_counts()<jupyter_output>Windows 10              236106
Windows 7               217315
intel mac os x 10.12    191264
intel mac os x 10.11     90419
WbxTPAgent               72320
iOS                      56063
Windows 8.1              54891
intel mac os x 10.6      41191
intel mac os x 10.10     19371
Other                    10262
Android 9.14              6826
Android 9.13              6088
intel mac os x 10.7       5517
Windows Vista             1529
intel mac os x 10.9       1069
Windows 8                  580
Name: USEROS, dtype: int64
Chrome 60                           234464
IE 11.0                             151433
Other                               144849
Firefox 54                          120736
Safari 10                            98060
wbxtpgw                              72320
Chrome 59                            45150
Chrome 10                            41191
Firefox 55                           40662
IE 6.0                                7843
Chrome 51                             7040
Andr[...]<jupyter_text>全部数据分析<jupyter_code># USERJMT 字段进行映射为[0,2]
def jmt_map(x):
    if 0 <= x <= 8:
        return 1
#     elif 6 <= x <= 20:
#         return 1
    elif 8 < x :
        return 0

df['USERJMT_DIS'] = df.USERJMT.map(jmt_map)
df.USERJMT_DIS.value_counts()
# 类别特征进行二值化编码
# dummies_HOUR = pd.get_dummies(df['HOUR'], prefix= 'HOUR')
dummies_USERTYPE = pd.get_dummies(df['USERTYPE'], prefix= 'USERTYPE')
dummies_PLATFORM = pd.get_dummies(df['PLATFORM'], prefix= 'PLATFORM')
# dummies_REGISTER = pd.get_dummies(df['REGISTER'], prefix= 'REGISTER')
dummies_REFNUM6 = pd.get_dummies(df['REFNUM6'], prefix= 'REFNUM6')
dummies_USEROS = pd.get_dummies(df['USEROS'], prefix= 'USEROS')
dummies_USERBROWSER = pd.get_dummies(df['USERBROWSER'], prefix= 'USERBROWSER')
dummies_DOWNLOADMETHOD = pd.get_dummies(df['DOWNLOADMETHOD'], prefix= 'DOWNLOADMETHOD')
dummies_SERVICETYPE = pd.get_dummies(df['SERVICETYPE'], prefix= 'SERVICETYPE')
# dummies_COUNTRY = pd.get_dummies(df['COUNTRY'], prefix= 'COUNTRY')
# dummies_REGION = pd.get_dummies(df['REGION'], prefix= 'REGION')
# dummies_CITY = pd.get_dummies(df['CITY'], prefix= 'CITY')
dummies_MACHINE_TYPE = pd.get_dummies(df['MACHINE_TYPE'], prefix= 'MACHINE_TYPE')


# jmt_feature = pd.concat([dummies_HOUR, dummies_USERTYPE, dummies_PLATFORM,df['REGISTER'],dummies_REFNUM6,dummies_USEROS,dummies_USERBROWSER,dummies_DOWNLOADMETHOD,dummies_SERVICETYPE,dummies_COUNTRY,dummies_REGION,dummies_CITY,dummies_MACHINE_TYPE], axis=1)
jmt_feature = pd.concat([dummies_USERTYPE, dummies_PLATFORM,df['REGISTER'],dummies_REFNUM6,dummies_USEROS,dummies_USERBROWSER,dummies_DOWNLOADMETHOD,dummies_SERVICETYPE,dummies_MACHINE_TYPE], axis=1)
jmt_target = df.loc[:,['USERJMT_DIS']]

pd.set_option('display.max_rows',None)
pd.set_option('display.max_columns',None)
print jmt_feature.shape
# 把数据集随机拆为训练集和测试集

from sklearn.cross_validation import train_test_split
jmt_sample_train, jmt_sample_test, jmt_target_train, jmt_target_test = train_test_split(jmt_feature, jmt_target, test_size=0.25, random_state=1)
# 转化为numpy数据格式，输入决策树训练. entropy 为信息增益
from sklearn.tree import DecisionTreeClassifier

jmt_DTC=DecisionTreeClassifier(criterion='gini',max_depth=7)
jmt_DTC=jmt_DTC.fit(jmt_sample_train, jmt_target_train)
print jmt_sample_train.columns.values
# 结果展示
from IPython.display import Image  
from sklearn.externals.six import StringIO
import pydotplus
from sklearn import tree

dot_data = StringIO() 
tree.export_graphviz(jmt_DTC, out_file=dot_data, 
  feature_names = ['USERTYPE_New','USERTYPE_Return','USERTYPE_Update','PLATFORM_0',
 'PLATFORM_1','PLATFORM_4','PLATFORM_5','PLATFORM_7','PLATFORM_10',
 'PLATFORM_11','PLATFORM_12','PLATFORM_13','PLATFORM_15','REGISTER',
 'REFNUM6_0','REFNUM6_2','REFNUM6_5','REFNUM6_8','REFNUM6_15','REFNUM6_16',
 'REFNUM6_17','REFNUM6_19','REFNUM6_22','REFNUM6_28','REFNUM6_41',
 'REFNUM6_43','REFNUM6_45','REFNUM6_51','USEROS_Android 9.13',
 'USEROS_Android 9.14','USEROS_Other','USEROS_WbxTPAgent',
 'USEROS_Windows 10','USEROS_Windows 7','USEROS_Windows 8',
 'USEROS_Windows 8.1','USEROS_Windows Vista','USEROS_iOS',
 'USEROS_intel mac os x 10.10','USEROS_intel mac os x 10.11',
 'USEROS_intel mac os x 10.12','USEROS_intel mac os x 10.6',
 'USEROS_intel mac os x 10.7','USEROS_intel mac os x 10.9',
 'USERBROWSER_Android 9.13.0.19130148.19130148',
 'USERBROWSER_Android 9.14.0.19140171.19140171','USERBROWSER_Chrome 10',
 'USERBROWSER_Chrome 34','USERBROWSER_Chrome 46','USERBROWSER_Chrome 51',
 'USERBROWSER_Chrome 52','USERBROWSER_Chrome 56','USERBROWSER_Chrome 57',
 'USERBROWSER_Chrome 58','USERBROWSER_Chrome 59','USERBROWSER_Chrome 60',
 'USERBROWSER_Chrome 61','USERBROWSER_Firefox 45','USERBROWSER_Firefox 47',
 'USERBROWSER_Firefox 51','USERBROWSER_Firefox 52','USERBROWSER_Firefox 53',
 'USERBROWSER_Firefox 54','USERBROWSER_Firefox 55','USERBROWSER_Firefox 56',
 'USERBROWSER_IE 10.0','USERBROWSER_IE 11.0','USERBROWSER_IE 6.0',
 'USERBROWSER_IE 7.0','USERBROWSER_IE 8.0','USERBROWSER_IE 9.0',
 'USERBROWSER_Other','USERBROWSER_Safari 10','USERBROWSER_Safari 9',
 'USERBROWSER_wbxtpgw','DOWNLOADMETHOD_ActiveX','DOWNLOADMETHOD_Extension',
 'DOWNLOADMETHOD_GPC Plugin','DOWNLOADMETHOD_Java Applet Java Client',
 'DOWNLOADMETHOD_Java Applet Native Client','DOWNLOADMETHOD_TFS',
 'DOWNLOADMETHOD_Unknown','SERVICETYPE_EC','SERVICETYPE_MC',
 'SERVICETYPE_SC','SERVICETYPE_TC','SERVICETYPE_UNKNOWN',
 'MACHINE_TYPE_Moblie','MACHINE_TYPE_Other','MACHINE_TYPE_PC'],
                    class_names=['0','1','2'],  
                    filled=True, rounded=True,  
                    special_characters=True)
graph = pydotplus.graph_from_dot_data(dot_data.getvalue())  
Image(graph.create_png())
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

# 决策树分类准确率计算
jmt_predict_test = jmt_DTC.predict(jmt_sample_test)
print accuracy_score(jmt_target_test, jmt_predict_test)
confusion = confusion_matrix(jmt_target_test, jmt_predict_test)
print confusion
# 混淆矩阵C 如下定义，对于第i行，第j列的元素C(i,j)表示正确分类为 group_i 类，实际预测为 group_j类。
# 也就是说，列表示预测的类别。行表示实际类别。
# 详细可见：http://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html#sklearn.metrics.confusion_matrix
print '每个分类的准确率、召回率、f1-score'
print classification_report(jmt_target_test, jmt_predict_test, target_names=['0', '1', '2'])<jupyter_output>每个分类的准确率、召回率、f1-score
             precision    recall  f1-score   support

          0       0.85      0.45      0.58    100660
          1       0.83      0.97      0.90    288306

avg / total       0.84      0.84      0.82    388966
{'fraction_non_alphanumeric': 0.11294964028776978, 'fraction_numerical': 0.06805755395683454, 'mean_word_length': 4.310160427807487, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '464176', 'license': 'Apache-2.0', 'n_tokens_mistral': 3044, 'n_tokens_neox': 2767, 'n_words': 413, 'path': 'jupyter_file/jmt/Untitled2.ipynb', 'repo_name': 'caserwin/daily-learning-python', 'size': '6950'}