<jupyter_start><jupyter_text>定义函数<jupyter_code>#找出Ｔｒｕｅ/Ｆａｌｓｅ的列，以便做特殊填充
def find_bool_columns(df):
    col_s = []
    for col in df:
        s = df[col]
        sample = _get_first_not_null(s, 5)
        if (sample=='True') | (sample=='False'):
            col_s.append(col)
    return col_s

#此函数的意图是，保存ｆｉｇ的地址
def get_plot(df,plot_col_list,colors={True:'orangered'}):
   
    cid_dict = {}  #记录ｆ地址      
    for c in plot_col_list:

        df[c] = df[c].astype('category')
 
        dist = get_categorical_distribution_by_group(df, model_key, c)

        f = compare_distribution(dist, model_key, c, '', colors = colors)

        cid_dict[c] = f
    return cid_dict

# 筛选出分类不只１种的列
def get_plot_col(df):
    keep_cols = []    
    for col in df:
        if (len(df.drop_duplicates(col))>1):
            keep_cols.append(col)
    df_plot = df[keep_cols]
    
    return df_plot<jupyter_output><empty_output><jupyter_text>部分变量<jupyter_code>from pandas import DataFrame
import pandas as pd
model_key = '是否坏账'
product = 'RWD'
product_name = '任小贷'
with_shumei = False

mail_list = [
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>'
]


group_member = [
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>',
    '<EMAIL>'
    
]

mail_list = group_member

mail_list = ['<EMAIL>']


cc_list = [
    #'<EMAIL>',
    #'<EMAIL>'
]

categorize_num = 10 #变量区间化数量

# 指定要画图的列，若为Ｎｏｎｅ或[],则全部要画
plot_col_list=[]<jupyter_output><empty_output><jupyter_text>删掉没用的cols<jupyter_code>drop_cols = [
    'order_no',
    '用户编号_y',
    '申请日期',
    '纬度',
    '经度',
    '居住城市',
    '单位城市',
    '用户注册时间',
    '认证通过时间',
    '身份证年龄'
]
df_model = load_dataframe('model_data.csv').drop(drop_cols,axis=1)

# save_dataframe(df_model, 'test.csv', index=False)

# df_test = load_dataframe('test.csv')

# df_model.equals(df_test)
# True<jupyter_output><empty_output><jupyter_text>填充空值<jupyter_code>bool_list = find_bool_columns(df_model)

fills_dict = split_dict_keys({tuple(bool_list):'False'})

df_model = fill_df_null(df_model, default_numeric_fill=0, default_non_numeric_fill='空',fills=fills_dict)<jupyter_output><empty_output><jupyter_text>区间化<jupyter_code>df_model = categorize_df(df_model, categorize_num, std_threshold=2)
from jfdata.utilities.publish import get_dtype_info, save_dict
test_dict = get_dtype_info(df_model)

# from json import dump, load
problem_list = []
for k,val in test_dict.items():
    try:
        save_dict(
            val,
            'save_dict_test'
        )
    except Exception as e:
        print(e)
        print(k)
        problem_list.append(k)
        print(val)
import numpy as np
x = np.int64(50)
y = int(50)
type(x), type(y)
from json import dump
with open('test', 'w') as f:
    dump(x, f)
with open('test', 'w') as f:
    dump(y, f)
from jfdata.utilities.publish import get_dtype_info, save_dict
test_dict = get_dtype_info(df_model)

# from json import dump, load
problem_list = []
for k,val in test_dict.items():
    try:
        save_dict(
            val,
            'save_dict_test'
        )
    except Exception as e:
        print(e)
        print(k)
        problem_list.append(k)
        print(val)
df_model[problem_list].dtypes
type(df_model['产品周期'][0])
df_model['产品周期'][0].dtype
from jfdata.utilities.publish import save_dataframe
save_dataframe(df_model, 'model_data_20170411.csv', index = False)
df_plot = get_plot_col(df_model)<jupyter_output><empty_output><jupyter_text>单变量可视化  筛选出要可视化的列及ｍｏｄｅｌ_key<jupyter_code>if plot_col_list:
    plot_col_list = plot_col_list
else:
    plot_col_list = list(set(list(df_plot.columns))-set([model_key]))<jupyter_output><empty_output><jupyter_text>编写可视化函数<jupyter_code>image_dict = get_plot(df_plot,plot_col_list)
# 读取图片地址，并存储下来
cid_dict = {}
i = 0
for k,v in image_dict.items():
    cid = 'image'+str(i).zfill(3)
    path = 'images/{}/{}.png'.format(product, cid)
    image_dict[k].savefig(path)
    cid_dict[k] = {'cid':cid,'path':path}
    i+=1<jupyter_output><empty_output><jupyter_text>邮件报告<jupyter_code>from jfdata.utilities.html_report import Tag
report = Tag('html',tag_class='mail_report')
report  = (
    report
    .add_title('{}坏账情况的分析和建模'.format(product_name))
    .add_list(
        '说明:',
        [
            '截止20170303时，逾期30天以上还款的借贷被看做是一笔坏账(不管最终是否会还款)',
            '分析的数据范围是20170203之前申请的借贷数据',
            '综合利用了支付， 小贷， 同盾分等不同来源的数据'
        ]
    ).add_list(
        '结论:',
        [
            '目前多次借贷的用户少数会出现填写信息和之前不一样的现象。 但是这个现象和是否产生坏账没有迹象能产生关联。建议对于第二次及以上借贷的用户， 采用上一次填写的内容作为默认值，不强迫用户重新填写，但允许用户需要时进行修改',
            '经纬度数据目前缺失非常严重， 无法经纬度数据对于坏账预测的影响',
            '用户填写了几个不同的信息那里的“空”，表示用户是首次借贷。 二次以及二次以上借贷的用户， 风险度明显比首次借贷的用户低。',
            '同盾分背后的明细数据，诸如在其他金融平台借贷的次数，对模型准确性有明显的提升'
        ]
    )
)
chapter_1 = Tag('div')

chapter_1 = (
    chapter_1
    .append(Tag('h2','支付数据单个维度和坏账现象一一比较'))
    .add_list(
        '说明1:',
        [
            '蓝色(False)表示未发生坏账的借贷, 红色(True)表示发生坏账的借贷',
            '纵轴表示某个特征维度的取值或者取值区间，横轴表示在对应的取值或者取值区间中所占的百分比。',
        ]
    )
)

for c in df_plot.columns:
    if c!=model_key:
        section = Tag('div')
        section.append(Tag('h3',c))
        section.add_image(cid_dict[c]['cid'], cid=True)        
        chapter_1.append(section)


report.append(chapter_1)
from jfdata.utilities.publish import send_mail
from premailer import transform
cid_dict_ = {}
for k, v in cid_dict.items():
    cid_dict_[v['cid']] = v['path']
html_text = report.to_html()

styles = '''
<style>
    h1 {
        font-size: 30px;
    }
    
    h2 {
        font-size: 25px;
    }
    
    h3 {
        font-size: 20px;
    }    
    
    img {
        width:90%
    }
    
    ul p {
        margin-top:0;
        margin-bottom:0;
    }
    
    .mail_report_chapter {
        margin: 20px 0 20px 0;
    }
</style>
'''

l = [s for s in html_text.split('\n') if s.replace('\t','').replace(' ','')!='']

html_text = transform(styles+'\n'.join(l))
from jfdata.utilities.publish import send_mail

send_mail(
    '<EMAIL>',
    'Jrsjkxzbbd2333',
    ['<EMAIL>'],
    '{}支付数据分析和建模结果'.format(product_name),
    html_text,
    local_images= cid_dict_,
    cc_addr = []
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1428801287208367, 'fraction_numerical': 0.012228479485116654, 'mean_word_length': 3.2285714285714286, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '244588', 'license': 'MIT', 'n_tokens_mistral': 2949, 'n_tokens_neox': 3031, 'n_words': 372, 'path': '0014_auto_mail/auto_mail.ipynb', 'repo_name': 'junjiecai/jupyter_demos', 'size': '6359'}