<jupyter_start><jupyter_text>4 推測統計 確率分布と統計的検定<jupyter_code>import pandas as pd
import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

%matplotlib inline
df_id_age = pd.read_excel('978-4-274-22101-9.xlsx', 'ID付きPOSデータ（IDデータ）').iloc[:, 0:3]
df_id_age.head()
df_id_pos = pd.read_excel('978-4-274-22101-9.xlsx', 'ID付きPOSデータ（POSデータ）')
df_id_pos = pd.merge(df_id_pos, df_id_age, on='顧客ID')
df_id_pos.head()
# 平均購入金額の95%信頼区間
n = df_id_pos['レシートNo'].nunique()
t_dist = stats.t(loc=df_id_pos.groupby(['レシートNo'])['税抜価格'].sum().mean(),
                 scale=np.sqrt(df_id_pos.groupby(['レシートNo'])['税抜価格'].sum().var() / n),
                 df=n-1)
bottom, up = t_dist.interval(alpha=0.95)
print('95% interval: {:.2f} < x < {:.2f}'.format(bottom, up))
# パン購買比率95%信頼区間
n = df_id_pos['レシートNo'].nunique()
bread = df_id_pos[df_id_pos['中カテゴリ名'] == 'パン']['レシートNo'].nunique()
bread_rate = bread / n
t_dist = stats.t(loc=bread_rate,
                 scale=np.sqrt(bread_rate * (1 - bread_rate) / n),
                 df=n-1)
bottom, up = t_dist.interval(alpha=0.95)
print('95% interval: {:.3f} < x < {:.3f}'.format(bottom, up))
# 30代と40代の平均購買金額比較、等分散性の検定（上側5％を超えるので下がると判定）

pd.options.display.float_format = '{:.0f}'.format

df_mat = pd.DataFrame(index=['平均', '分散', 'ケース数'])
df_40 = df_id_pos[df_id_pos['年代'] == 40]
df_50 = df_id_pos[df_id_pos['年代'] == 50]
df_40_amount = df_40.groupby(['レシートNo'])['税抜価格'].sum()
df_50_amount = df_50.groupby(['レシートNo'])['税抜価格'].sum()
df_mat['40代'] = [df_40_amount.mean(), df_40_amount.var(), df_40['レシートNo'].nunique()]
df_mat['50代'] = [df_50_amount.mean(), df_50_amount.var(), df_50['レシートNo'].nunique()]
df_mat
f = df_mat.loc['分散', '50代'] / df_mat.loc['分散', '40代']
dfx = df_mat.loc['ケース数', '50代'] - 1
dfy = df_mat.loc['ケース数', '40代'] - 1

print('F value:', f)
# 年代別大カテゴリ紅梅点数集計表から、年代ごとに購買するカテゴリに違いがあるか検定
df_cross_age_bcat = df_id_pos.groupby(['年代', '大カテゴリ名']).size().loc[20:80].unstack()
df_cross_age_bcat
x2, p, dof, expected = stats.chi2_contingency(df_cross_age_bcat)

print("χ square {}".format(x2))
print("probability {}".format(p))  # 有意なので差があると判定
print("degree of feedom {}".format(dof))
print(expected)<jupyter_output>χ square 1575.9120249030825
probability 1.204826244161524e-285
degree of feedom 66
[[1.30564736e+00 9.81567035e+00 2.03494468e+02 6.53989438e+01
  1.75796091e+02 8.03206280e+01 1.19559994e+02 1.05384394e+02
  1.43131592e+02 2.32871533e+02 3.61617689e+01 1.24759269e+02]
 [5.42376778e+00 4.07751114e+01 8.45332950e+02 2.71672654e+02
  7.30271591e+02 3.33658572e+02 4.96662164e+02 4.37775542e+02
  5.94580543e+02 9.67367725e+02 1.50218997e+02 5.18260382e+02]
 [1.68074436e+01 1.26355960e+02 2.61956014e+03 8.41872845e+02
  2.26300223e+03 1.03395791e+03 1.53908162e+03 1.35660080e+03
  1.84251600e+03 2.99772762e+03 4.65506161e+02 1.60601126e+03]
 [1.37646214e+01 1.03480457e+02 2.14531456e+03 6.89460052e+02
  1.85330795e+03 8.46770010e+02 1.26044604e+03 1.11100158e+03
  1.50894662e+03 2.45501854e+03 3.81230852e+02 1.31525873e+03]
 [1.05216267e+01 7.91000862e+01 1.63987067e+03 5.27020764e+02
  1.41666188e+03 6.47267926e+02 9.63480385e+02 8.49245581e+02
  1.15343332e+03 1.87660727e+03 2.91411482e+0[...]
{'fraction_non_alphanumeric': 0.18520859671302148, 'fraction_numerical': 0.25, 'mean_word_length': 5.47239263803681, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '617043', 'license': 'MIT', 'n_tokens_mistral': 2154, 'n_tokens_neox': 1705, 'n_words': 205, 'path': 'notebooks/chapter4.ipynb', 'repo_name': 'ihiroky/statistics_for_marketing', 'size': '3164'}