<jupyter_start><jupyter_text>Read in data<jupyter_code>data_path = "D:\\Users\\Nicholas\\Projects\\repos\\spc_charts\\data\\raw\\"
data = pd.read_csv(Path(data_path) / 'nCov-2019 SG - Sheet1.csv')
data['Date'] = pd.to_datetime(data['Date'])<jupyter_output><empty_output><jupyter_text>Plot<jupyter_code>fig, ax = plt.subplots(figsize=(15,5))
sns.relplot(x='Date', y='Count', data=data, kind='line', ax=ax)
plt.close()<jupyter_output><empty_output><jupyter_text>Get subset of data<jupyter_code>data_filter = (data['Date'] >= '2020-02-15') & (data['Date'] < '2020-04')
in_control_filter = (data['Date'] >= '2020-02-15') & (data['Date'] < '2020-03-06')
cutoff = '2020-02-15'
fig, ax = plt.subplots(figsize=(15,5))
sns.relplot(x='Date', y='Count', data=data[data_filter], kind='line', ax=ax)
plt.close()
fig, ax = plt.subplots(figsize=(15,5))
sns.relplot(x='Date', y='Count', data=data[in_control_filter], kind='line', ax=ax)
plt.close()<jupyter_output><empty_output><jupyter_text>Hypothesis test  Auto correlation<jupyter_code>_ = plot_acf(data.loc[in_control_filter, 'Count'], lags=10)
acorr_ljungbox(data.loc[in_control_filter, 'Count'], lags=10)<jupyter_output><empty_output><jupyter_text>Check if poisson distribution is appropriate<jupyter_code>import scipy.stats as stats
import scipy as sp
sample_mean = data.loc[in_control_filter, 'Count'].mean()
sample_variance = data.loc[in_control_filter, 'Count'].var()
sample_mean
sample_variance
fig, ax = plt.subplots()
_ = stats.probplot(data.loc[in_control_filter, 'Count'], dist=stats.poisson, sparams=(sample_mean, ), plot=ax)
_ = plt.title('QQ Plot for Poisson')<jupyter_output><empty_output><jupyter_text>Use Chisq goodness of fit<jupyter_code>obs = data.loc[in_control_filter, 'Count']
obs_x = obs.sort_values().unique().tolist()
theoretical_freq = pd.Series(stats.poisson.pmf(obs_x, obs.mean()), index = obs_x)
obs_freq = obs.value_counts() / obs.value_counts().sum()
compare_df = pd.concat([theoretical_freq, obs_freq], axis=1)
compare_df.columns = ['theoretical_freq', 'obs_freq']
compare_df
stats.chisquare(f_obs = compare_df.obs_freq, f_exp=compare_df.theoretical_freq)<jupyter_output><empty_output><jupyter_text>Use Poisson Control Chart (C Chart)<jupyter_code>def c_chart(arg, mu0):
    
    UCL = mu0 + 3 * np.sqrt(mu0)
    center = mu0
    LCL = mu0 - 3 * np.sqrt(mu0)
    
    # Counts cannot be negative
    if LCL < 0:
        LCL = 0
    
    ret_df = pd.DataFrame({
        'x':arg.tolist(),
        'UCL':UCL,
        'Center':center,
        'LCL':LCL
    })
    
    return ret_df
    
x_ind_df = c_chart(data.loc[in_control_filter, 'Count'], data.loc[in_control_filter, 'Count'].mean())
x_ind_df = pd.concat([data.loc[in_control_filter, 'Date'].reset_index(drop=True), x_ind_df], axis=1)
fig, ax = plt.subplots(figsize=(15,5))
sns.relplot(x='Date', y='x', data=x_ind_df, kind='line', ax=ax, color='blue', marker='o')
sns.lineplot(x='Date', y='UCL', data=x_ind_df, drawstyle='steps-pre', ax=ax, color='red')
sns.lineplot(x='Date', y='LCL', data=x_ind_df, drawstyle='steps-pre', ax=ax, color='red')
sns.relplot(x='Date', y='Center', data=x_ind_df, kind='line', ax=ax, color='black')
plt.close()
plt.close()
plt.title('Individual Measurement Chart')
plt.ylabel('Counts of COVID-19 Cases')
ax.set(xticks=x_ind_df.Date.values)
plt.xticks(rotation=45)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18116591928251122, 'fraction_numerical': 0.02242152466367713, 'mean_word_length': 6.502242152466367, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '612759', 'license': 'FTL', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1326, 'n_words': 203, 'path': 'notebooks/3.0-hwant-process-covid19.ipynb', 'repo_name': 'howewenann/surveillance', 'size': '3345'}