<jupyter_start><jupyter_text>calculate shock<jupyter_code>wv_data_file = 'wv_cases1'
df = wv1
df['demand_shock'] = None

for prov in df.province.unique():
    for ind in df.industry.unique():
        prov_row_idx = df[(df.province==prov)&(df.industry==ind)].index 
        demand_data_points = df.loc[prov_row_idx, 'total_demand'].values
        std_demand = np.std(demand_data_points)
        df.loc[[0, 1], 'demand_shock'] = np.nan # need at least two points to fit a line
        
        for time_pointer in range(2, len(prov_row_idx)):
            y = demand_data_points[max(0, time_pointer-4):time_pointer]
            expected_demand = np.mean(y)
            demand_shock = demand_data_points[time_pointer] - expected_demand
    #         print(x, y, expected_demand, demand_shock)
            df.loc[prov_row_idx[time_pointer], 'demand_shock'] = demand_shock/std_demand
    
print(df.shape)


    # create placeholder
for offset in range(1, 9):
    df['demand_shock_plus_%s' % offset] = None
# shift IV the other way by up to 2 months    
for offset in range(1, 9):
    df['demand_shock_minus_%s' % offset] = None

for prov in df.province.unique():
    for ind in df.industry.unique():
        prov_row_idx = df[(df.province==prov)&(df.industry==ind)].index 

        for offset in range(1, 9):
            df.loc[prov_row_idx, 'demand_shock_plus_%s' % offset] = df.loc[prov_row_idx, 'demand_shock'].shift(offset)
        # shift IV the other way by up to 2 months    
        for offset in range(1, 9):
            df.loc[prov_row_idx, 'demand_shock_minus_%s' % offset] = df.loc[prov_row_idx, 'demand_shock'].shift(-offset)
        
print(df.shape)
for offset in range(1, 6):
    df['perc_abuse_minus_%s' % offset] = None

for prov in df.province.unique():
    for ind in df.industry.unique():
        prov_row_idx = df[(df.province==prov)&(df.industry==ind)].index 

        for offset in range(1, 6):
            df.loc[prov_row_idx, 'perc_abuse_minus_%s' % offset] = df.loc[prov_row_idx, 'perc_abuse'].shift(-offset)
        # shift IV the other way by up to 2 months    
        
print(df.shape)
df.head()
def yr_wk_to_float(yr_wk):
    yr, wk = yr_wk.split('-')
    return int(yr) + float(wk)/100

df['yr_wk_float'] = df.year_week.apply(yr_wk_to_float)
wv_data_file = 'wv_cases1'

df.to_csv(
    data_dir + 'regression_data_%s_causal_ma_detrend_20210520.csv' % wv_data_file, 
    index=False)
df.head()
'regression_data_%s_causal_ma_detrend_20210520.csv' % wv_data_file
df.columns
df.industry.unique()<jupyter_output><empty_output><jupyter_text>Add USD IV<jupyter_code># folder = '/Users/boyuliu/Dropbox/Boyu-Joann/Data/exchange_rate/China/'
# iv = pd.read_csv(folder + 'weekly_CNY_exchange_rate.csv')
folder = '/Users/boyuliu/Dropbox/Boyu-Joann/Data/exchange_rate/'
iv = pd.read_csv(folder + 'weekly_exchange_rate.csv')
print(iv.year_week.max(), iv.year_week.min(), iv.shape)
iv.head()
# weekly = pd.merge(weekly, iv[['year_week','ex_rate']], on='year_week', how='left')
# weekly.shape
iv['month'] = iv['fake_date'].apply(lambda x: x[:-3])
iv['quarter'] = iv['fake_date'].apply(lambda x: x[:5] + str((int(x[5:7])-1)//3 + 1))
iv['ex_rate_diff'] = iv['ex_rate'] - iv['ex_rate'].shift(1)
# shift IV by up to 2 months
for offset in range(1, 9):
    iv['ex_rate_diff_plus_%s' % offset] = iv['ex_rate_diff'].shift(offset)
# shift IV the other way by up to 2 months    
for offset in range(1, 9):
    iv['ex_rate_diff_minus_%s' % offset] = iv['ex_rate_diff'].shift(-offset)
#     iv['ex_rate_diff_%s' % offset] = iv['ex_rate'] - iv['ex_rate'].shift(-offset)    
iv.head()
print(wv_data_file)
df = pd.read_csv(data_dir + 'regression_data_%s_causal_ma_detrend_20210520.csv' % wv_data_file)
print(df.shape)
df = pd.merge(df.drop(iv.columns[1:], axis=1), iv, on='year_week', how='left')
print(df.shape)
#     dup_cols = [col for col in df.columns if col[-1]=='x' or col[-1]=='y']
#     df = df.drop(dup_cols, axis=1)

df.to_csv(
    data_dir + 'regression_data_%s_causal_ma_detrend_USD_20210520.csv' % wv_data_file, 
    index=False)
df.head()<jupyter_output>wv_cases1
(33561, 66)
(33561, 66)<jupyter_text>plot effect of transformation<jupyter_code>df = pd.read_csv(data_dir+'regression_data_wv_cases1_causal_ma_detrend_20210301.csv')
plot_order = df.groupby('province').mean().sort_values(by='total_demand', ascending=False).index.values
plot_order
# 
# plot = sns.barplot(x='ID', y='Amount', data=df, order=plot_order)
# # a4_dims = (24, 16)
# # a4_dims = (16, 12)
# a4_dims = (12, 6)
# fig, ax = plt.subplots(figsize=a4_dims)

# df['total_demand'] = df['total_demand'].astype(float)
# df['name_len'] = df['province'].apply(len)
# df.sort_values('name_len', inplace=True)
# bxplot = sns.boxplot(data=df, x='province', y='total_demand', order=plot_order)
# for item in bxplot.get_xticklabels():
#     item.set_rotation(45)
#     item.set_fontsize(8) 
# plt.ylabel('Weekly total excess demand', fontsize=15, fontweight='black', color = '#333F4B')
# plt.xlabel('Province', fontsize=15, fontweight='black', color = '#333F4B')
# plt.title('Excess demand for Thai provinces', fontsize=15, fontweight='black', color = '#333F4B')    
# plt.savefig('../../../plots/paper/demand_by_province.jpg', bbox_inches='tight')
# plt.show()
# a4_dims = (24, 16)
# a4_dims = (16, 12)
a4_dims = (12, 6)
fig, ax = plt.subplots(figsize=a4_dims)

df['total_demand'] = df['total_demand'].astype(float)
df['name_len'] = df['province'].apply(len)
df.sort_values('name_len', inplace=True)
bxplot = sns.boxplot(data=df, x='province', y='total_demand', order=plot_order)
tick_locs = range(df.province.nunique())
plt.xticks(ticks=tick_locs, labels=tick_locs)
plt.ylabel('Weekly total labor shortage', fontsize=15, fontweight='black', color = '#333F4B')
plt.xlabel('Province', fontsize=15, fontweight='black', color = '#333F4B')
plt.title('Labor shortage among Thai provinces', fontsize=15, fontweight='black', color = '#333F4B')    
plt.savefig('../../../plots/paper/demand_by_province_nolegend.jpg', bbox_inches='tight')
plt.show()
# # a4_dims = (16, 12)
# a4_dims = (12, 6)
# fig, ax = plt.subplots(figsize=a4_dims)

# df['demand_shock'] = df['demand_shock'].astype(float)
# bxplot = sns.boxplot(data=df, x='province', y='demand_shock', order=plot_order)
# for item in bxplot.get_xticklabels():
#     item.set_rotation(45)
#     item.set_fontsize(8) 
# plt.ylabel('Weekly total excess demand shock', fontsize=15, fontweight='black', color = '#333F4B')
# plt.xlabel('Province', fontsize=15, fontweight='black', color = '#333F4B')
# plt.title('Excess demand shocks for Thai provinces', fontsize=15, fontweight='black', color = '#333F4B')    
# plt.savefig('../../../plots/paper/demand_shock_by_province.jpg', bbox_inches='tight')
# plt.show()
# a4_dims = (16, 12)
a4_dims = (12, 6)
fig, ax = plt.subplots(figsize=a4_dims)

df['demand_shock'] = df['demand_shock'].astype(float)
bxplot = sns.boxplot(data=df, x='province', y='demand_shock', order=plot_order)
tick_locs = range(df.province.nunique())
plt.xticks(ticks=tick_locs, labels=tick_locs)
plt.ylabel('Weekly total labor shortage shock', fontsize=15, fontweight='black', color = '#333F4B')
plt.xlabel('Province', fontsize=15, fontweight='black', color = '#333F4B')
plt.title('Labor shortage shocks among Thai provinces', fontsize=15, fontweight='black', color = '#333F4B')    
plt.savefig('../../../plots/paper/demand_shock_by_province_nolegend.jpg', bbox_inches='tight')
plt.show()
plot_data = df.groupby('province').mean().sort_values(by='total_demand', ascending=False).reset_index()
plot_data['idx'] = plot_data.index
plot_data['legend'] = plot_data.apply(lambda row: str(row.idx)+ ' - '+str(row.province), axis=1)
print('\n'.join(plot_data.legend))
# print(plot_data.province.values)<jupyter_output>0 - Bangkok
1 - Samut Sakhon
2 - Samut Prakan
3 - Pathum Thani
4 - Chonburi
5 - Nakhon Pathom
6 - Nonthaburi
7 - Songkhla
8 - Phetchaburi
9 - Rayong
10 - Phuket
11 - Ratchaburi
12 - Phra Nakhon Si Ayutthaya
13 - Surat Thani
14 - Kanchanaburi
15 - Tak
16 - Chachoengsao
17 - Lopburi
18 - Nakhon Ratchasima
19 - Suphanburi
20 - Saraburi
21 - Prachuap Khiri Khan
22 - Nakhon Nayok
23 - Trang
24 - <NAME>
25 - Chumphon
26 - Krabi
27 - Phang Nga
28 - Nakhon <NAME>ammarat
29 - Khon Kaen
30 - <NAME>
31 - <NAME>
32 - Phetchabun
33 - Lampang
34 - Nak<NAME>
35 - Chanthaburi
36 - Phitsanulok
37 - Chaiyaphum
38 - Kamphaeng Phet
39 - Chiang Rai
40 - Nong Khai
41 - Phrae
42 - Chainat
43 - Ang Thong
44 - <NAME>
45 - Udon Thani
46 - Sukhothai
47 - Sing Buri
48 - Sa Kaeo
49 - Lamphun
50 - Trat
51 - <NAME>
52 - Uthai Thani
53 - Narathiwat
54 - Satun
55 - Ranong
56 - Kalasin
57 - Si Saket<jupyter_text>other shock periods<jupyter_code>wv_data_file = 'wv_cases1'
df = wv1
df['demand_shock'] = None
period_length = 8

for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 
    demand_data_points = df.loc[prov_row_idx, 'total_demand'].values
    std_demand = np.std(demand_data_points)
    df.loc[[0, 1], 'demand_shock'] = np.nan # need at least two points to fit a line
    
#     i = 0
    for time_pointer in range(2, len(prov_row_idx)):
        y = demand_data_points[max(0, time_pointer-period_length):time_pointer]
        expected_demand = np.mean(y)
        demand_shock = demand_data_points[time_pointer] - expected_demand
#         print(x, y, expected_demand, demand_shock)
        df.loc[prov_row_idx[time_pointer], 'demand_shock'] = demand_shock/std_demand

#         i+=1
#         if i>3: break

    
print(df.shape)


    # create placeholder
for offset in range(1, 9):
    df['demand_shock_plus_%s' % offset] = None
# shift IV the other way by up to 2 months    
for offset in range(1, 9):
    df['demand_shock_minus_%s' % offset] = None

for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 

    for offset in range(1, 9):
        df.loc[prov_row_idx, 'demand_shock_plus_%s' % offset] = df.loc[prov_row_idx, 'demand_shock'].shift(offset)
    # shift IV the other way by up to 2 months    
    for offset in range(1, 9):
        df.loc[prov_row_idx, 'demand_shock_minus_%s' % offset] = df.loc[prov_row_idx, 'demand_shock'].shift(-offset)

print(df.shape)
def yr_wk_to_float(yr_wk):
    yr, wk = yr_wk.split('-')
    return int(yr) + float(wk)/100

df['yr_wk_float'] = df.year_week.apply(yr_wk_to_float)
wv_data_file = 'wv_cases1'

df.to_csv(
    data_dir + 'regression_data_%s_causal_ma_detrend_8w_20210301.csv' % wv_data_file, 
    index=False)
df.head()
wv_data_file = 'wv_cases1'
df = wv1
df['demand_shock'] = None
period_length = 12

for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 
    demand_data_points = df.loc[prov_row_idx, 'total_demand'].values
    std_demand = np.std(demand_data_points)
    df.loc[[0, 1], 'demand_shock'] = np.nan # need at least two points to fit a line
    
#     i = 0
    for time_pointer in range(2, len(prov_row_idx)):
        y = demand_data_points[max(0, time_pointer-period_length):time_pointer]
        expected_demand = np.mean(y)
        demand_shock = demand_data_points[time_pointer] - expected_demand
#         print(x, y, expected_demand, demand_shock)
        df.loc[prov_row_idx[time_pointer], 'demand_shock'] = demand_shock/std_demand

#         i+=1
#         if i>3: break

    
print(df.shape)


    # create placeholder
for offset in range(1, 9):
    df['demand_shock_plus_%s' % offset] = None
# shift IV the other way by up to 2 months    
for offset in range(1, 9):
    df['demand_shock_minus_%s' % offset] = None

for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 

    for offset in range(1, 9):
        df.loc[prov_row_idx, 'demand_shock_plus_%s' % offset] = df.loc[prov_row_idx, 'demand_shock'].shift(offset)
    # shift IV the other way by up to 2 months    
    for offset in range(1, 9):
        df.loc[prov_row_idx, 'demand_shock_minus_%s' % offset] = df.loc[prov_row_idx, 'demand_shock'].shift(-offset)

print(df.shape)
def yr_wk_to_float(yr_wk):
    yr, wk = yr_wk.split('-')
    return int(yr) + float(wk)/100

df['yr_wk_float'] = df.year_week.apply(yr_wk_to_float)
wv_data_file = 'wv_cases1'

df.to_csv(
    data_dir + 'regression_data_%s_causal_ma_detrend_12w_20210301.csv' % wv_data_file, 
    index=False)
df.head()
wv_data_file = 'wv_cases1'
'regression_data_%s_causal_ma_detrend_12w_20210301.csv' % wv_data_file<jupyter_output><empty_output><jupyter_text>std of 12 weeks<jupyter_code>wv_data_file = 'wv_cases1'
df = wv1
df['demand_shock'] = None
period_length = 8
eps = 1e-6

for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 
    demand_data_points = df.loc[prov_row_idx, 'total_demand'].values
#     std_demand = np.std(demand_data_points)
    df.loc[[0, 1, 2, 3], 'demand_shock'] = np.nan # need at least two points to fit a line
    
#     i = 0
    for time_pointer in range(4, len(prov_row_idx)):
        y = demand_data_points[max(0, time_pointer-period_length):time_pointer]
        expected_demand = np.mean(y)
        demand_shock = demand_data_points[time_pointer] - expected_demand
        running_std_demand = np.std(y)
#         print(x, y, expected_demand, demand_shock)
        df.loc[prov_row_idx[time_pointer], 'demand_shock'] = demand_shock/(running_std_demand+eps)

#         i+=1
#         if i>3: break

    
print(df.shape)


    # create placeholder
for offset in range(1, 9):
    df['demand_shock_plus_%s' % offset] = None
# shift IV the other way by up to 2 months    
for offset in range(1, 9):
    df['demand_shock_minus_%s' % offset] = None

for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 

    for offset in range(1, 9):
        df.loc[prov_row_idx, 'demand_shock_plus_%s' % offset] = df.loc[prov_row_idx, 'demand_shock'].shift(offset)
    # shift IV the other way by up to 2 months    
    for offset in range(1, 9):
        df.loc[prov_row_idx, 'demand_shock_minus_%s' % offset] = df.loc[prov_row_idx, 'demand_shock'].shift(-offset)

print(df.shape)
def yr_wk_to_float(yr_wk):
    yr, wk = yr_wk.split('-')
    return int(yr) + float(wk)/100

df['yr_wk_float'] = df.year_week.apply(yr_wk_to_float)
wv_data_file = 'wv_cases1'

df.to_csv(
    data_dir + 'regression_data_%s_causal_ma_detrend_8w_std_20210217.csv' % wv_data_file, 
    index=False)
df.head()
df[pd.isnull(df['demand_shock'])]['year_week'].value_counts()
df[pd.isnull(df['demand_shock'])]['province'].value_counts()<jupyter_output><empty_output><jupyter_text>bigger provinces<jupyter_code>df = pd.read_csv(data_dir+'regression_data_wv_cases1_causal_ma_detrend_20210301.csv')
by_prov_counts = df.groupby('province').sum()['wv_count'].to_dict()
print(by_prov_counts)
lg_provs_40 = [prov for prov in by_prov_counts.keys() if by_prov_counts[prov]>=40]
df = df[df.province.isin(lg_provs_40)]
wv_data_file_lg_prov = 'regression_data_wv_cases1_causal_ma_detrend__province_40_20210301.csv'
df.to_csv(data_dir + wv_data_file_lg_prov)
df = pd.read_csv(data_dir+'regression_data_wv_cases1_causal_ma_detrend_20210301.csv')
# by_prov_counts = df.groupby('province').sum()['wv_count'].to_dict()
lg_provs_20 = [prov for prov in by_prov_counts.keys() if by_prov_counts[prov]>=20]
df = df[df.province.isin(lg_provs_20)]
wv_data_file_lg_prov = 'regression_data_wv_cases1_causal_ma_detrend__province_20_20210301.csv'
df.to_csv(data_dir + wv_data_file_lg_prov)
# by province
for prov in lg_provs_40:
    df[df.province==prov].to_csv(data_dir + 'regression_data_wv_cases1_causal_ma_detrend_%s_20210301.csv' % prov)
lg_provs_40
len(lg_provs_40)
df.head()<jupyter_output><empty_output><jupyter_text>extra iv labor abuse<jupyter_code>df = pd.read_csv(
        data_dir + 'regression_data_wv_cases1_causal_ma_detrend_20210301.csv')
print(df.shape)
df.head()
df['perc_abuse_minus_1'] = None 
df['perc_abuse_minus_2'] = None 
df['perc_abuse_minus_3'] = None 
df['perc_abuse_minus_4'] = None    
df['perc_abuse_minus_5'] = None   
df['perc_abuse_minus_6'] = None   
df['perc_abuse_minus_7'] = None   
for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 
    df.loc[prov_row_idx, 'perc_abuse_minus_1'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-1)
    df.loc[prov_row_idx, 'perc_abuse_minus_2'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-2)
    df.loc[prov_row_idx, 'perc_abuse_minus_3'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-3)
    df.loc[prov_row_idx, 'perc_abuse_minus_4'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-4)
    df.loc[prov_row_idx, 'perc_abuse_minus_5'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-5)
    df.loc[prov_row_idx, 'perc_abuse_minus_6'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-6)
    df.loc[prov_row_idx, 'perc_abuse_minus_7'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-7)
df.head()    
df[['province', 'perc_abuse', 'perc_abuse_minus_1','perc_abuse_minus_2','perc_abuse_minus_3','perc_abuse_minus_4','perc_abuse_minus_5']].\
     fillna(0).groupby('province').sum().head()
df.to_csv(
        data_dir + 'regression_data_%s_extra_iv_20210318.csv' % 'wv_cases1', 
        index=False)
data_dir + 'regression_data_%s_extra_iv_20210318.csv' % 'wv_cases1'<jupyter_output><empty_output><jupyter_text>other shock periods<jupyter_code>df = pd.read_csv(
        data_dir + 'regression_data_wv_cases1_causal_ma_detrend_8w_20210301.csv')
print(df.shape)
df['perc_abuse_minus_1'] = None 
df['perc_abuse_minus_2'] = None 
df['perc_abuse_minus_3'] = None 
df['perc_abuse_minus_4'] = None    
df['perc_abuse_minus_5'] = None   
df['perc_abuse_minus_6'] = None   
df['perc_abuse_minus_7'] = None   
for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 
    df.loc[prov_row_idx, 'perc_abuse_minus_1'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-1)
    df.loc[prov_row_idx, 'perc_abuse_minus_2'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-2)
    df.loc[prov_row_idx, 'perc_abuse_minus_3'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-3)
    df.loc[prov_row_idx, 'perc_abuse_minus_4'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-4)
    df.loc[prov_row_idx, 'perc_abuse_minus_5'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-5)
    df.loc[prov_row_idx, 'perc_abuse_minus_6'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-6)
    df.loc[prov_row_idx, 'perc_abuse_minus_7'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-7)
df.to_csv(
        data_dir + 'regression_data_wv_cases1_causal_extra_iv_8w_20210318.csv', 
        index=False)
df = pd.read_csv(
        data_dir + 'regression_data_wv_cases1_causal_ma_detrend_12w_20210301.csv')
print(df.shape)
df['perc_abuse_minus_1'] = None 
df['perc_abuse_minus_2'] = None 
df['perc_abuse_minus_3'] = None 
df['perc_abuse_minus_4'] = None    
df['perc_abuse_minus_5'] = None   
df['perc_abuse_minus_6'] = None   
df['perc_abuse_minus_7'] = None   
for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 
    df.loc[prov_row_idx, 'perc_abuse_minus_1'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-1)
    df.loc[prov_row_idx, 'perc_abuse_minus_2'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-2)
    df.loc[prov_row_idx, 'perc_abuse_minus_3'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-3)
    df.loc[prov_row_idx, 'perc_abuse_minus_4'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-4)
    df.loc[prov_row_idx, 'perc_abuse_minus_5'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-5)
    df.loc[prov_row_idx, 'perc_abuse_minus_6'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-6)
    df.loc[prov_row_idx, 'perc_abuse_minus_7'] = df.loc[prov_row_idx, 'perc_abuse'].shift(-7)
df.to_csv(
        data_dir + 'regression_data_wv_cases1_causal_extra_iv_12w_20210318.csv', 
        index=False)<jupyter_output>(6554, 60)<jupyter_text>sandbox<jupyter_code>wv_data_file = 'wv_cases1'
df = wv1
df['demand_shock'] = None

for prov in df.province.unique():
    prov_row_idx = df[df.province==prov].index 
    demand_data_points = df.loc[prov_row_idx, 'total_demand'].values
    std_demand = np.std(demand_data_points)
    df.loc[[0, 1], 'demand_shock'] = np.nan # need at least two points to fit a line
    
    i = 0
    for time_pointer in range(2, len(prov_row_idx)):
        y = demand_data_points[max(0, time_pointer-4):time_pointer]
        expected_demand = np.mean(y)
        demand_shock = demand_data_points[time_pointer] - expected_demand
#         print(x, y, expected_demand, demand_shock)
        df.loc[prov_row_idx[time_pointer], 'demand_shock'] = demand_shock/std_demand
        import pdb; pdb.set_trace()
        i+=1
        if i>3: break

    
print(df.shape)<jupyter_output>> <ipython-input-10-c68b0ace961a>(19)<module>()
-> i+=1
(Pdb) n
> <ipython-input-10-c68b0ace961a>(20)<module>()
-> if i>3: break
(Pdb) n
> <ipython-input-10-c68b0ace961a>(12)<module>()
-> for time_pointer in range(2, len(prov_row_idx)):
(Pdb) y
array([0., 0.])
(Pdb) expected_demand
0.0
(Pdb) demand_shock
0.0
(Pdb) n
> <ipython-input-10-c68b0ace961a>(13)<module>()
-> y = demand_data_points[max(0, time_pointer-4):time_pointer]
(Pdb) n
> <ipython-input-10-c68b0ace961a>(14)<module>()
-> expected_demand = np.mean(y)
(Pdb) n
> <ipython-input-10-c68b0ace961a>(15)<module>()
-> demand_shock = demand_data_points[time_pointer] - expected_demand
(Pdb) n
> <ipython-input-10-c68b0ace961a>(17)<module>()
-> df.loc[prov_row_idx[time_pointer], 'demand_shock'] = demand_shock/std_demand
(Pdb) n
> <ipython-input-10-c68b0ace961a>(18)<module>()
-> import pdb; pdb.set_trace()
(Pdb) demand_shock
10.0
(Pdb) expected_demand
0.0
(Pdb) n
> <ipython-input-10-c68b0ace961a>(19)<module>()
-> i+=1
(Pdb) n
> <ipython-in[...]
{'fraction_non_alphanumeric': 0.14922940820412534, 'fraction_numerical': 0.03641104437305024, 'mean_word_length': 4.72593974940016, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '505812', 'license': 'MIT', 'n_tokens_mistral': 9386, 'n_tokens_neox': 8716, 'n_words': 1465, 'path': 'notebooks/data_cleaning_w_2020/5-1. Detrend demand w industry - 0520.ipynb', 'repo_name': 'lbyiuou0329/Thai-Labor-Abuse-Project-Code', 'size': '21506'}