<jupyter_start><jupyter_text>To download files from Kaggle register a user first and create a file in your homedirectory `.kaggle/kaggle.json` with content `{"username":"YOURUSERNAME","key":"YOURKEY"}`<jupyter_code>import kaggle

kaggle.api.authenticate()
kaggle.api.dataset_download_files('pratyushakar/rossmann-store-sales', path='../data/raw', unzip=True)<jupyter_output><empty_output><jupyter_text>Data cleansing and basic feature engineering<jupyter_code>stores = pd.read_csv('../data/raw/store.csv')
df = (pd.read_csv('../data/raw/train.csv', infer_datetime_format=True, parse_dates=['Date'])
         .assign(StateHoliday=lambda df: df.StateHoliday.map({'a': 1, 'b': 2, 'c': 3, '0': 0, 0: 0}))
         .merge(stores, how='left', on='Store')
         .sort_values(['Store', 'Date'])
         .assign(Assortment=lambda df: df.Assortment.map({'a': 1, 'b': 2, 'c': 3}))
         .assign(StoreType=lambda df: df.StoreType.map({'a': 1, 'b': 2, 'c': 3, 'd': 4}))
         .assign(StoreId=lambda df: df.groupby(['Store']).ngroup()) # make IDs contiguous
         .assign(Timestep=lambda df: df.groupby(['Store'])['Date'].cumcount())
         .assign(StoreVariant=lambda df: df['StoreType']*10 + df['Assortment'])
         .drop([c for c in stores.columns if c.startswith('Competition')], axis=1)
         .drop([c for c in stores.columns if c.startswith('Promo2S')], axis=1)
         .drop(['PromoInterval', 'StoreType', 'Assortment'], axis=1)
     )
# Use this to split stores up into 'virtual' smaller stores just to demonstrate the scalability of this approach
# df = (df.groupby('Store', as_index=False).apply(lambda df: df.assign(part=gen_partitions(len(df), 7, 22)))
#         .reset_index(drop=True)
#         .query('part >= 0')
#         .assign(Store=lambda x: x.Store*1000+x.part)
#         .drop(['part'], axis=1)
#         .sort_values(['StoreId', 'Date'])
#         .reset_index(drop=True))<jupyter_output><empty_output><jupyter_text>Dummy encoding of categorical values<jupyter_code>edf = encode(df.drop(columns='Customers'), ['DayOfWeek', 'StoreVariant', 'StateHoliday'], drop_first=False)
edf = edf.drop(columns=['Date', 'Store'])
edf.loc[edf['Open'] == 0, 'Sales'] = np.nan
edf = edf.drop(columns='Open')  # Keep the open column
edf = reorder_cols(edf, first=['StoreId', 'Timestep'], last='Sales')
edf.to_csv('../data/preprocessed/edf.csv', index=False)<jupyter_output><empty_output><jupyter_text>Generate a multi-dimensional array for input<jupyter_code>X = make_cube(edf, ['StoreId', 'Timestep'])
X_train, X_test = X[:1000, ...], X[1000:, ...]
np.savez('../data/preprocessed/X_train.npz', X_train)
np.savez('../data/preprocessed/X_test.npz', X_test)
df.to_csv('../data/result/df.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19184564979978158, 'fraction_numerical': 0.01383327266108482, 'mean_word_length': 4.846808510638298, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '644751', 'license': 'MIT', 'n_tokens_mistral': 1034, 'n_tokens_neox': 984, 'n_words': 186, 'path': 'notebooks/01-preprocessing.ipynb', 'repo_name': 'arita37/bhm-at-scale', 'size': '2747'}