<jupyter_start><jupyter_text>Visual sanity check<jupyter_code>#page = '(236984)_Astier_fr.wikipedia.org_all-access_all-agents'
#page = '龍抬頭_zh.wikipedia.org_mobile-web_all-agents'
#page = "'Tis_the_Season_(Vince_Gill_and_Olivia_Newton-John_album)_en.wikipedia.org_mobile-web_all-agents"
#page='Peter_Townsend_(RAF_officer)_en.wikipedia.org_mobile-web_all-agents'
page="Heahmund_en.wikipedia.org_desktop_all-agents"
prev.loc[page].fillna(0).plot(logy=True)

#gt.loc[page].fillna(0).plot(logy=True)
f_preds.loc[page].plot(logy=True)
def read_keys():
    import os.path
    key_file = 'data/keys2.pkl'
    if os.path.exists(key_file):
        return pd.read_pickle(key_file)
    else:
        print('Reading keys...')
        raw_keys = pd.read_csv('data/key_2.csv.zip')
        print('Processing keys...')
        pagedate = raw_keys.Page.str.rsplit('_', expand=True, n=1).rename(columns={0:'page',1:'date_str'})
        keys = raw_keys.drop('Page', axis=1).assign(page=pagedate.page, date=pd.to_datetime(pagedate.date_str))
        del raw_keys, pagedate
        print('Pivoting keys...')
        pkeys = keys.pivot(index='page', columns='date', values='Id')
        print('Storing keys...')
        pkeys.to_pickle(key_file)
        return pkeys
keys = read_keys()    
subm_preds = f_preds.loc[:, '2017-09-13':]
assert np.all(subm_preds.index == keys.index)
assert np.all(subm_preds.columns == keys.columns)
answers = pd.DataFrame({'Id':keys.values.flatten(), 'Visits':np.round(subm_preds).astype(np.int64).values.flatten()})
answers.to_csv('data/submission.csv.gz', compression='gzip', index=False, header=True)

f_preds
prev.loc[missing_pages, '2016-12-15':]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16745562130177516, 'fraction_numerical': 0.01893491124260355, 'mean_word_length': 4.85121107266436, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '223153', 'license': 'MIT', 'n_tokens_mistral': 683, 'n_tokens_neox': 656, 'n_words': 73, 'path': 'submission-final.ipynb', 'repo_name': 'tatamiQ/kaggle-web-traffic', 'size': '1690'}