| #!/usr/bin/env python | |
| # coding: utf-8 | |
| # In[ ]: | |
| from numerapi import NumerAPI | |
| import os | |
| import pandas as pd | |
| import numpy as np | |
| # In[ ]: | |
| napi = NumerAPI() | |
| data_path='./data/' | |
| # In[ ]: | |
| napi.download_dataset("v4.2/validation_int8.parquet", data_path+"validation_int8.parquet") | |
| validation_data=pd.read_parquet(data_path+"validation_int8.parquet") | |
| recent_eras=list(validation_data.loc[validation_data['data_type']=='validation']['era'].unique()[-2:]) | |
| validation_subset=validation_data[validation_data['era'].isin(recent_eras)] | |
| validation_subset.to_parquet(data_path+"validation_subset_int8.parquet",index=False) | |
| # napi.download_dataset('v4.2/live_int8.parquet',data_path+'live_int8.parquet') | |
| # In[ ]: | |
| # print("Now please copy the file to server via: scp ../../data/validation_subset_int8.parquet gms@gms1:/home/gms/numerai/data/.") | |