<jupyter_start><jupyter_text>Rather than having one X entry for each day, tune the code so that there can be one X entry for each location on each day. This will be put the data in a form that is convenient for feature engineering.<jupyter_code>import os
os.chdir('..')
import solar.wrangle.wrangle
from solar.wrangle.wrangle import SolarData
reload(solar.wrangle.wrangle)
reload(solar.wrangle.subset)
from solar.wrangle.wrangle import SolarData
from solar.wrangle.subset import Subset

station = ['ACME', 'BEAV']
train_dates = ['2005-07-29','2005-08-29']
test_dates = ['2008-12-29', '2009-01-29']
#var = ['dswrf_sfc', 'uswrf_sfc', 'spfh_2m', 'ulwrf_tatm']
var = ['dswrf_sfc']
models = [0]
times = [18]
lats = range(37,38)
longs = range(265,268)
new_feat = {'grid'}
X_par = {'train_dates':train_dates, 'test_dates':test_dates, 'var':var, 'models':models, 'times':times, 'lats':lats,
        'longs':longs}
y_par = {'station':station, 'train_dates':train_dates, 'lats':lats, 'longs':longs}
trainX, trainy, testX = SolarData.load('solar/data/kaggle_solar/train/', 'solar/data/kaggle_solar/train.csv', 
                                'solar/data/kaggle_solar/test', 'solar/data/kaggle_solar/station_info.csv', 
                                new_feat, station=station, train_dates=train_dates,
                                test_dates=test_dates, lats=lats, longs=longs, var=var, models=models, times=times)
import solar.wrangle.engineer
reload(solar.wrangle.engineer)
reload(solar.wrangle.subset)
reload(solar.wrangle.wrangle)
from solar.wrangle.engineer import Engineer

station = ['ACME', 'BEAV']
train_dates = ['2005-07-29','2005-08-29']
test_dates = ['2008-12-29', '2009-01-29']
var = ['dswrf_sfc', 'uswrf_sfc', 'spfh_2m', 'ulwrf_tatm']
#var = ['dswrf_sfc']
models = [0,1]
times = [18,21]
lats = range(37,38)
longs = range(265,268)
X_par = {'train_dates':train_dates, 'test_dates':test_dates, 'var':var, 'models':models, 'times':times, 'lats':lats,
        'longs':longs}
y_par = {'station':station, 'train_dates':train_dates, 'lats':lats, 'longs':longs}
new_feats = {'grid'}
trainX, trainy, testX, locy = Engineer.engineer('solar/data/kaggle_solar/train/', 'solar/data/kaggle_solar/train.csv', 
                                'solar/data/kaggle_solar/test/', 'solar/data/kaggle_solar/station_info.csv',
                                X_par, y_par, new_feats)
trainy
import solar.wrangle.wrangle
import solar.wrangle.engineer
import solar.wrangle.subset
reload(solar.wrangle.wrangle)
reload(solar.wrangle.engineer)
reload(solar.wrangle.subset)

from solar.wrangle.wrangle import SolarData
from solar.wrangle.engineer import Engineer

station = ['ACME', 'BEAV']
#station = ['all']
train_dates = ['2005-01-01','2005-02-01']
test_dates = ['2008-01-01', '2008-02-01']
#train_dates = []
#test_dates = []
var = ['dswrf_sfc', 'uswrf_sfc', 'spfh_2m', 'ulwrf_tatm']
#var = ['all']
#models = [0]
#times = [21]

models = []
times = []
#lats = range(37,40)
#longs = range(265,270)
lats = []
longs = []
new_features={'grid'}
Xparams, Yparams, station = SolarData.parse_parameters(station=station, train_dates=train_dates, test_dates=test_dates, \
                                                      var=var, lats=lats, longs=longs, models=models, times=times)
%prun input_data = Engineer.engineer('solar/data/kaggle_solar/train/', 'solar/data/kaggle_solar/train.csv', \
                                    'solar/data/kaggle_solar/test', 'solar/data/kaggle_solar/station_info.csv', \
                                    Xparams, Yparams, new_features)
import netCDF4 as nc
X = nc.Dataset('solar/data/kaggle_solar/train/dswrf_sfc_latlon_subset_19940101_20071231.nc','r+').variables.values()
X[-1][4018,0,0,1,8]
input_data[0]
X[-1][4018,0,2,3,6]
reload(solar.wrangle.wrangle)
reload(solar.wrangle.engineer)
reload(solar.wrangle.subset)

from solar.wrangle.wrangle import SolarData
from solar.wrangle.engineer import Engineer

station = ['ACME', 'BEAV']
#station = ['all']
train_dates = ['2005-01-01','2005-02-01']
test_dates = ['2008-01-01', '2008-02-01']
#train_dates = []
#test_dates = []
var = ['dswrf_sfc', 'uswrf_sfc', 'spfh_2m', 'ulwrf_tatm']
#var = ['all']
#models = [0]
#times = [21]

models = []
times = []
#lats = range(37,40)
#longs = range(265,270)
lats = []
longs = []
new_feats = {'grid'}
%prun input_data = SolarData.load('solar/data/kaggle_solar/train/', 'solar/data/kaggle_solar/train.csv', \
                                  'solar/data/kaggle_solar/test', 'solar/data/kaggle_solar/station_info.csv', \
                                  new_feats, station=station, train_dates=train_dates, test_dates=test_dates, \
                                  var=var, lats=lats, longs=longs, models=models, times=times)

input_data[0]
X[-1][4018,0,2,3,6]
input_data[1].columns
input_data[1].unstack('location').reset_index().set_index(['date','location']).sort_index()[0]
import numpy as np
np.stack()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16372847011144884, 'fraction_numerical': 0.047821681864235056, 'mean_word_length': 3.9908998988877653, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '316750', 'license': 'MIT', 'n_tokens_mistral': 2016, 'n_tokens_neox': 1864, 'n_words': 318, 'path': 'notebooks/ExpandX.ipynb', 'repo_name': 'joelmpiper/ga_project', 'size': '4935'}