<jupyter_start><jupyter_text>Go up one level to access all of the scripts from the main director<jupyter_code>import requests

# The direct link to the Kaggle data set
data_url = 'http://www.kaggle.com/c/digit-recognizer/download/train.csv'

# The local path where the data set is saved.
local_filename = "train.csv"

# Kaggle Username and Password
kaggle_info = {'UserName': "my_username", 'Password': "<PASSWORD>"}

# Attempts to download the CSV file. Gets rejected because we are not logged in.
r = requests.get(data_url)

# Login to Kaggle and retrieve the data.
r = requests.post(r.url, data = kaggle_info, prefetch = False)

# Writes the data to a local file one chunk at a time.
f = open(local_filename, 'w')
for chunk in r.iter_content(chunk_size = 512 * 1024): # Reads 512KB at a time into memory
    if chunk: # filter out keep-alive new chunks
        f.write(chunk)
f.close()
import os
os.chdir('..')<jupyter_output><empty_output><jupyter_text>Create some simple X, y training values<jupyter_code>reload(solar.wrangle.wrangle)
reload(solar.wrangle.subset)

from solar.wrangle.wrangle import SolarData
from solar.wrangle.subset import Subset
station = ['ACME', 'BEAV']
train_dates = ['2005-07-29','2005-08-29']
test_dates = ['2008-12-29', '2009-01-29']
#var = ['dswrf_sfc', 'uswrf_sfc', 'spfh_2m', 'ulwrf_tatm']
var = ['dswrf_sfc']
models = [0]
times = [18]
lats = range(37,38)
longs = range(265,268)
input_data = SolarData.load('solar/data/kaggle_solar/train/', 'solar/data/kaggle_solar/train.csv', 
                                'solar/data/kaggle_solar/test', 'solar/data/kaggle_solar/station_info.csv',
                                var=var, lats=lats, longs=longs, test_dates=test_dates, train_dates=train_dates,
                                models=models, times=times, station=station)
trainX = input_data[0]
trainy = input_data[1]
trainy
import solar.analyze.model
from solar.analyze.model import Model
reload(solar.analyze.model)
from solar.analyze.model import Model

from sklearn.linear_model import Ridge
from sklearn import metrics

error_formula = 'mean_absolute_error'

trainX, trainy, model = Model.model(input_data, Ridge, {'alpha':[0.2,0.3]}, 10, 
                    error_formula, normalize=True)
import numpy as np
from sklearn.neighbors import KNeighborsRegressor
trainX, trainy, model = Model.model(input_data, KNeighborsRegressor, 
                                    {'n_neighbors':np.arange(1,10)}, 10, 
                                    error_formula)
model.fit(trainX,trainy)
model.best_score_
model.predict(trainX.iloc[20:21,:])
print trainX.shape
print trainy.shape
print model.predict(trainX.iloc[0:1,:])
import datetime
strftime("%Y%m%d-%H%M%S")
reload(solar.wrangle.wrangle)
reload(solar.wrangle.subset)

from solar.wrangle.wrangle import SolarData
#station = ['ACME', 'BEAV']
station = ['all']
#train_dates = ['2005-07-29','2005-08-29']
train_dates = []
#test_dates = ['2008-12-29', '2009-01-29']
test_dates = []
#var = ['dswrf_sfc', 'uswrf_sfc', 'spfh_2m', 'ulwrf_tatm']
var = ['dswrf_sfc']
models = [0]
#times = [18]
times = []
#lats = range(37,38)
lats = []
#longs = range(265,268)
longs = []
trainX, trainy, testX = SolarData.load('solar/data/kaggle_solar/train/', 'solar/data/kaggle_solar/train.csv', 
                                'solar/data/kaggle_solar/test', 'solar/data/kaggle_solar/station_info.csv',
                                var=var, lats=lats, longs=longs, test_dates=test_dates, train_dates=train_dates,
                                models=models, times=times, station=station)
reload(solar.analyze.model)
from solar.analyze.model import Model

from sklearn.linear_model import Ridge
from sklearn import metrics

error_formula = 'mean_absolute_error'

model = Model.model([trainX,trainy,testX], Ridge, {'alpha':np.logspace(-3,1,8,base=10)}, 10, 
                    error_formula, normalize=True)
reload(solar.report.submission)
from solar.report.submission import Submission

preds = Submission.make_submission_file(model, trainy, testX)
preds.shape
model.best_score_
import cPickle as pickle
model = pickle.load(open('solar/data/kaggle_solar/model.p','rb'))
model.grid_scores_
import solar.wrangle.wrangle
import solar.wrangle.subset
import solar.wrangle.engineer
import solar.analyze.model
reload(solar.wrangle.wrangle)
reload(solar.wrangle.subset)
reload(solar.wrangle.engineer)

from solar.wrangle.wrangle import SolarData
from solar.wrangle.subset import Subset
#station = ['ACME', 'BEAV']
station = ['all']
#train_dates = ['2005-07-29','2005-08-29']
#test_dates = ['2008-12-29', '2009-01-29']
train_dates = []
test_dates = []
#var = ['dswrf_sfc', 'uswrf_sfc', 'spfh_2m', 'ulwrf_tatm']
#var = ['dswrf_sfc']
var = ['all']
models = [0,1]
#times = [18]
#models = []
times = []
#lats = range(37,38)
#longs = range(265,268)
lats=[]
longs=[]
new_feats = {'grid'}
%prun input_data = SolarData.load('solar/data/kaggle_solar/train/', 'solar/data/kaggle_solar/train.csv', \
                                'solar/data/kaggle_solar/test', 'solar/data/kaggle_solar/station_info.csv', new_feats, \
                                var=var, lats=lats, longs=longs, test_dates=test_dates, train_dates=train_dates, \
                                models=models, times=times, station=station)
reload(solar.analyze.model)
import numpy as np
from solar.analyze.model import Model

from sklearn.linear_model import Ridge
from sklearn import metrics

error_formula = 'mean_absolute_error'

model = Model.model(input_data, Ridge, {'alpha':np.logspace(-3,1,8,base=10)}, 10, 
                    error_formula, normalize=True)
model.best_score_
import solar.report.submission
input_data[2].unstack('station').shape
reload(solar.report.submission)
from solar.report.submission import Submission

preds = Submission.make_submission_file(model, input_data[1], input_data[2], {'grid'})
preds.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13487344997452014, 'fraction_numerical': 0.03142517411245117, 'mean_word_length': 4.032478632478632, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '275959', 'license': 'MIT', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2081, 'n_words': 429, 'path': 'notebooks/TestModel.ipynb', 'repo_name': 'joelmpiper/ga_project', 'size': '5888'}