<jupyter_start><jupyter_text>Reading all the the optimized hyperparameter .csv files and collating them into the Locations.csv file Import the necessary packages<jupyter_code>import geopandas as gp
import pandas as pd

import optuna
from statsmodels.tsa.statespace.sarimax import SARIMAX
import numpy as np
import sklearn
from sklearn.model_selection import TimeSeriesSplit
from sklearn.metrics import r2_score

import os

from hyperparam_opt import *
pwd<jupyter_output><empty_output><jupyter_text>Sort the hyperparameter results files so that they are in ascending order.<jupyter_code>arr = os.listdir('../../hyperparameters_2/')

def filenumber(x):
    x = os.path.splitext(x)[0]
    return(int(x[31:]))

arr = sorted(arr, key = filenumber) 
arr
# appending the filepath to the filename.
for k in range(len(arr)):
    arr[k] = '../../hyperparameters_2/'+ arr[k]
arr
df = pd.read_csv(arr[0])
df = df.sort_values('value',ascending=False)
df.head(1)<jupyter_output><empty_output><jupyter_text>Extracting the best performing hyperparameters from the .csv files for each location. format = [D, P, Q, d, p, q]<jupyter_code>hyperparam=[]

for k in range(len(arr)):
    df = pd.read_csv(arr[k])
    df = df.sort_values('value',ascending=False)
    hyperparam.append(df[['params_D','params_P','params_Q','params_d','params_p','params_q']].head(1).values[0])
hyperparam = np.array(hyperparam)
hyperparam = np.array(hyperparam)
hyperparam<jupyter_output><empty_output><jupyter_text>Writing the hyperparametes to the locations.csv file<jupyter_code>dfloc = pd.read_csv('../../Locations.csv')
dfloc['p'] = hyperparam[:,4]
dfloc['d'] = hyperparam[:,3]
dfloc['q'] = hyperparam[:,5]
dfloc['P'] = hyperparam[:,1]
dfloc['D'] = hyperparam[:,0]
dfloc['Q'] = hyperparam[:,2]
dfloc
dfloc.to_csv('../../hyperparameters_2/locations_2.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14502164502164502, 'fraction_numerical': 0.00974025974025974, 'mean_word_length': 6.2795275590551185, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '270217', 'license': 'MIT', 'n_tokens_mistral': 666, 'n_tokens_neox': 661, 'n_words': 146, 'path': 'serf/working/write_opt_hyperparam_to_location.ipynb', 'repo_name': 'Solar-Wind-ML-DIRECT-2022-Project/2022-Project-Repo', 'size': '1848'}