<jupyter_start><jupyter_text>This notebook is for reconstructing solar. It explores using an extra eigenvector to reconstruct data, based on the predicted solar without cloud cover. This notebook reconstructs 2007 data after being trained on 2007 data.  Importing Training Data<jupyter_code>p1 = '/results/forcing/atmospheric/GEM2.5/gemlam/gemlam_y2007m01d03.nc'
d1 = xr.open_dataset(p1)
list(d1.keys())
myFiles = glob.glob('/home/arandhawa/*') # returns list
myFiles.sort()
myFiles<jupyter_output><empty_output><jupyter_text>| Description | HRDPS       | CANRCM      | | ----------- | ----------- | ----------- || Near-Surface Air Temperature | tair | tas || Precipitation | precip | pr || Sea Level Pressure | atmpres | psl || Near Surface Specific Humidity | qair | huss || Shortwave radiation | solar | rsds || Longwave radiation | therm_rad | rlds |<jupyter_code>variables = [['tair', 'tas', 'Near-Surface Air Temperature'], 
             ['precip', 'pr', 'Precipitation'], 
             ['atmpres', 'psl', 'Sea Level Pressure'], 
             ['qair', 'huss', 'Near Surface Specific Humidity'], 
             ['solar', 'rsds', 'Shortwave radiation'], 
             ['therm_rad', 'rlds', 'Longwave radiation'],
             ['u_wind', 'uas', 'u-winds'], 
             ['v_wind', 'vas', 'vas']]
def gen_solar(t, day):
    day_time = ((3*t - 8)%24 )*3600 ##converting 3 hourly UTC to seconds from midnight PST
    latitude = 49.10509             ##latitude of salish sea
    
    hour = (day_time / 3600.0 - 12.0) * 15.0  ##degrees
    declination = 23.45*np.pi/180.0*np.sin((284.0 + day)/365.25*2.0*np.pi)  ##radians
    
    ## Convert latitude of centre of model domain from degrees to radians
    lat = np.pi * latitude / 180.0
    
    a = np.sin(declination) * np.sin(lat) 
    b = np.cos(declination) * np.cos(lat)
    cos_Z = a + b * np.cos(np.pi / 180.0 * hour)      ##solar elevation
    hour_angle = np.tan(lat)*np.tan(declination)  ## cos of -hour_angle in radians
    if (hour_angle > 1):   ## so far North in summer that there is no night
       day_length = 24 + 0.0000001
    elif (hour_angle < -1): ## so far North in winter that there is no day
       day_length = -0.0000001
    else:
       day_length =  np.arccos(-hour_angle) / 15.0 * 2.0 * 180.0 / np.pi ## hours: 15 = 360/24
    
    sunrise = 12.0 - 0.5 * day_length  ##hours
    sunset = 12.0 + 0.5 * day_length   ##hours
    
    Q_o = 1368.0     ## W/m^2 Solar constant
    Qso = Q_o*(1.0+0.033*np.cos(day/365.25*2.0*np.pi))
    
    if day_time / 3600.0 > sunrise and day_time / 3600.0 < sunset:
       I_incident = Qso * cos_Z   
    else:
       I_incident = 0
    
    return I_incident

solar_eig = np.zeros(363*8)
i = 0
for day in range(3, 365):
    for t in range(8):
        solar_eig[i] = gen_solar(t, day)
        i += 1
plt.plot(solar_eig)
var = variables[4]
name = var[2]   ##for graph titles - name
data_name_hr = var[0]    ##variable of HRDPS data to work with
data_name_can = var[1] ##variable of CANRCM data to work with
p_can07 = '/home/arandhawa/canrcm_' + var[1] + '_2007.nc'  
##2007 HRDPS import

files = glob.glob('/results/forcing/atmospheric/GEM2.5/gemlam/gemlam_y2007m??d??.nc')
##list of data files, febrary 2007 - each file covers one day, with hourly data
##the grid is 266 by 256 points, with resolution 2.5km
##relevant data in this file is u_wind and v_wind data
files.sort()
print(len(files))

## 3-hour averaged matrix

hr07 = np.empty( (8*len(files), 266, 256)) 

for i in range(len(files)):
    dayX = xr.open_dataset(files[i])
    ##adding 1 day of 3-hour averages to new data array
    hr07[8*i:8*i + 8,:,:] = np.array( dayX[ data_name_hr ] ).reshape(8, 3, 266, 256).mean(axis = 1) 

del dayX

print(hr07.shape)
##CANRCM 2007 import
d1 = xr.open_dataset(p_can07)
can07 = d1[data_name_can][16:,140:165,60:85] ##the first two days are removed to be consistent with 2007 HRDPS

print(can07.shape)
print(d1.coords['time'][1])<jupyter_output><xarray.DataArray 'time' ()>
array(cftime.DatetimeNoLeap(2007, 1, 1, 6, 0, 0, 0, has_year_zero=True),
      dtype=object)
Coordinates:
    time     object 2007-01-01 06:00:00
Attributes:
    long_name:      time
    standard_name:  time
    axis:           T
    bounds:         time_bnds<jupyter_text>PCA Functions<jupyter_code>##transforms and concatenates two data sets
def transform2(data1, data2):
    A_mat = transform(data1)
    B_mat = transform(data2)
    return np.concatenate((A_mat, B_mat), axis=0) 
##inverse function of transform2 - splits data matrix and returns two data sets
def reverse2(matrix, orig_shape):
    split4 = int( matrix.shape[0]/2 )
    u_data = reverse(matrix[:split4,:], orig_shape) ##reconstructing u_winds from n PCs
    v_data = reverse(matrix[split4:,:], orig_shape) ##reconstructing v_winds from n PCs
    return (u_data, v_data)
##performs PCA analysis using sklearn.pca
def doPCA(comp, matrix):
    pca = PCA(n_components = comp) ##adjust the number of principle conponents to be calculated
    PCs = pca.fit_transform(matrix)
    eigvecs = pca.components_
    mean = pca.mean_
    return (PCs, eigvecs, mean)
##data must be converted into a 2D matrix for pca analysis
##transform takes a 3D data array (time, a, b) -> (a*b, time)
##(the data grid is flattened a column using numpy.flatten)

def transform(xarr):
    arr = np.array(xarr) ##converting to numpy array
    arr = arr.reshape(arr.shape[0], arr.shape[1]*arr.shape[2]) ##reshaping from size (a, b, c) to (a, b*c)
    arr = arr.transpose()
    return arr

def reverse(mat, orig_shape):
    arr = np.copy(mat)
    arr = arr.transpose()
    arr = arr.reshape(-1, orig_shape[1], orig_shape[2]) ##reshaping back to original array shape
    return arr
##graphing percentage of original data represented by the first n principle conponents
def graph_variance(matrix, n):
    pcaG = PCA(n_components = n) ##Number of principle conponents to show
    PCsG = pcaG.fit_transform(matrix)
    plt.plot(np.cumsum(pcaG.explained_variance_ratio_))
    plt.xlabel('number of components')
    plt.ylabel('cumulative explained variance');
    plt.show()
    del pcaG
    del PCsG
##can be used to visualize principle conponents for u/v winds
def graph_nPCs(PCs, eigvecs, n, orig_shape):
    fig, ax = plt.subplots(n, 3, figsize=(10, 3*n))
    
    ax[0, 0].set_title("u-conponent")
    ax[0, 1].set_title("v-component")
    ax[0, 2].set_title("time-loadings")
    
    for i in range(n):
        mode_u, mode_v = get_mode(PCs, i, orig_shape)
        colors = ax[i, 0].pcolormesh(mode_u, cmap = 'bwr')
        fig.colorbar(colors, ax = ax[i,0])
        colors = ax[i, 1].pcolormesh(mode_v, cmap = 'bwr')
        fig.colorbar(colors, ax = ax[i,1])
        ax[i, 2].plot(eigvecs[i])

    plt.tight_layout()
    plt.show()

##converts PCs (column vectors) to 2d conpoents for u and v wind
def get_mode(PCs, n, orig_shape): 
    split = int(PCs.shape[0]/2)
    mode_u = PCs[:split, n].reshape(orig_shape[1], orig_shape[2])
    mode_v = PCs[split:, n].reshape(orig_shape[1], orig_shape[2])
    return (mode_u, mode_v)<jupyter_output><empty_output><jupyter_text>PCAs on Training Data<jupyter_code>##PCA on CANRCM 2007
can07_mat = transform(can07)
can07_PCs, can07_eigs, can07_mean = doPCA(100, can07_mat)

##PCA on HRDPS 2007
hr07_mat = transform(hr07)
hr07_PCs, hr07_eigs, hr07_mean = doPCA(100, hr07_mat)
## combining the eigenvectors and mean together in one array for analysis
## the mean is the average value at each timestamp (average row) - behaves as a zeroth eigenvector
can07_me = np.concatenate((can07_mean.reshape(1, -1), can07_eigs))
hr07_me = np.concatenate((hr07_mean.reshape(1, -1), hr07_eigs))<jupyter_output><empty_output><jupyter_text>Multiple Linear Regression Functions<jupyter_code>##functions that use multiple linear regression to fit eigenvectors
##takes CANRCM eigenvectors (x1, x2, x3, x4...) and HRDPS eigenvectors (y1, y2, y3...)
##For each y from 0:result_size, approximates yn = a0 + a1*x1 + a2*x2 + a3*x3 ... using num_vec x's
##getCoefs returns (coeficients, intercept)
##fit_modes returns each approximation and the R^2 value of each fit as (results, scores)

def getCoefs(vectors, num_vec, data, num_modes, type = 'LS'):  
    
    X = vectors[0:num_vec,:].T
    coefs = np.zeros((num_modes, X.shape[1]))
    intercept = np.zeros(num_modes)
    
    if type == 'LS':
        for i in range(num_modes):
            y = data[i,:]
            reg = LinearRegression().fit(X, y)
            coefs[i] = reg.coef_[0:num_vec]
            intercept[i] =  reg.intercept_
    elif type == 'MAE':
        for i in range(num_modes):
            y = data[i,:]
            reg = QuantileRegressor(quantile = 0.5, alpha = 0, solver = 'highs').fit(X, y)
            coefs[i] = reg.coef_[0:num_vec]
            intercept[i] =  reg.intercept_
    
    return (coefs, intercept)


def fit_modes(vectors, num_vec, data, result_size, type = 'LS'):  
    
    X = vectors[0:num_vec,:].T
    result = np.zeros((result_size, X.shape[0]))
    scores = np.zeros(result_size)
    
    if type == 'LS':
        for i in range(result_size):
            y = data[i,:]
            reg = LinearRegression().fit(X, y)
            result[i] = reg.predict(X)
            scores[i] = reg.score(X, y)
            
    elif type == 'MAE':
        for i in range(result_size):
            y = data[i,:]
            reg = QuantileRegressor(quantile = 0.5, alpha = 0, solver = 'highs').fit(X, y)
            result[i] = reg.predict(X)
            scores[i] = reg.score(X, y)
    
    return (result, scores)

##returns the ratio of the average energy between two sets of eigenvectors (element-wise)
##"energy" is defined as value^2 - two sets of eigenvectors with the same "energy" would
##recreate data with approximately the same kinetic energy (v^2)

def getEnergyCoefs(eigs, old_eigs):
    coefs = np.sqrt( (old_eigs[0:eigs.shape[0]]**2).mean(axis = 1)/(eigs**2).mean(axis = 1))
    return coefs<jupyter_output><empty_output><jupyter_text>Importing data for downscaling<jupyter_code>##supposed to be 2008 canrcm data
##d1 = xr.open_dataset(p_can07)
##duplicating feb 28 twice to account for leap year (feb 29th is missing in CANRCM)
##can08 = np.concatenate((d1[data_name_can][:472,140:165,60:85], d1[data_name_can][464:472,140:165,60:85], d1[data_name_can][472:,140:165,60:85] ))
##not duplicating feb 28 twice - for 2007
##can08 = d1[data_name_can][16:,140:165,60:85]<jupyter_output><empty_output><jupyter_text>Projection Function<jupyter_code>##scalar projection of u onto v - with extra 1/norm factor (for math reasons)
##projectData projects the data onto each principle conponent, at each time
##output is a set of eigenvectors

def project(u, v):  
    v_norm = np.sqrt(np.sum(v**2))    
    return np.dot(u, v)/v_norm**2

def projectData(data_mat, new_PCs, n):
    time = data_mat.shape[1]
    proj = np.empty((n, time))

    for j in range(n):
        for i in range(time):
            proj[j, i] = project(data_mat[:,i], new_PCs[:,j])
            
    return proj<jupyter_output><empty_output><jupyter_text>Overall Function<jupyter_code>##this function allows you to add extra eigenvectors to potentially improve the fit
##can_comb should be an array with the following format: [mean, extra eigs..., PCA eigs]
def reconstruct_extra(downscale_mat, mean, can_PCs, can_comb, hr_PCs, hr_me, n, r, extra_eigs08, method = 'LS'):

    coefs = getCoefs(can_comb, n + 1, hr_me, r + 1, type = method)
    proj = np.concatenate((mean.reshape(1, -1), extra_eigs08.reshape(1, -1), projectData(downscale_mat - mean, can_PCs, n - 1)), axis = 0)
    pred_eigs = np.matmul(coefs[0], proj) + coefs[1].reshape(-1, 1)  ##multiple linear regression output
    recon = np.matmul(hr_PCs[:,0:r], pred_eigs[1:r+1]) + pred_eigs[0]
    data_rec = reverse(recon, (-1, 266, 256))
    
    return data_rec<jupyter_output><empty_output><jupyter_text>Reconstructing Data<jupyter_code>HB = hr07[:,142,139]
SS = hr07[:,174,107]
SH = hr07[:,129,149]
##combining mean, PCA eigenvectors, and solar eigenvector together
can07_comb = np.concatenate((can07_mean.reshape(1, -1), solar_eig.reshape(1, -1), can07_eigs))
data_rec = reconstruct_extra(can07_mat, can07_mean, can07_PCs, can07_comb, hr07_PCs, hr07_me, 85, 20, solar_eig, method = 'LS')
##for precip and humidity, replace negative values with 0
if data_name_hr == 'precip' or data_name_hr == 'qair' or data_name_hr == 'solar' or data_name_hr == 'therm_rad':
    data_rec[data_rec < 0] = 0
if data_name_hr == 'solar':
    night_temp = np.mean(data_rec[2::8], axis = 0)
    data_rec[data_rec < night_temp + 0.1] = 0 
    
HB_rec = data_rec[:,142,139]
SS_rec = data_rec[:,174,107]
SH_rec = data_rec[:,129,149]<jupyter_output><empty_output><jupyter_text>Finding Optimal Number of Modes<jupyter_code>##choose how many variables to use in reconstruction
## organized [n, r]
##num_modes = {'tair': [55, 80], 'precip': [12, 50], 'atmpres': [35, 75], 'qair': [85, 20],'solar':  [85, 20],'therm_rad': [50, 20], 'u_wind': [70, 50], 'v_wind': [70, 50]}
print(name)
runs = []

for n in [65, 70, 75, 80]:
    for r in [40, 45, 50, 55, 60]:
        data_rec = reconstruct(can08_mat, mean_2008, can07_PCs, can07_me, hr07_PCs, hr07_me, n, r, method = 'LS', make_positive = 'true')
        HB_rec = data_rec[:,142,139]
        SS_rec = data_rec[:,174,107]
        SH_rec = data_rec[:,129,149]
        avg_rmse = np.sqrt( mean_squared_error(HB, HB_rec)) + np.sqrt( mean_squared_error(SS, SS_rec)) + np.sqrt( mean_squared_error(SH, SH_rec))
        avg_rmse /= 3
        runs.append([n, r, avg_rmse])
        
arr = np.array(runs)
arr_min = np.argmin( arr[:, 2] )
print("n, r, rmse")
print(runs)
print("minimum:", arr[arr_min])<jupyter_output><empty_output><jupyter_text>Import previous runs from here<jupyter_code>glob.glob('/ocean/arandhawa/MOAD/analysis-armaan/notebooks/recon08_*.csv')
variables = [['tair', 'tas', 'Near-Surface Air Temperature'], 
             ['precip', 'pr', 'Precipitation'], 
             ['atmpres', 'psl', 'Sea Level Pressure'], 
             ['qair', 'huss', 'Near Surface Specific Humidity'], 
             ['solar', 'rsds', 'Shortwave radiation'], 
             ['therm_rad', 'rlds', 'Longwave radiation'],
             ['u_wind', 'uas', 'u-winds'], 
             ['v_wind', 'vas', 'vas']]
##import results of previous runs
var = variables[4]
df = pd.read_csv('/ocean/arandhawa/MOAD/analysis-armaan/notebooks/recon08_' + var[0] + '.csv')
name = var[2]

HB = np.array(df['HB_hr'])
SS = np.array(df['SS_hr'])
SH = np.array(df['SH_hr'])
HB_rec = np.array(df['HB_recon'])
SS_rec = np.array(df['SS_recon'])
SH_rec = np.array(df['SH_recon'])
data_rec.shape
night_temp = np.mean(data_rec[2::8], axis = 0)
night_temp.shape<jupyter_output><empty_output><jupyter_text>Analysis<jupyter_code>fig, ax = plt.subplots(1, 3, figsize=(15, 4))
alpha_val = 0.7

ax[0].plot(HB[:150], label = "HRDPS", alpha = alpha_val)
ax[0].plot(HB_rec[:150], label = "Reconstructed", alpha = alpha_val)
ax[0].set_title("Halibut Bank, " + name)
ax[0].set_xlabel("Time (hours)")
ax[0].set_ylabel(name)
ax[0].legend()

ax[1].plot(SS[:150], label = "HRDPS", alpha = alpha_val)
ax[1].plot(SS_rec[:150], label = "Reconstructed", alpha = alpha_val)
ax[1].set_title("Sentry Shoal, " + name)
ax[1].set_xlabel("Time (hours)")
ax[1].set_ylabel(name)
ax[1].legend()

ax[2].plot(SH[:150], label = "HRDPS", alpha = alpha_val)
ax[2].plot(SH_rec[:150], label = "Reconstructed", alpha = alpha_val)
ax[2].set_title("Sand Heads, " + name)
ax[2].set_xlabel("Time (hours)")
ax[2].set_ylabel(name)
ax[2].legend()
print("HB RMSE:", np.sqrt( mean_squared_error(HB, HB_rec)), 
                            "SS RMSE:", np.sqrt( mean_squared_error(SS, SS_rec)), 
                                                  "SH RMSE:", np.sqrt( mean_squared_error(SH, SH_rec)))

avg_rmse = np.sqrt( mean_squared_error(HB, HB_rec)) + np.sqrt( mean_squared_error(SS, SS_rec)) + np.sqrt( mean_squared_error(SH, SH_rec))
avg_rmse = avg_rmse/3
print("Average RMSE:", avg_rmse)
plt.tight_layout()
print("HB:", np.mean(HB), "HB_recon:", np.mean(HB_rec), "ratio", np.mean(HB_rec)/np.mean(HB))
print("SH:", np.mean(SH), "SH_recon:", np.mean(SH_rec), "ratio", np.mean(SH_rec)/np.mean(SH))
print("SS:", np.mean(SS), "SS_recon:", np.mean(SS_rec), "ratio", np.mean(SS_rec)/np.mean(SS))

print("HB:", np.mean(HB), "HB_recon:", np.mean(HB_rec), "ratio", np.mean(HB_rec[:150])/np.mean(HB[:150]))
print("SH:", np.mean(SH), "SH_recon:", np.mean(SH_rec), "ratio", np.mean(SH_rec[:150])/np.mean(SH[:150]))
print("SS:", np.mean(SS), "SS_recon:", np.mean(SS_rec), "ratio", np.mean(SS_rec[:150])/np.mean(SS[:150]))
def moving_average(x, w):
    return np.convolve(x, np.ones(w), 'valid') / w

plt.plot(moving_average(HB,40))
plt.plot(moving_average(HB_rec,40))
plt.plot(np.cumsum(SS))
plt.plot(np.cumsum(SS_rec))
min = np.min([np.min(HB), np.min(SH), np.min(SS)])
max = np.max([np.max(HB), np.max(SH), np.max(SS)])
diff = max - min
nums = np.arange(min, max, (max - min)/16)

fig, ax = plt.subplots(1, 3, figsize=(10, 5))

ax[0].set_title("Halibut Bank")
ax[0].hist(HB, bins = nums, label = "HRDPS", alpha = 0.7)
ax[0].hist(HB_rec, bins = nums, label = "Reconstructed", alpha = 0.7)
ax[0].set_xlabel(name)
ax[0].set_ylabel("Number of data points")
ax[0].legend()

ax[1].set_title("Sentry Shoal")
ax[1].hist(SS, bins = nums, label = "HRDPS", alpha = 0.7)
ax[1].hist(SS_rec, bins = nums, label = "Reconstructed", alpha = 0.7)
ax[1].set_xlabel(name)
ax[1].set_ylabel("Number of data points")
ax[1].legend()

ax[2].set_title("Sand Heads")
ax[2].hist(SH, bins = nums,  label = "HRDPS", alpha = 0.7)
ax[2].hist(SH_rec, bins = nums, label = "Reconstructed", alpha = 0.7)
ax[2].set_xlabel(name)
ax[2].set_ylabel("Number of data points")
ax[2].legend()

plt.tight_layout()
a1 = np.histogram(HB, bins = nums)
b1 = np.histogram(HB_rec, bins = nums)

a2 = np.histogram(SS, bins = nums)
b2 = np.histogram(SS_rec, bins = nums)

a3 = np.histogram(SH, bins = nums)
b3 = np.histogram(SH_rec, bins = nums)

# calc middle of bin and width of bin
axes = [(a1[1][x]+a1[1][x+1])/2 for x in range(len(a1[1])-1)] 
w = a1[1][1] - a1[1][0]
##plot differences
fig, ax = plt.subplots(1, 3, figsize = (10, 3))

ax[0].bar(axes, b1[0] - a1[0], width=w, label="Reconstruct - HRDPS 2008")
ax[0].set_title("Halibut Bank")
ax[1].bar(axes, b2[0] - a2[0], width=w, label="Reconstruct - HRDPS 2008")
ax[1].set_title("Sentry Shoal")
ax[2].bar(axes, b3[0] - a3[0], width=w, label="Reconstruct - HRDPS 2008")
ax[2].set_title("Sand Heads")

for i in range(3):
    ax[i].set_xlabel(name)
    ax[i].set_ylabel("Data Points")
plt.tight_layout()
metrics = np.zeros((3, 3))

def helper(x, y, ax, i):
    
    reg = LinearRegression().fit(x.reshape(-1, 1), y)
    a = reg.coef_[0]
    b = reg.intercept_
    y_pred = a*x + b
    
    bias = np.mean( y - x )
    r_squared = reg.score(x.reshape(-1, 1), y)
    rmse = np.sqrt( mean_squared_error(y, y_pred) )
    
    global metrics
    metrics[i, 0] = r_squared
    metrics[i, 1] = bias
    metrics[i, 2] = rmse
    
    im = ax[i].hist2d(x, y, bins = 10, range = [[min, max],[min, max]], cmap=plt.cm.Reds)
    ax[i].plot(x, a*x+b, color = 'black')
    ax[i].plot(x, x, color = 'blue')
    ax[i].annotate("r-squared = {:.3f}".format(r_squared), (min + diff/16, max - diff*3/34))
    ax[i].annotate("bias = {:.5f}".format(bias), (min + diff/16, max - diff*5/34))
    ax[i].annotate("RMS error = {:.5f}".format(rmse), (min + diff/16, max - diff*7/34))
    ax[i].annotate("y = {:.3f}x + {:.3f}".format(a, b), (min + diff/16, max - diff*9/34))
    ax[i].set_ylabel("HRDPS data")
    ax[i].set_xlabel("reconstructed data")
    ##ax[i].set_ylim(-14, 20)
    ##ax[i].set_xlim(-14, 20)
    ## plt.colorbar(im,ax=ax[i,j]) 
    ##https://stackoverflow.com/questions/23876588/matplotlib-colorbar-in-each-subplot

fig, ax = plt.subplots(1, 3, figsize = (14, 4))
##Habibut bank
y = HB
x = HB_rec
ax[0].set_title("Halibut Bank, " + name)
helper(x, y, ax, 0)

##Sentry Shoal
y = SS
x = SS_rec
ax[1].set_title("Sentry Shoa, " + name)
helper(x, y, ax, 1)

##Sand Heads
y = SH
x = SH_rec
ax[2].set_title("Sand Heads, " + name)
helper(x, y, ax, 2)

plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Exports<jupyter_code>dict = {
    "HB_hr" : HB,
    "SS_hr" : SS,
    "SH_hr" : SH,
    "HB_recon" : HB_rec,
    "SS_recon" : SS_rec,
    "SH_recon" : SH_rec,
}

export = pd.DataFrame(dict)
filepath = '/ocean/arandhawa/MOAD/analysis-armaan/notebooks/recon08_' + data_name_hr + '.csv'
print(filepath)
## export.to_csv(filepath, index = False, header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16562028519978586, 'fraction_numerical': 0.044726724095975084, 'mean_word_length': 3.788627359589839, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 56, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '141396', 'license': 'Apache-2.0', 'n_tokens_mistral': 8764, 'n_tokens_neox': 7961, 'n_words': 1876, 'path': 'notebooks/generalPredictions4-solar2.ipynb', 'repo_name': 'UBC-MOAD/analysis-armaan', 'size': '20547'}