<jupyter_start><jupyter_text>This file is using POD to deal with FPC data  Mounting your google driveYou can use google drive to store and access files e.g. storing and loading data from numpy or CSV files.  Use the following command to mount your GDrive and access your files.<jupyter_code>from google.colab import drive
drive.mount('/content/gdrive/')
!pip install ffmpeg
!pip install vtk
import os
# change the current path. The user can adjust the path depend on the requirement
os.chdir("/content/gdrive/MyDrive/Cola-Notebooks/FYP/YF")
import vtktools
# !unzip csv_data.zip 
%matplotlib inline
import numpy as np
import pandas as pd
import scipy
import numpy.linalg as la
import scipy.linalg as sl
import scipy.sparse.linalg as spl
import matplotlib.pyplot as plt
import torch.nn as nn  # Neural network module
import scipy.sparse as sp
import scipy.optimize as sop
import progressbar
# making slopes
import torch
from torch.utils.data import TensorDataset
import torch.nn.functional as F
from matplotlib.pyplot import LinearLocator
import matplotlib as mpl
import matplotlib.colors as colors


# create an animation
from matplotlib import animation
from IPython.display import HTML

from matplotlib import animation
import math
import ffmpeg

!pip install pycm livelossplot
%pylab inline
from livelossplot import PlotLosses

from torch.utils.data import DataLoader
import torch.utils.data as Data

import time
import platform
print('python version', platform.python_version())
print('torch version', torch.__version__)
print('numpy version', np.version.version)
def set_seed(seed):
    """
    Use this to set ALL the random seeds to a fixed value and take out any randomness from cuda kernels
    """
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

    torch.backends.cudnn.benchmark = True  ##uses the inbuilt cudnn auto-tuner to find the fastest convolution algorithms. -
    torch.backends.cudnn.enabled   = True

    return True

device = 'cuda'  # Set out device to GPU

print('Cuda installed, running on GPU!')  # print sentence
# These functions are saved in function.py and the note are also added to that file
def saveIndex(path_train, path_valid, path_test,train_index, valid_index, test_index):
    # save training and validation loss    
    np.savetxt(path_train,train_index, delimiter=',')
    np.savetxt(path_valid,valid_index, delimiter=',')
    np.savetxt(path_test,test_index, delimiter=',')

def getIndex(path_train,path_valid,path_test):
    train_index = np.loadtxt(path_train,delimiter=",")
    valid_index = np.loadtxt(path_valid,delimiter=",")
    test_index = np.loadtxt(path_test,delimiter=",")
    return train_index,valid_index,test_index<jupyter_output><empty_output><jupyter_text>SVD  Load data<jupyter_code>os.chdir('/content/gdrive/MyDrive/Cola-Notebooks/FYP/YF')
print(os.getcwd())

# read in the data (1000 csv files)
nTrain = 1600
nValid = 200
nTest = 200
nTotal = nTrain + nValid + nTest
nNodes = 20550 # should really work this out


# The below method to load data is too slow. Therefore, we use load pt file

# [:, :, 2] is speed, [:, :, 3] is u, [:, :, 4] is v
# (speed not really needed)
# [:, :, 0] and [:, :, 1] are the SFC orderings

# training_data = np.zeros((nTrain,nNodes,5))
# for i in range(nTrain):
#     data = np.loadtxt('csv_data/data_' +str(int(train_index[i]))+ '.csv', delimiter=',')
#     training_data[i,:,:] = data
# training_data = np.array(training_data)
# print('size training data', training_data.shape)

# valid_data = np.zeros((nValid,nNodes,5))
# for i in range(nValid):
#     data = np.loadtxt('csv_data/data_' +str(int(valid_index[i]))+ '.csv', delimiter=',')
#     valid_data[i,:,:] = data
# valid_data = np.array(valid_data)
# print('size validation data', valid_data.shape)

# test_data = np.zeros((nTest,nNodes,5))
# for i in range(nTest):
#     data = np.loadtxt('csv_data/data_' +str(int(test_index[i]))+ '.csv', delimiter=',')
#     test_data[i,:,:] = data
# test_data = np.array(test_data)
# print('size test data', test_data.shape)

# total_data = np.zeros((nTotal,nNodes,5))
# for i in range(len(train_index)):
#     total_data[int(train_index[i]),:,:] = training_data[i,:,:]

# for i in range(len(valid_index)):
#     total_data[int(valid_index[i]),:,:] = valid_data[i,:,:]

# for i in range(len(test_index)):
#     total_data[int(test_index[i]),:,:] = test_data[i,:,:]
# print('size total data', total_data.shape)


# Before we save the pt file, we must load the data according to the above method
# torch.save(training_data, '/content/gdrive/MyDrive/FPC_new_random_train.pt')
# torch.save(valid_data, '/content/gdrive/MyDrive/FPC_new_random_valid.pt')
# torch.save(test_data, '/content/gdrive/MyDrive/FPC_new_random_test.pt')
# torch.save(total_data, '/content/gdrive/MyDrive/FPC_new_random_total.pt')

# load the data, this method save the time
training_data = torch.load('/content/gdrive/MyDrive/FPC_new_random_train.pt')
valid_data = torch.load('/content/gdrive/MyDrive/FPC_new_random_valid.pt')
test_data = torch.load('/content/gdrive/MyDrive/FPC_new_random_test.pt')
total_data = torch.load('/content/gdrive/MyDrive/FPC_new_random_total.pt')
print(training_data.shape)
print(valid_data.shape)
print(test_data.shape)
print(total_data.shape)
# Load the index of different part data
total_index = []
path_train = "/content/gdrive/MyDrive/Cola-Notebooks/FYP/YF/"+"new_FPC_train_index.csv"
path_valid = "/content/gdrive/MyDrive/Cola-Notebooks/FYP/YF/"+"new_FPC_valid_index.csv"
path_test = "/content/gdrive/MyDrive/Cola-Notebooks/FYP/YF/"+"new_FPC_test_index.csv"
# saveIndex(path_train, path_valid, path_test,train_index, valid_index, test_index)

train_index,valid_index,test_index= getIndex(path_train,path_valid,path_test)
print(test_index)
for i in range(2000):
    total_index.append(i)
# rescale the data so that u and v data lies in the range [-1,1] (and speed in [0,1])
ma = np.max(training_data[:, :, 2])
mi = np.min(training_data[:, :, 2])
k = 1./(ma - mi)
b = 1 - k*ma #k*mi
training_data[:, :, 2] = k * training_data[:, :, 2] + b #- b
# this won't be used

ma = np.max(training_data[:, :, 3])
mi = np.min(training_data[:, :, 3])
ku = 2./(ma - mi)
bu = 1 - ku*ma 
training_data[:, :, 3] = ku * training_data[:, :, 3] + bu
valid_data[:, :, 3] = ku * valid_data[:, :, 3] + bu
test_data[:, :, 3] = ku * test_data[:, :, 3] + bu
total_data[:, :, 3] = ku * total_data[:, :, 3] + bu

ma = np.max(training_data[:, :, 4])
mi = np.min(training_data[:, :, 4])
kv = 2./(ma - mi)
bv = 1 - kv*ma
training_data[:, :, 4] = kv * training_data[:, :, 4] + bv
valid_data[:, :, 4] = kv * valid_data[:, :, 4] + bv
test_data[:, :, 4] = kv * test_data[:, :, 4] + bv
total_data[:, :, 4] = kv * total_data[:, :, 4] + bv
# Because it has two parameters. Reshapr the data from(2000,20550,2) to (2000,20550*2)
snaps = np.zeros([nTrain,nNodes])
snaps1 = np.zeros([nTrain,nNodes])
snaps2 = np.zeros([nValid,nNodes])
snaps3 = np.zeros([nValid,nNodes])
snaps4 = np.zeros([nTest,nNodes])
snaps5 = np.zeros([nTest,nNodes])

for i in range(nTrain):
    snaps[i,:] = training_data[i,:,3]
    snaps1[i,:] = training_data[i,:,4]

snapstrain = np.c_[snaps,snaps1]


for i in range(nValid):
    snaps2[i,:] = valid_data[i,:,3]
    snaps3[i,:] = valid_data[i,:,4]

snapsvalid = np.c_[snaps2,snaps3]


for i in range(nTest):
    snaps4[i,:] = test_data[i,:,3]
    snaps5[i,:] = test_data[i,:,4]

snapstest = np.c_[snaps4,snaps5]

snapstotal = np.zeros([total_data.shape[0],20550*2])
for i in range(nTrain+nValid+nTest):
    snapstotal[i,:nNodes] = total_data[i,:,3]
    snapstotal[i,nNodes:] = total_data[i,:,4]

print('snapstrain shape: ',snapstrain.shape)
print('snapsvalid shape: ',snapsvalid.shape)
print('snapstest shape: ',snapstest.shape)
print('snapstotal shape: ',snapstotal.shape)
# The solution is usually arranged in a column vector
snapstrain = snapstrain.transpose()
snapsvalid = snapsvalid.transpose()
snapstest = snapstest.transpose()
snapstotal = snapstotal.transpose()
print('snapetrain shape',snapstrain.shape)
print('snapevalid shape',snapsvalid.shape)
print('snapetest shape',snapstest.shape)
print('snapetotal shape',snapstotal.shape)<jupyter_output>snapetrain shape (41100, 1600)
snapevalid shape (41100, 200)
snapetest shape (41100, 200)
snapetotal shape (41100, 2000)<jupyter_text>SVD of the snapshot matrix<jupyter_code># Using la.svd to do the singular value decomposition
t_train_0 = time.time()
u,sigma2,vt = la.svd(snapstrain,False)
t_train_1 = time.time()
print('u shape:',u.shape)
print('sigma2 shape:',sigma2.shape)
print('vt shape: ',vt.shape)
print("Train time: ",t_train_1-t_train_0)<jupyter_output>u shape: (41100, 1600)
sigma2 shape: (1600,)
vt shape:  (1600, 1600)
Train time:  14.94536280632019<jupyter_text>MSE equation$$\frac1N{\textstyle\sum_{p=1}^N}{\textstyle(}{\textstyle{\scriptstyle u}^{t_p}}{\textstyle-}{\textstyle R}{\textstyle{\scriptstyle R}^T}{\textstyle{\scriptstyle u}^{t_p}}{\textstyle)}{\textstyle\cdot}{\textstyle(u^{t_p}-RR^Tu^{t_p})}$$The relative square error$$\frac{\frac1N\sum_{p=1}^N(u^{t_p}-RR^Tu^{t_p})\cdot(u^{t_p}-RR^Tu^{t_p})}{\frac1N\sum_{p=1}^N(u^{t_p})\cdot(u^{t_p})}$$<jupyter_code># The function is used to calculate the mean square error accoridng to the above equation
def MSE(S,index, R, RT):
    m = zeros([2*nNodes,len(index)])
    n = zeros([len(index),nNodes,2])
    for i in range(len(index)):
        middle = RT@S[:,int(index[i])]
        m[:,i] = S[:,int(index[i])] - R @ middle
        n[i,:,0] = m[:nNodes,i]
        n[i,:,1] = m[nNodes:,i]
    return (n**2).mean()


# def Error(S, index, R, RT):
#     m = zeros([2*nNodes,len(index)])
#     n = zeros([len(index),nNodes,2])
#     error = zeros([len(index),i])
#     for i in range(len(index)):
#         middle = RT@S[:,int(index[i])]
#         m[:,i] = S[:,int(index[i])] - R @ middle
#         n[i,:,0] = m[:nNodes,i]
#         n[i,:,1] = m[nNodes:,i]
#         error[i,0] = (n[i,:,:]**2).mean()
#     return error

# THis function is used to calculate the relative square error according to the above equation
def relative_error(S, index, R, Rt):
    m = zeros([2*nNodes,len(index)])
    n = zeros([len(index),nNodes,2])
    h = zeros([len(index),nNodes,2])
    for i in range(len(index)):
        middle = RT@S[:,int(index[i])]
        m[:,i] = S[:,int(index[i])] - R @ middle
        h[i,:,0] = S[nNodes:,int(index[i])]
        h[i,:,1] = S[:nNodes,int(index[i])]
        n[i,:,0] = m[:nNodes,i]
        n[i,:,1] = m[nNodes:,i]
    return (n**2).mean()/(h**2).mean()

# This function is used to reconstruct data from the reduced variable
def orginal(S,R,RT):
    N = len(S[1])
    # print('N shape',N)
    orginal = np.zeros([2*nNodes,N])
    for i in range(N):
        middle = RT@S[:,i]
        orginal[:,i] = R@middle
    return orginal<jupyter_output><empty_output><jupyter_text>1 variable<jupyter_code># Choose the number of basis function
R = u[:,0:1]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print(result.shape)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  0.021071473294830976
The valid MSE:  0.02082311044842053
The test MSE:  0.02129882757690237
The total MSE:  0.0210693724383971
The train relative square error:  0.2930264195339251
The valid relative square error:  0.2906072995373513
The test relative square error:  0.2948952056775621
The total relative square error:  0.29297307116573224<jupyter_text>2 variable<jupyter_code># Choose the number of basis function
R = u[:,0:2]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  0.01153585065129064
The valid MSE:  0.010870498227003558
The test MSE:  0.0123944830594379
The total MSE:  0.011555178649676633
The train relative square error:  0.16042110417855854
The valid relative square error:  0.15170865765708355
The test relative square error:  0.17160914692993542
The total relative square error:  0.16067664980352925<jupyter_text>4 variable<jupyter_code># Choose the number of basis function
R = u[:,0:4]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  0.002778288171449406
The valid MSE:  0.0027700291476035767
The test MSE:  0.0025904994995278835
The total MSE:  0.00275868340187267
The train relative square error:  0.038635733910119346
The valid relative square error:  0.0386585228090113
The test relative square error:  0.03586703915803049
The total relative square error:  0.03835994408393747<jupyter_text>8 variable<jupyter_code># Choose the number of basis function
R = u[:,0:8]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  0.0016314512027842728
The valid MSE:  0.001567985800200461
The test MSE:  0.0014226494733828626
The total MSE:  0.0016042244895857491
The train relative square error:  0.02268746460711236
The valid relative square error:  0.021882807577564962
The test relative square error:  0.01969744614089834
The total relative square error:  0.02230700401387807<jupyter_text>16 variable<jupyter_code># Choose the number of basis function
R = u[:,0:16]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  0.0007154732169125436
The valid MSE:  0.0006605267440753894
The test MSE:  0.0006522750422584215
The total MSE:  0.0007036587521634185
The train relative square error:  0.009949591663138793
The valid relative square error:  0.009218310292471612
The test relative square error:  0.009031144181557475
The total relative square error:  0.009784490082783268<jupyter_text>32 variable<jupyter_code># Choose the number of basis function
R = u[:,0:32]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  0.00026879591002737074
The valid MSE:  0.0002628113410994816
The test MSE:  0.0002591130344208928
The total MSE:  0.000267229165573934
The train relative square error:  0.0037379589931191533
The valid relative square error:  0.0036677946992546085
The test relative square error:  0.0035875773585843265
The total relative square error:  0.0037158652718375686<jupyter_text>64 variable<jupyter_code># Choose the number of basis function
R = u[:,0:64]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  7.060790934629863e-05
The valid MSE:  8.062858622131723e-05
The test MSE:  7.631131863237528e-05
The total MSE:  7.218031796240788e-05
The train relative square error:  0.000981895407930365
The valid relative square error:  0.001125252433604068
The test relative square error:  0.0010565765614265407
The total relative square error:  0.0010036791315448646<jupyter_text>Convert csv to vtu Convet csv to vtu so that we can watch them using paraview<jupyter_code>print('result shape: ',result.shape)
result_recover = result.transpose()
print('results_recover shape:',result_recover.shape)
# Splite the velocity u and v, convert shape from (2000,41100) to (2000,20550,2)
rec_results = np.zeros([2000,20550,2])
for i in range(2000):
    rec_results[i,:,0] = result_recover[i,:nNodes]
    rec_results[i,:,1] = result_recover[i,nNodes:]
print("rec_results shape: ",rec_results.shape)
# Before convert csv file to vtu file, the range of data must be recovered
rec_results[:,:,0] = (rec_results[:,:,0] - bu)/ku
rec_results[:,:,1] = (rec_results[:,:,1] - bv)/kv
total_data[:, :, 3] = (total_data[:, :, 3] - bu)/ku
total_data[:, :, 4] = (total_data[:, :, 4] - bv)/kv
results = rec_results
print('results shape', results.shape)
N = results.shape[1] * results.shape[2]
results = results.reshape((results.shape[0],N), order='F')
print('results shape', results.shape, type(results))
# The path can be defined by user depending on the requirements
path = "./HAE/All_results/SVD"+"_LV"+str(64)+"_result.csv"
## write results to file
np.savetxt(path, results , delimiter=',')
# convert the error data to csv 
error = rec_results - total_data[:,:,3:5]
results = error
print('results shape', results.shape)
N = results.shape[1] * results.shape[2]
results = results.reshape((results.shape[0],N), order='F')
print('results shape', results.shape, type(results))
# The path can be defined by user depending on the requirements
path = "./HAE/All_results/SVD"+"_LV"+str(64)+"_error.csv"
## write results to file
np.savetxt(path, results , delimiter=',')<jupyter_output>results shape (2000, 20550, 2)
results shape (2000, 41100) <class 'numpy.ndarray'><jupyter_text>128 variable<jupyter_code># Choose the number of basis function
R = u[:,0:128]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  1.2090126037297648e-05
The valid MSE:  1.7100131500400123e-05
The test MSE:  1.6411809948911704e-05
The total MSE:  1.3023294974769332e-05
The train relative square error:  0.00016812902898879288
The valid relative square error:  0.00023864941068118934
The test relative square error:  0.00022723147802153608
The total relative square error:  0.00018109104751986096<jupyter_text>256 variable<jupyter_code># Choose the number of basis function
R = u[:,0:256]
print('R shape:', R.shape)
RT = R.transpose()
print('RT shape:',RT.shape)
t_pre_0 = time.time()
result = orginal(snapstotal,R,RT)
t_pre_1 = time.time()
print('Reconstruction:',t_pre_1-t_pre_0)
print("The train MSE: ",MSE(snapstotal, train_index, R, RT))
print("The valid MSE: ",MSE(snapstotal, valid_index, R, RT))
print("The test MSE: ",MSE(snapstotal, test_index, R, RT))
print("The total MSE: ",MSE(snapstotal, total_index, R, RT))
print("The train relative square error: ",relative_error(snapstotal, train_index, R, RT))
print("The valid relative square error: ",relative_error(snapstotal, valid_index, R, RT))
print("The test relative square error: ",relative_error(snapstotal, test_index, R, RT))
print("The total relative square error: ",relative_error(snapstotal, total_index, R, RT))<jupyter_output>The train MSE:  1.5128647299761146e-06
The valid MSE:  3.39044289008276e-06
The test MSE:  3.22734056735203e-06
The total MSE:  1.8720701297243711e-06
The train relative square error:  2.103836446845922e-05
The valid relative square error:  4.731701610876773e-05
The test relative square error:  4.468449058824828e-05
The total relative square error:  2.6031441465406337e-05
{'fraction_non_alphanumeric': 0.1447483451979518, 'fraction_numerical': 0.07210357603763373, 'mean_word_length': 5.869316557048899, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '285827', 'license': 'MIT', 'n_tokens_mistral': 9938, 'n_tokens_neox': 8943, 'n_words': 2229, 'path': 'fpc_methods/FPC_POD.ipynb', 'repo_name': 'isFany/IC-9', 'size': '24021'}