<jupyter_start><jupyter_text>PCA compression of atmospheres and spectra We are going to use PCA to reduce the dimensionallity of our atmospheres (T and v columns) and spectra.<jupyter_code>import numpy as np 
import matplotlib.pyplot as plt 
from astropy.io import fits
from sklearn.decomposition import PCA, IncrementalPCA
original = fits.open("C:/Data/50G.ngrey.288x100x288_atmos_61.fits")[0].data
#cnn_compressed = fits.open("C:/Data/50G.ngrey.288x100x288_atmos_50_compressed.fits")[0].data # we will do this later
original.shape
temp = original[2,:,:,:].reshape(288*288,61)
temp.shape
n_components = 61
ipca = IncrementalPCA(n_components=n_components, batch_size=100)
temp_transformed = ipca.fit_transform(temp)

#pca = PCA(n_components=n_components)
#X_pca = pca.fit_transform(X)
temp_transformed.shape
temp_compressed = ipca.inverse_transform(temp_transformed)
temp_compressed.shape
plt.figure(figsize=[12,7])
plt.subplot(221)
index = np.random.randint(0,82843)
plt.plot(temp[index])
plt.plot(temp_compressed[index])
plt.subplot(222)
index = np.random.randint(0,82843)
plt.plot(temp[index])
plt.plot(temp_compressed[index])
plt.subplot(223)
index = np.random.randint(0,82843)
plt.plot(temp[index])
plt.plot(temp_compressed[index])
plt.subplot(224)
index = np.random.randint(0,82843)
plt.plot(temp[index])
plt.plot(temp_compressed[index])
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Let's look at the basis vectors now<jupyter_code>vectors = np.squeeze(ipca.components_)

vectors.shape

plt.figure(figsize=[8,6])
plt.plot(vectors[0])
print (np.sum(vectors[0]*vectors[6]))<jupyter_output>-1.3877787807814457e-17<jupyter_text>Let's look at the original and compressed temperature distributions<jupyter_code>temp_compressed_cube = temp_compressed.reshape(288,288,61)
d = 50
plt.figure(figsize=[15,4])
plt.subplot(131)
plt.imshow(original[2,:,:,d].T,cmap='magma',origin='lower')
plt.colorbar()
plt.subplot(132)
plt.imshow(temp_compressed_cube[:,:,d].T,cmap='magma',origin='lower')
plt.colorbar()
plt.subplot(133)
plt.plot(original[2,:,:,d].flatten(),temp_compressed[:,d],'o',alpha=0.002)
plt.plot(original[2,:,:,d].flatten(),original[2,:,:,d].flatten())
plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1539846915803692, 'fraction_numerical': 0.060783430886987844, 'mean_word_length': 9.00900900900901, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '424383', 'license': 'MIT', 'n_tokens_mistral': 984, 'n_tokens_neox': 864, 'n_words': 125, 'path': 'PCA_comp.ipynb', 'repo_name': 'ivanzmilic/hinodeenc', 'size': '2221'}