<jupyter_start><jupyter_text>Load things<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from ipywidgets import interact
import os,sys
# sys.addaudithook
rpath = os.getcwd()
sys.path.insert(0,"../")
# import utils


import dimred ### library of dimension reduction
plt.style.use('fivethirtyeight')

plt.style.available
import ipywidgets
from ipywidgets import interact

from dimred.data.loader import LoadOne
from dimred.data.preprocess import MinMaxScalar,ZeroMeanScalar,MeanMaxScalar,AvgMaxScalar
# dimred.
from dimred.data.preprocess import scale_sanity,Scalar,Shaper,scaleMax

from dimred.models.linear.transform import Kurtosis
from dimred.models.linear.transform import co_variance,co_kurtosis
from dimred.tester.plotting import plot_embedding,plot_compare
from dimred.tester.metrics import mean_sq_error,mean_abs_error,abs_err

from dimred.gaspipe import transform_step,cantera_step,build_dictionary,retain_analysis
from dimred.tester.plotting import plot_spectra,img_compare,plot_bars<jupyter_output><empty_output><jupyter_text>Read data<jupyter_code>loader = LoadOne()
xregions = [loader.getData()]
loader.plotImg(spec=12)#,aspect=0.9)

domains = (1,4)
xregions.extend(loader.getDomain(domains))

for x in xregions:
    print(x.shape)<jupyter_output>(201, 1024, 17)
(201, 256, 17)
(201, 256, 17)
(201, 256, 17)
(201, 256, 17)<jupyter_text>Transformation- xreduced is 4D vector:- project back to original space- Low rank newly reconstructed matrix<jupyter_code>def mf_embed(region_id=1):
    xrig = xregions[region_id].copy()
    fig = plt.figure(figsize=(12,5))
    ax = fig.add_subplot(121, projection='3d')
    cmv,xold,xnew = transform_step(xrig,moment=co_variance, scalar=AvgMaxScalar,plt_ax=ax,verbose=False)

    ax = fig.add_subplot(122, projection='3d')
    cmk,kold,knew = transform_step(xrig,moment=co_kurtosis, scalar=AvgMaxScalar,plt_ax=ax,verbose=False)        


mf_embed(0)
interact(mf_embed,time_step=range(5))
def mf_orient(region_id=1):
    # time_step = 100
    xrig = xregions[region_id].copy()

    cmv,xold,xnew = transform_step(xrig,moment=co_variance, scalar=AvgMaxScalar,verbose=False,plots=False)
    cmk,kold,knew = transform_step(xrig,moment=co_kurtosis, scalar=AvgMaxScalar,verbose=False,plots=False)
    plot_spectra(cmv.s,cmk.s,cmv.u,cmk.u)

mf_orient()

interact(mf_orient,region_id=range(5))
def mf_retain(region_id=0,scale='log'):
    # xrig = loader.getTime(time_step,verbose=-1)[:,:14]
    spr = Shaper()
    xrig = spr.fit_transform(xregions[region_id].copy())

    verr = retain_analysis(xrig,moment=co_variance).reshape(-1,1)
    kerr = retain_analysis(xrig,moment=co_kurtosis).reshape(-1,1)
    fig = plot_compare(verr,kerr,titler="Moment comparison",species=0,labels=["Variance","Kurtosis"])
    fig.axes[0].set_xlabel("Number of retained vectors")
    fig.axes[0].set_ylabel("Species reconstruction error")
    fig.axes[0].set_yscale(scale)

mf_retain(2)

interact(mf_retain,region_id=range(5),scale=['linear','log'])<jupyter_output><empty_output><jupyter_text>Interactive!<jupyter_code>region_id = 1
num_retain=4
spr = Shaper()

xrig = spr.fit_transform(xregions[region_id])
total = build_dictionary(xrig,retain=num_retain)

def f(moment,source,specs=0):
    img_compare(spr.transform2(total[moment]['old'][source]),
                 spr.transform2(total[moment]['new'][source]),species=specs)

interact(f,moment=total.keys(),source=total['covariance']['old'].keys(),specs=loader.varid)
def f(moment,source,specs=0):
    plot_compare(total[moment]['old'][source],
                 total[moment]['new'][source],species=specs)

interact(f,moment=total.keys(),source=total['covariance']['old'].keys(),specs=loader.varid)
# spr.transform2(total[moment]['old'][source])
def f(source):
    moment = "covariance"
    errcv = abs_err(total[moment]['old'][source] ,total[moment]['new'][source])
    moment = "cokurtosis"
    errck = abs_err(total[moment]['old'][source] ,total[moment]['new'][source])

    plot_bars(errcv,errck,horz=True)

interact(f,source=total['covariance']['old'].keys())
plt.style.use('seaborn')
loader.varid<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14468599033816426, 'fraction_numerical': 0.021497584541062802, 'mean_word_length': 7.249003984063745, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '271466', 'license': 'MIT', 'n_tokens_mistral': 1591, 'n_tokens_neox': 1513, 'n_words': 211, 'path': 'examples/combust_domain.ipynb', 'repo_name': 'kulkarnisp/dim-red', 'size': '4140'}