<jupyter_start><jupyter_text>Load things<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
plt.style.use('ggplot')

import pandas as pd

import os,sys
# sys.addaudithook
rpath = os.getcwd()
sys.path.insert(0,"../")
# import utils

import dimred ### library of dimension reduction
from dimred.newpipe import Elbow
from dimred.newcistern import Knob

from dimred.data.preprocess import MinMaxScalar,AvgMaxScalar,MaxAvgScalar,scaleData
from dimred.models.linear.transform import co_kurtosis,ra_kurtosis,val_kurtosis
from dimred.data.loader import LoadOne,LoadMPI,LoadNumpy
from dimred.flowpipe import ServicePIpe
# from dimred.data.oldloader2 import localpath__

from dimred.models.linear.transform import co_kurtosis,co_variance,Kurtosis
from dimred.data.preprocess import Shaper,Scalar

class Servicer:
    def __init__(self,x,moment=co_variance,scalar=AvgMaxScalar,n_retain=4) -> None:
        # self.xin = x.copy()
        self.xout = x.copy()
        self.shaper= Shaper()
        self.scalar = Scalar(scalar)
        self.model = Kurtosis(moment=moment,n_retain=n_retain)
        self.pipeline = [self.shaper,self.scalar,self.model]
        ## Todo add post processing class
    
    def encode(self):
        for p in self.pipeline:
            self.xout = p.fit_transform(self.xout)

    def decode(self,xenc):
        for p in reversed(self.pipeline):
            xenc = p.transform2(xenc)
        return xenc
        # return x
    
loader = LoadNumpy('oned','onedsin')
def get_dot(u1,u2):
    return [v1@v2 for v1,v2 in zip(u1,u2)]

def get_fmm(u1,s1):
    return (u1*s1).sum(axis=1)

def make_df(arr,cols):
    df = pd.DataFrame(arr)
    df.columns = cols
    return df.T
def feat_comp(loader,tidx=2,Nd=4):
    gd = loader.x[tidx]
    gp = ServicePIpe(gd)

    dx = int(len(gd)/Nd)
    sdms = [gd[dx*i:dx*(i+1)] for i in range(Nd)] 
    sps = [Servicer(dm) for dm in sdms]

    temp = [p.encode() for p in sps+[gp]]
    
    fmms = [get_fmm(sp.model.u,sp.model.s) for sp in sps]
    locs = [get_dot(gp.model.u,sp.model.u) for sp in sps]

    cols = loader.varid
    return make_df(fmms,cols) ,make_df(locs,cols)   
def plot_time(tidx=10):
    df1,df2 = feat_comp(loader,tidx)
    fig,[ax1,ax2] = plt.subplots(ncols=2,figsize=(14,6))
    df1.plot.bar(ax=ax1)
    df2.plot.bar(ax=ax2)
from ipywidgets import interactive
interactive(plot_time,tidx=range(1,201))

dots = [getDot(gp.model.u,sp.model.u) for sp in sps]
import pandas as pd
df = pd.DataFrame(np.array(dots))
df.columns = loader.varid
df.T.plot.bar()
dms[0].shape
rn = Elbow('ethane')
rn.MyScalar = MinMaxScalar

rn.mf_build(10);
# rn.mf_alldata();
topbar = Knob(rn)

topbar
rn = Elbow('methane')
rn.MyScalar = MaxAvgScalar

rn.mf_build(10);
# rn.mf_alldata();
topbar = Knob(rn)

topbar
rn.loader.varid<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12975471027372912, 'fraction_numerical': 0.015286171347316033, 'mean_word_length': 3.8350515463917527, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '261403', 'license': 'MIT', 'n_tokens_mistral': 1126, 'n_tokens_neox': 1108, 'n_words': 219, 'path': 'examples/combust_data.ipynb', 'repo_name': 'kulkarnisp/dim-red', 'size': '2813'}