<jupyter_start><jupyter_text>MNIST<jupyter_code>import models
architecture = getattr(models, "LinearOneLayer") #LinearOneLayer LogRegression

import data
loaders, num_classes = data.loaders(
    "MNIST",
    "data",
    1024,
    1,
    "VGG",
    True)

model1 = architecture.base(num_classes=10, **architecture.kwargs)
model2 = architecture.base(num_classes=10, **architecture.kwargs)

model1.load_state_dict(torch.load('curves_mnist/LinearOneLayer/LongTraining/curve16/checkpoint-30.pt')['model_state'])
model2.load_state_dict(torch.load('curves_mnist/LinearOneLayer/LongTraining/curve17/checkpoint-30.pt')['model_state'])

import pickle
data_path = '/home/ivan/dnn-mode-connectivity/data/MNIST.pickle'
with open(data_path, 'rb') as handle:
    dataset, B = pickle.load(handle)
b2 = torch.FloatTensor([np.array(B).mean(0)])
b = np.array(B).mean(0)

train_dataset, test_dataset = dataset[:-8000], dataset[-8000:]
modes = [dataset[-4000:-2000], dataset[-2000:]]
def samples(model):
    p = [list(model.parameters())[i].data.cpu().numpy() for i in range(len(list(model.parameters())))]
    return p

def samples_per_layer(model, bias=True):
    p = samples(model)
    if bias:
        p = [np.hstack([p[i], p[i+1][:, None]]) for i in range(0, len(p), 2)]        
    return p

def samples_butterfly(model, bias=True):
    if bias:
        return None
    else:   
        p = samples_per_layer(model, bias=bias)
        parameters = [np.hstack([p[i], p[i+1].T]) for i in range(0, len(p), 2)]
    return parameters

def get_model(W, architecture, bias=False, per_layer=True):
    model_sampled = architecture.base(num_classes=10, **architecture.kwargs)
    model_samples = np.array(W)  

    if per_layer:
        for parameter, w in zip(model_sampled.parameters(), W):
            parameter.data.copy_(torch.from_numpy(w))
    else:
        for i, parameter in enumerate(model_sampled.parameters()):
            w = W[i//2]
            if i % 2 == 0:
                offset = 0
            N = parameter.data.shape[1]
            w_part = w[:, offset:offset+N]
            offset = N
            if i % 2 == 0:
                parameter.data.copy_(torch.from_numpy(w_part))
            else:
                parameter.data.copy_(torch.from_numpy(w_part.T))
            

    return model_sampled

def get_model(W, B, architecture):
    model_sampled = architecture.base(num_classes=10, **architecture.kwargs)
    model_samples = np.array(W)  # .cpu().data.numpy()
    SIZE = model_sampled.middle_dim

    offset = 0
    for parameter in list(model_sampled.parameters())[:-1]:
        size = int(np.prod(parameter.size()) / SIZE)
        value = model_samples[:, offset:offset + size]
        if size == 10 or size == 1:
            value = value.T
        value = value.reshape(parameter.size())
        parameter.data.copy_(torch.from_numpy(value))
        offset += size

    list(model_sampled.parameters())[-1].data.copy_(B.mean(0))  # torch.from_numpy(

    return model_sampled

def samples(model):
    p1 = list(model.parameters())[0].data.cpu().numpy()
    p2 = list(model.parameters())[1].data.cpu().numpy()
    p3 = list(model.parameters())[2].transpose(0, 1).data.cpu().numpy()
    samples = np.hstack([p1, p2[:, None], p3])
    return samples

def transform(x1, x2, E12, E22_inv):
#     print(x1.shape, x2.shape, E12.shape, E22_inv.shape)
    y1 = x1 - E12 @ E22_inv @ x2
    return np.concatenate([y1, x2]).T
   
def inv_transform(y1, y2, E12, E22_inv):
#     print(x1.shape, x2.shape, E12.shape, E22_inv.shape)
    x1 = y1 + E12 @ E22_inv @ y2
    return np.concatenate([x1, y2]).T
parameters1 = samples(model1)
parameters2 = samples(model2)

S = np.vstack([parameters1, parameters2])
V2, sigma2 = find_PCA(parameters2.T)
V1, sigma1 = find_PCA(parameters1.T)
V, sigma = find_PCA(S.T)
sigma[:5]
parameters1 = parameters1@V.T*(1/np.sqrt(sigma+1e-8))
parameters2 = parameters2@V.T*(1/np.sqrt(sigma+1e-8))
# parameters1 = <EMAIL>@<EMAIL>*(1/np.sqrt(sigma1))
# parameters2 = parameters2@V2.T*(1/np.sqrt(sigma2))
(parameters1.T@parameters1)[0][:5]
(parameters2.T@parameters2)[0][:5]
from sklearn.decomposition import PCA, KernelPCA

def find_PCA(param):
    
#     U, s2, V = np.linalg.svd(param)
    cov = param@param.T/len(param.T)
    U, sigma, V = np.linalg.svd(cov)

    return V, sigma
#     transPSA = PCA(n_components=K, svd_solver='full', random_state=self.random_state)
#     S = transPSA.fit_transform(S)
# V = find_PCA(parameters1.T)
# parameters = 1/np.sqrt(2)*(parameters1+parameters2)
# parameters1 = samples(model1)
# parameters2 = samples(model2)

N = 5
from pylab import rcParams
fig, ax = plt.subplots(1, N)
rcParams['figure.figsize'] = 15, 15
rcParams['figure.dpi'] = 100
    
V1, sigma1 = find_PCA(parameters1.T)

for ind, im in enumerate(V1[:N, :784]):
    
    im = np.abs(im.reshape((28, 28)))
    ax[ind].imshow(im)
plt.show()

V2, sigma2 = find_PCA(parameters2.T)

fig, ax = plt.subplots(1, N)
rcParams['figure.figsize'] = 15, 15
rcParams['figure.dpi'] = 100

for ind, im in enumerate(V2[:N, :784]):
    
    im = np.abs(im.reshape((28, 28)))
    ax[ind].imshow(im)
plt.show()

fig, ax = plt.subplots(1, N)
rcParams['figure.figsize'] = 15, 15
rcParams['figure.dpi'] = 100

# V = 0.5*(V1+V2)
parameters = (parameters1+parameters2)*(1/2)
V_0, sigma_0 = find_PCA(parameters.T)

for ind, im in enumerate(V_0[:N, :784]):
    
    im = np.abs(im.reshape((28, 28)))
    ax[ind].imshow(im)
plt.show()
sigma1[:5], sigma2[:5], sigma_0[:5]
parameters_check = parameters*2*np.sqrt(sigma+1e-8)@V
_, sigma_check = find_PCA(parameters_check.T)
_, sigma1 = find_PCA(parameters1.T)
sigma_check[:5], sigma1[:5]
# V[0]
model = get_model((parameters*np.sqrt(sigma+1e-8))@V, b2, architecture)
model.cuda();
model.eval();

test_model(model, loaders, cuda=True)
def get_ev(model, index=0):
    W11 = list(model.parameters())[index].data.numpy()
    A = W11.T @ W11
    l, v = np.linalg.eig(A)
    v = np.real(v)
    l = np.real(l)
    return v, l
    

def heat_image(model):
    
    ev, e = get_ev(model)
    N = 5
    fig, ax = plt.subplots(1, N)
    
#     W11 = list(model.parameters())[0].data.numpy()
#     _, _, vh = np.linalg.svd(W11)
#     ev = vh.T @ ev
#     ev = ev>ev.mean()
    
    
    rcParams['figure.figsize'] = 15, 15
    rcParams['figure.dpi'] = 100
    
    for ind, (im, l) in enumerate(zip(ev.T[:N], e[:N])):
        im = np.abs(im.reshape((28, 28)))

        ax[ind].imshow(im)
        ax[ind].set_title(l)
        
#     plt.colorbar()
#     plt.axis('off')
    plt.show()
parameters1 = samples(model1)
parameters2 = samples(model2)

parameters = np.concatenate([parameters1, parameters2])

p = parameters - parameters.mean(0) 
cov = p.T @ p
E12 = cov[:-10, -10:]
E22 = cov[-10:, -10:]
E22_inv = np.linalg.inv(E22)

parameters1_tr = transform(parameters1[:, :-10].T, parameters1[:, -10:].T, E12,  E22_inv)
parameters2_tr = transform(parameters2[:, :-10].T, parameters2[:, -10:].T, E12,  E22_inv)

func = 'arc_connect'
print(func)
cntr = Connector(parameters1_tr, parameters2_tr)
f = getattr(cntr, func)
parameters_res = f()[1]

parameters_res_tr = inv_transform(parameters_res[:, :-10].T, parameters_res[:, -10:].T, E12,  E22_inv)

model = get_model(parameters_res_tr, b2, architecture)
model.cuda();
model.eval();
test_model(model, loaders, cuda=True)
model = get_model(parameters_res_tr, b2, architecture)
model.cuda();
model.eval();
test_model(model, loaders, cuda=True)
model1.cuda();
model1.eval();
test_model(model1, loaders, cuda=True)
param1 = [parameters1[:, :-10].T, parameters1[:, -10:]]
param2 = [parameters2[:, :-10].T, parameters2[:, -10:]]

for func in ['lin_connect', 'arc_connect', 'arc_connect_PCA']:
    parameters_res = []
    print(func)
    for p1, p2 in zip(param1, param2):
        cntr = Connector(p1, p2)
        print(p1.shape)
        f = getattr(cntr, func)
        if 'PCA' in func:
            K = min(300, p1.shape[0], p1.shape[1])
            res = f(K=K)[1]
        elif 'third_cumulant' in func:
            K = min(100, p1.shape[0], p1.shape[1])
            res = f(K=K)[1] 
        else:
            res = f()[1]
            
        print('res', res.shape)
        parameters_res.append(res)
        
    parameters_res = np.hstack([parameters_res[0].T, parameters_res[1]])
    model = get_model(parameters_res, b2, architecture)
    model.cuda();
    model.eval();
    test_model(model, loaders, cuda=True)
func = 'arc_connect'
print(func)
cntr = Connector(parameters1_tr, parameters2_tr)
f = getattr(cntr, func)
parameters_res = f()[1]
parameters_res_tr = inv_transform(parameters_res[:, :-10].T, parameters_res[:, -10:].T, E12,  E22_inv)
model = get_model(parameters_res_tr, b2, architecture)
model.cuda();
model.eval();
test_model(model, loaders, cuda=True)
# parameters_res_tr.shape
param_new = parameters1[:3]+parameters2[3:]
model = get_model(param_new, architecture, per_layer=True)
model.cuda();
model.eval();
test_model(model, loaders, cuda=True)
for func in ['lin_connect', 'arc_connect', 'arc_connect_PCA']:
    parameters_res = []
    print(func)
    for p1, p2 in zip(parameters1, parameters2):
        cntr = Connector(p1, p2)
        f = getattr(cntr, func)
        if 'PCA' in func:
            K = min(300, p1.shape[0], p1.shape[1])
            res = f(K=K)[1]
        elif 'third_cumulant' in func:
            K = min(100, p1.shape[0], p1.shape[1])
            res = f(K=K)[1] 
        else:
            res = f()[1]
        parameters_res.append(res)

    model = get_model(parameters_res, architecture, per_layer=True)
    model.cuda();
    model.eval();
    test_model(model, loaders, cuda=True)<jupyter_output><empty_output><jupyter_text>MNIST PCA<jupyter_code># import models
# architecture = getattr(models, "LinearOneLayer") #LinearOneLayer LogRegression

# import data
# loaders, num_classes = data.loaders(
#     "MNIST",
#     "data",
#     1024,
#     1,
#     "VGG",
#     True)

# import pickle
# data_path = '/home/ivan/dnn-mode-connectivity/data/MNIST_PCA.pickle'
# with open(data_path, 'rb') as handle:
#     dataset, B, transformation = pickle.load(handle)
# b2 = torch.FloatTensor([np.array(B).mean(0)])
# b = np.array(B).mean(0)

# train_dataset, test_dataset = dataset[:-8000], dataset[-8000:]
# modes = [dataset[-4000:-2000], dataset[-2000:]]
test_dataset.shape
# from flows.models import RealNVP, RealNVP_OneLayer_full, IAF_OneLayer_full
# flow = RealNVP_OneLayer_full(in_dim=80, dim_middle=200, N_layers=5, batch_norm=False, data_b2=b, transPSA=transformation)
# flow.load_state_dict(torch.load('/home/ivan/distribution_connector/flow_models/RealNVP/MNIST_PCA_bijection3/checkpoint-2.pt')['model_state'])

# flow.cuda();
# test_flow(flow, architecture, loaders, b2, cntr=None, verbose=True,
#               test_sampling=True, test_flow=False, cuda=True, transPSA=transformation)
# flow.cuda();
# stat_dict, stat_dict_mean, stat_dict_std = gather_statistics(dataset[-8000:], 
#                                                              loaders, b2, architecture, 
#                                                              flow=flow, K=80, t=0.5, 
#                                                              verbose=True, func_name=['flow_connect'],
#                                                              save_stat=True, path='stat/MNIST/bijectionPCA',
#                                                              transPSA=transformation)<jupyter_output><empty_output><jupyter_text>Gavering sample and error time plots on MNIST<jupyter_code>funcs = [func for func in dir(cntr) if callable(getattr(cntr, func)) and 'connect' in func 
         and 'flow' not in func and 'third_cumulant' not in func]
funcs
# for func_name in funcs:
#     t = np.linspace(0,1, 61)[1:-1]
#     stat = test_func(modes, loaders, b, architecture, func_name, flow=None, K=80, t=t, 
#                      show=False, verbose=False, save_pic=True, path='MNIST', save_samples=False,
#                      test_models=True, compute_distribution_stat=False, show_type='hex')
#     t = np.linspace(0,1, 5)[1:-1]
#     stat = test_func(modes, loaders, b, architecture, func_name, flow=None, K=80, t=t, 
#                      show=False, verbose=False, save_pic=False, path='MNIST', save_samples=True)<jupyter_output><empty_output><jupyter_text>flow<jupyter_code># '/home/ivan/distribution_connector/flow_models/Flow_IAF/MNIST_flow/checkpoint-150.pt'
# from flows.models import RealNVP, RealNVP_OneLayer_full, IAF_OneLayer_full
# flow = IAF_OneLayer_full(in_dim=795, dim_middle=795, N_layers=2, batch_norm=False, data_b2=b)
# flow.load_state_dict(torch.load('/home/ivan/distribution_connector/flow_models/Flow_IAF/MNIST_flow/checkpoint-120.pt')['model_state'])


# flow.cuda();

from flows.models import RealNVP, RealNVP_OneLayer_full

flow = RealNVP_OneLayer_full(in_dim=795, dim_middle=2000, N_layers=1, batch_norm=True, data_b2=b)
flow.load_state_dict(torch.load('/home/ivan/distribution_connector/flow_models/RealNVP/MNIST_bijection/checkpoint-20.pt')['model_state'])
flow.cuda();

t = np.linspace(0,1, 61)[1:-1]
stat = test_func(modes, loaders, b2, architecture, 'flow_connect', flow=flow, K=80, t=t, 
                     show=False, verbose=False, save_pic=True, path='MNIST', save_samples=False,
                     test_models=True, compute_distribution_stat=False, show_type='hex')<jupyter_output><empty_output><jupyter_text>Computing mean and std for different methods on MNIST  flow<jupyter_code># checkpoint-20.pt -- top for MNIST bijection
# checkpoint-7 top for CIFAR bijection
# checkpoint-120 top for MNIST Flow IAF 
# '/home/ivan/distribution_connector/flow_models/Flow_IAF/MNIST_flow/checkpoint-150.pt'
# from flows.models import RealNVP, RealNVP_OneLayer_full, IAF_OneLayer_full
# flow = IAF_OneLayer_full(in_dim=795, dim_middle=795, N_layers=2, batch_norm=False, data_b2=b)
# flow.load_state_dict(torch.load('/home/ivan/distribution_connector/flow_models/Flow_IAF/MNIST_flow/checkpoint-120.pt')['model_state'])




# from flows.models import RealNVP, RealNVP_OneLayer_full

# flow = RealNVP_OneLayer_full(in_dim=795, dim_middle=795, N_layers=1, batch_norm=True, data_b2=b)
# flow.load_state_dict(torch.load('/home/ivan/distribution_connector/flow_models/RealNVP/MNIST_flow1/checkpoint-8.pt')['model_state'])


# model = flow
# b = b2
# cuda = True
# verbose = True
# transPSA = None
# model.eval()
# res = model.sample(K=2000).cpu().data.numpy()
# if transPSA is not None:
#     res = transPSA.inverse_transform(res)
# m = get_model(res, b, architecture)
# if cuda:
#     m.cuda()
# result_s = test_model(m, loaders, verbose=verbose, cuda=cuda)
flow.cuda();
test_flow(flow, architecture, loaders, b2, cntr=None, verbose=True,
              test_sampling=True, test_flow=False, cuda=True, transPSA=None)
# flow.cuda();
# stat_dict, stat_dict_mean, stat_dict_std = gather_statistics(dataset[-8000:], 
#                                                              loaders, b2, architecture, 
#                                                              flow=flow, K=80, t=0.5, 
#                                                              verbose=True, func_name=['flow_connect'],
#                                                              save_stat=True, path='stat/MNIST/IAFflow')<jupyter_output><empty_output><jupyter_text>basic<jupyter_code>test_dataset = dataset[-8000:]
funcs = [func for func in dir(cntr) if callable(getattr(cntr, func)) and 'connect' in func 
         and 'flow' not in func and 'third_cumulant' in func]
stat_dict, stat_dict_mean, stat_dict_std = gather_statistics(test_dataset, 
                                                             loaders, b2, architecture, 
                                                             flow=None, K=80, t=0.5, 
                                                             verbose=True, func_name=funcs,
                                                             save_stat=True, path='stat/MNIST/third_cumulant')
stat_dict_mean, stat_dict_std<jupyter_output><empty_output><jupyter_text>CIFAR10<jupyter_code>import models
architecture = getattr(models, "LinearOneLayerCF") #LinearOneLayer LogRegression

import data
loaders, num_classes = data.loaders(
    "CIFAR10",
    "data",
    1024,
    1,
    "VGG",
    False)

model1 = architecture.base(num_classes=10, **architecture.kwargs)
model2 = architecture.base(num_classes=10, **architecture.kwargs)

model1.load_state_dict(torch.load('curves/LinearOneLayer/curve16/checkpoint-400.pt')['model_state'])
model2.load_state_dict(torch.load('curves/LinearOneLayer/curve17/checkpoint-400.pt')['model_state'])

import pickle
data_path = '/home/ivan/dnn-mode-connectivity/data/CIFAR.pickle'
with open(data_path, 'rb') as handle:
    dataset, B = pickle.load(handle)
b2 =  torch.FloatTensor([np.array(B).mean(0)])
b = np.array(B).mean(0)

train_dataset, test_dataset = dataset[:-8000], dataset[-8000:]
modes = [dataset[-4000:-2000], dataset[-2000:]]
parameters1 = samples(model1)
parameters2 = samples(model2)

parameters = np.concatenate([parameters1, parameters2])

cov = parameters.T @ parameters
E12 = cov[:-10, -10:]
E22 = cov[-10:, -10:]
E22_inv = np.linalg.inv(E22)

parameters1_tr = transform(parameters1[:, :-10].T, parameters1[:, -10:].T, E12,  E22_inv)
parameters2_tr = transform(parameters2[:, :-10].T, parameters2[:, -10:].T, E12,  E22_inv)
param1 = [parameters1[:, :-10], parameters1[:, -10:].T]
param2 = [parameters2[:, :-10], parameters2[:, -10:].T]

for func in ['lin_connect', 'arc_connect', 'arc_connect_PCA']:
    parameters_res = []
    print(func)
    for p1, p2 in zip(param1, param2):
        cntr = Connector(p1, p2)
        print(p1.shape)
        f = getattr(cntr, func)
        if 'PCA' in func:
            K = min(300, p1.shape[0], p1.shape[1])
            res = f(K=K)[1]
        elif 'third_cumulant' in func:
            K = min(100, p1.shape[0], p1.shape[1])
            res = f(K=K)[1] 
        else:
            res = f()[1]
            
        print('res', res.shape)
        parameters_res.append(res)
        
    parameters_res = np.hstack([parameters_res[0], parameters_res[1].T])
    model = get_model(parameters_res, b2, architecture)
    model.cuda();
    model.eval();
    test_model(model, loaders, cuda=True)
param1 = [parameters1[:, :-10], parameters1[:, -10:]]
param2 = [parameters2[:, :-10], parameters2[:, -10:]]

for func in ['lin_connect', 'arc_connect', 'arc_connect_PCA']:
    parameters_res = []
    print(func)
    for p1, p2 in zip(param1, param2):
        cntr = Connector(p1, p2)
        print(p1.shape)
        f = getattr(cntr, func)
        if 'PCA' in func:
            K = min(300, p1.shape[0], p1.shape[1])
            res = f(K=K)[1]
        elif 'third_cumulant' in func:
            K = min(100, p1.shape[0], p1.shape[1])
            res = f(K=K)[1] 
        else:
            res = f()[1]
            
        print('res', res.shape)
        parameters_res.append(res)
        
    parameters_res = np.hstack([parameters_res[0], parameters_res[1]])
    model = get_model(parameters_res, b2, architecture)
    model.cuda();
    model.eval();
    test_model(model, loaders, cuda=True)
param1 = [parameters1[:, :-10].T, parameters1[:, -10:]]
param2 = [parameters2[:, :-10].T, parameters2[:, -10:]]

for func in ['lin_connect', 'arc_connect', 'arc_connect_PCA']:
    parameters_res = []
    print(func)
    for p1, p2 in zip(param1, param2):
        cntr = Connector(p1, p2)
        print(p1.shape)
        f = getattr(cntr, func)
        if 'PCA' in func:
            K = min(300, p1.shape[0], p1.shape[1])
            res = f(K=K)[1]
        elif 'third_cumulant' in func:
            K = min(100, p1.shape[0], p1.shape[1])
            res = f(K=K)[1] 
        else:
            res = f()[1]
            
        print('res', res.shape)
        parameters_res.append(res)
        
    parameters_res = np.hstack([parameters_res[0].T, parameters_res[1]])
    model = get_model(parameters_res, b2, architecture)
    model.cuda();
    model.eval();
    test_model(model, loaders, cuda=True)
param1 = [parameters1[:, :-10].T, parameters1[:, -10:].T]
param2 = [parameters2[:, :-10].T, parameters2[:, -10:].T]

for func in ['lin_connect', 'arc_connect', 'arc_connect_PCA']:
    parameters_res = []
    print(func)
    for p1, p2 in zip(param1, param2):
        cntr = Connector(p1, p2)
        print(p1.shape)
        f = getattr(cntr, func)
        if 'PCA' in func:
            K = min(300, p1.shape[0], p1.shape[1])
            res = f(K=K)[1]
        elif 'third_cumulant' in func:
            K = min(100, p1.shape[0], p1.shape[1])
            res = f(K=K)[1] 
        else:
            res = f()[1]
            
        print('res', res.shape)
        parameters_res.append(res)
        
    parameters_res = np.hstack([parameters_res[0].T, parameters_res[1].T])
    model = get_model(parameters_res, b2, architecture)
    model.cuda();
    model.eval();
    test_model(model, loaders, cuda=True)
model1.cuda();
model1.eval();
test_model(model1, loaders, cuda=True)
func = 'arc_connect'
parameters_res = []
print(func)
cntr = Connector(parameters1_tr, parameters2_tr)
f = getattr(cntr, func)
parameters_res = f()[1]
# parameters_res.shape
parameters_res_tr = inv_transform(parameters_res[:, :-10].T, parameters_res[:, -10:].T, E12,  E22_inv)
model = get_model(parameters_res_tr, b2, architecture)
model.cuda();
model.eval();
test_model(model, loaders, cuda=True)<jupyter_output>train results {'nll': 1.4964728350067138, 'loss': 1.4964728350067138, 'accuracy': 51.208}
test results {'nll': 2.692305062484741, 'loss': 2.692305062484741, 'accuracy': 40.93}<jupyter_text>Gavering sample and error time plots on CIFAR10<jupyter_code># for func_name in funcs:
#     t = np.linspace(0,1, 61)[1:-1]
#     stat = test_func(modes, loaders, b2, architecture, func_name, flow=None, K=400, t=t, 
#                      show=False, verbose=False, save_pic=True, path='CIFAR', save_samples=False)
#     t = np.linspace(0,1, 5)[1:-1]
#     stat = test_func(modes, loaders, b2, architecture, func_name, flow=None, K=400, t=t, 
#                      show=False, verbose=False, save_pic=False, path='CIFAR', save_samples=True)<jupyter_output><empty_output><jupyter_text>RNVP flow<jupyter_code># t = np.linspace(0,1, 61)[1:-1]
# stat = test_func(modes, loaders, b2, architecture, func_name='flow_connect', flow=flow, K=400, t=t, 
#                  show=False, verbose=False, save_pic=True, test_models=True, path='CIFAR', save_samples=False)
# t = np.linspace(0,1, 5)[1:-1]
# stat = test_func(modes, loaders, b2, architecture, func_name='flow_connect', flow=flow, K=400, t=t, 
#                  show=False, verbose=False, save_pic=False, path='CIFAR', save_samples=True)<jupyter_output><empty_output><jupyter_text>Computing mean and std for different methods on CIFAR10<jupyter_code>model1.cuda()
test_model(model1, loaders, cuda=True)<jupyter_output>train results {'nll': 0.8489676392555237, 'loss': 0.8489676392555237, 'accuracy': 70.572}
test results {'nll': 1.313883560180664, 'loss': 1.313883560180664, 'accuracy': 58.9}<jupyter_text>flow<jupyter_code>from flows.models import RealNVP, RealNVP_OneLayer_full, IAF_OneLayer_full

flow = RealNVP_OneLayer_full(in_dim=3083, dim_middle=3000, N_layers=1, batch_norm=False, data_b2=b)
flow.load_state_dict(torch.load('/home/ivan/distribution_connector/flow_models/RealNVP/CIFAR_test1/checkpoint-7.pt')['model_state'])

# from flows.models import RealNVP, RealNVP_OneLayer_full, IAF_OneLayer_full
# flow = IAF_OneLayer_full(in_dim=3083, dim_middle=3000, N_layers=1, batch_norm=False, data_b2=b)
# flow.load_state_dict(torch.load('/home/ivan/distribution_connector/flow_models/RealNVP/CIFAR_test1/checkpoint-7.pt')['model_state'])
flow.cuda();
stat_dict, stat_dict_mean, stat_dict_std = gather_statistics(dataset[-8000:], 
                                                             loaders, b2, architecture, 
                                                             flow=flow, K=80, t=0.5, 
                                                             verbose=True, func_name=['flow_connect'],
                                                             save_stat=True, path='stat/CIFAR/bijection')<jupyter_output>0%|          | 0/4 [00:00<?, ?it/s][A[A<jupyter_text>basic<jupyter_code>test_dataset = dataset[-8000:]
funcs = [func for func in dir(cntr) if callable(getattr(cntr, func)) and 'connect' in func 
         and 'flow' not in func and 'third_cumulant' in func]
funcs
stat_dict, stat_dict_mean, stat_dict_std = gather_statistics(test_dataset, 
                                                             loaders, b2, architecture, 
                                                             flow=None, K=200, t=0.5, 
                                                             verbose=True, func_name=funcs,
                                                             save_stat=True, path='stat/CIFAR/third_cumulant')<jupyter_output>0%|          | 0/4 [00:00<?, ?it/s][A<jupyter_text>Checking<jupyter_code>with open('data/stat/CIFAR/third_cumulanterror_stat.pickle', 'rb') as handle:
        load = pickle.load(handle)
load[1]
load[2]
stat_dict_mean, stat_dict_std<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14935632549268912, 'fraction_numerical': 0.043467895740623014, 'mean_word_length': 3.1389574083209997, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '412272', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 9614, 'n_tokens_neox': 9042, 'n_words': 1655, 'path': 'expl-onelayer_exp_PCA.ipynb', 'repo_name': 'avecplezir/dnn-mode-connectivity', 'size': '25169'}