<jupyter_start><jupyter_text>plot mean difference<jupyter_code>n_steps = 17
alphas = th.linspace(-0.5,1.5,n_steps).cuda()
mean_dir = means_per_cluster[1] - means_per_cluster[0]

vals = means_per_cluster[0].unsqueeze(0) + (alphas.unsqueeze(1) * mean_dir.unsqueeze(0))

inverted = invert(feature_model, vals).squeeze()


alphas =np.linspace(0,1,len(inverted))
colors = (1 - alphas)[:,None] * np.array(seaborn.color_palette()[0])[None] + (
    alphas[:,None] * np.array(seaborn.color_palette()[1])[None])
plt.figure(figsize=(12,6))
for i_line in range(len(colors)):
    lw = 0.7
    if i_line in [4,17-4]:
        lw=2
    label = ''
    if i_line == 4:
        label = 'Right'
    elif i_line == (17-4):
        label = 'Rest'
    plt.plot(var_to_np(inverted)[i_line], color=colors[i_line], lw=lw, label=label);
plt.title("Morph between two class means")
plt.legend()
n_steps = 17
alphas = np.linspace(-0.5,1.5,n_steps)
mean_dir = np.mean(var_to_np(inputs[1]).squeeze(), axis=0) - np.mean(var_to_np(inputs[0]).squeeze(), axis=0)

inverted = np.mean(var_to_np(inputs[0]).squeeze(), axis=0)[None] + (alphas[:,None] * mean_dir[None])


alphas =np.linspace(0,1,len(inverted))
colors = (1 - alphas)[:,None] * np.array(seaborn.color_palette()[0])[None] + (
    alphas[:,None] * np.array(seaborn.color_palette()[1])[None])
plt.figure(figsize=(12,6))
for i_line in range(len(colors)):
    lw = 0.7
    if i_line in [4,17-4]:
        lw=2
    label = ''
    if i_line == 4:
        label = 'Right'
    elif i_line == (17-4):
        label = 'Rest'
    plt.plot(inverted[i_line], color=colors[i_line], lw=lw, label=label);
plt.title("Morph between two class means in input space")
plt.legend()
plt.figure(figsize=(6,2))

for i_class in range(len(means_per_cluster)):
    mean = means_per_cluster[i_class]
    inverted = invert(feature_model, get_gauss_samples(1, mean, mean*0)).squeeze()
    plt.plot(var_to_np(inverted),
            color=seaborn.color_palette()[i_class],
            label=['Right', 'Rest'][i_class] + ' Fake Mode')

    data_mean = np.mean(var_to_np(inputs[i_class]), axis=0).squeeze()
    plt.plot(data_mean,
            color=seaborn.color_palette()[i_class],
             ls='--', label=['Right', 'Rest'][i_class] + ' Real Mean')
plt.legend(bbox_to_anchor=(1,1,0,0))
plt.title("Compare inverted modes and real means")
part_df = df.loc[[0,2000,4000],['sampled_wd0', 'sampled_wd1', 'wd_diff0', 'wd_diff1', '1nn_train']]
display(part_df)

for i_epoch in [2000,4000]:
    print("Epoch {:d}".format(i_epoch))
    print('Sampled_wd: {:.2f}'.format(np.mean(np.mean(df.loc[i_epoch-5:i_epoch, ['sampled_wd0', 'sampled_wd1']]))))
    print('wd_diff: {:.2f}'.format(np.mean(np.mean(df.loc[i_epoch-5:i_epoch, ['wd_diff0', 'wd_diff1']]))))
    print('1nn: {:.1f}%'.format(np.mean(np.mean(df.loc[i_epoch-5:i_epoch, ['1nn_train',]])) * 100))
    print("")

for i_epoch in [8000,10000,12000]:
    print("Epoch {:d}".format(i_epoch))
    print('Sampled_wd: {:.2f}'.format(np.mean(np.mean(df.loc[i_epoch-5:i_epoch, ['sampled_wd0', 'sampled_wd1']]))))
    print('wd_diff: {:.2f}'.format(np.mean(np.mean(df.loc[i_epoch-5:i_epoch, ['wd_diff0', 'wd_diff1']]))))
    print('1nn: {:.1f}%'.format(np.mean(np.mean(df.loc[i_epoch-5:i_epoch, ['1nn_train',]])) * 100))
    print("")

df.plot()
plt.ylim(-1,5)
df.loc[:,['sampled_wd0', 'sampled_wd1']].plot()
plt.ylim(-1,5)
for i_class in range(2):
    plt.figure(figsize=(8,5))

    mean = means_per_cluster[i_class]
    std = th.exp(stds_per_cluster[i_class])
    samples = get_gauss_samples(40, mean, std)
    inverted = invert(feature_model, samples)

    plt.plot(var_to_np(inverted.squeeze()).T, lw=1, alpha=0.7,
            color=seaborn.color_palette()[i_class], );
    plt.plot(var_to_np(inputs[i_class].squeeze()).T,  lw=1, alpha=0.7,
            color='black',);<jupyter_output><empty_output><jupyter_text>semidual  match ring to gaussian<jupyter_code>angles = np.linspace(0,2*np.pi,endpoint=False, num=8)
x = np.cos(angles)
y = np.sin(angles)

plt.figure(figsize=(5,5))
plt.plot(x,y, marker='o', ls ='')

xs = np.array([x,y]).T
rng = RandomState(20192501)

v = np.zeros(len(xs))
v_avg = np.zeros(len(v))
n = 1000
lr = 10
for k in range(1,100):
    a = rng.randn(1,2)

    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v
    i_min_diff = np.argmin(diffs)
    this_lr = lr / np.sqrt(k)
    v[i_min_diff] -= (1 * this_lr)
    v += ((1/len(v)) * this_lr)
    v_avg = (v) * (1/k) + ((k-1) / n) * v_avg

v
plt.figure(figsize=(5,5))
for k in range(1,500):
    a = rng.randn(1,2)
    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v_avg
    i_min_diff = np.argmin(diffs)
    plt.plot(a[0,0], a[0,1], color=seaborn.color_palette()[i_min_diff], ls='', marker='o')

plt.figure(figsize=(5,5))
for k in range(1,500):
    a = rng.randn(1,2)
    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v_avg
    i_min_diff = np.argmin(diffs)
    plt.plot(a[0,0], a[0,1], color=seaborn.color_palette()[i_min_diff], ls='', marker='o')

xs = np.array([np.zeros(7),np.linspace(-1,1,7)]).T
rng = RandomState(20192501)

v = np.zeros(len(xs))
v_avg = np.zeros(len(v))
n = 100000
lr = 1000
for k in range(1,n):
    a = rng.randn(1,2)

    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v
    i_min_diff = np.argmin(diffs)
    this_lr = lr / np.sqrt(k)
    v[i_min_diff] -= (1 * this_lr)
    v += ((1/len(v)) * this_lr)
    v_avg = (v) * (1/k) + ((k-1) / n) * v_avg

v_avg
plt.figure(figsize=(5,5))
for k in range(1,500):
    a = rng.randn(1,2)
    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v_avg
    i_min_diff = np.argmin(diffs)
    plt.plot(a[0,0], a[0,1], color=seaborn.color_palette()[i_min_diff], ls='', marker='o')

v
plt.figure(figsize=(5,5))
for k in range(1,500):
    a = rng.rand(1,2)
    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v_avg
    i_min_diff = np.argmin(diffs)
    plt.plot(a[0,0], a[0,1], color=seaborn.color_palette()[i_min_diff], ls='', marker='o')

xs = np.array([np.zeros(7),np.linspace(-1,1,7)]).T
rng = RandomState(20192501)

v = np.zeros(len(xs))
v_avg = np.zeros(len(v))
n = 10000
lr = 1
for k in range(1,n):
    a = rng.rand(1,2)

    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v
    i_min_diff = np.argmin(diffs)
    this_lr = lr / np.sqrt(k)
    v[i_min_diff] -= (1 * this_lr)
    v += ((1/len(v)) * this_lr)
    v_avg = ((v) * (1/k)) + (((k-1) / n) * v_avg)

v_avg
bincount = np.zeros(len(xs))
plt.figure(figsize=(5,5))
for k in range(1,1500):
    a = rng.rand(1,2)
    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v
    i_min_diff = np.argmin(diffs)
    bincount[i_min_diff] += 1
    plt.plot(a[0,0], a[0,1], color=seaborn.color_palette()[i_min_diff], ls='', marker='o')
bincount
for i_class in range(2):
    mean = means_per_cluster[i_class]
    std = th.exp(stds_per_cluster[i_class])
    samples = get_gauss_samples(40, mean, std)
    inverted = invert(feature_model, samples)

    plt.plot(var_to_np(inverted.squeeze()).T, lw=1, alpha=0.7,
            color=seaborn.color_palette()[i_class], );
    plt.plot(var_to_np(inputs[i_class].squeeze()).T,  lw=1, alpha=0.7,
            color='black',);
i_class = 0
xs = var_to_np(inputs[i_class].squeeze())
rng = RandomState(20192501)
mean = means_per_cluster[i_class]
std = th.exp(stds_per_cluster[i_class])
v = np.zeros(len(xs))
v_avg = np.zeros(len(v))
n = 10000
lr = 1
for k in range(1,n):
    a = var_to_np(get_gauss_samples(1, mean, std).squeeze())

    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v
    i_min_diff = np.argmin(diffs)
    this_lr = lr / np.sqrt(k)
    v[i_min_diff] -= (1 * this_lr)
    v += ((1/len(v)) * this_lr)
    v_avg = ((v) * (1/k)) + (((k-1) / n) * v_avg)

v_avg
bincount = np.zeros(len(xs))
matches = [[] for _ in range(20)]
plt.figure(figsize=(5,5))
for k in range(1,1500):
    a = var_to_np(get_gauss_samples(1, mean, std).squeeze())
    diffs =  np.sqrt(np.sum(np.square(a - xs), axis=1))
    diffs = diffs - v
    i_min_diff = np.argmin(diffs)
    bincount[i_min_diff] += 1
    matches[i_min_diff].append(a)
bincount
for reference, match in zip(xs, matches):
    plt.figure(figsize=(12,4))
    plt.plot(np.array(match).T, color=seaborn.color_palette()[0]);
    plt.plot(reference, color='black')
mean = means_per_cluster[i_class]
std = th.exp(stds_per_cluster[i_class])
samples = get_gauss_samples(5000, mean, std)
inverted = var_to_np(invert(feature_model, samples).squeeze())
ins = var_to_np(inputs[i_class].squeeze())

dist_matrix = np.sqrt(np.sum(np.square(ins[:,None] - inverted[None]), axis=2))

coupling = ot.emd([],[], dist_matrix)

for i_in in range(len(ins)):
    plt.figure(figsize=(12,4))
    mask = coupling[i_in] > 1 / (len(inverted) + 1)
    matched_samples = inverted[mask]
    reference = ins[i_in]
    plt.plot(np.array(matched_samples).T, color=seaborn.color_palette()[0]);
    plt.plot(reference, color='black')
    
plt.plot(np.array(matched_samples).T, color=seaborn.color_palette()[0]);
dist_matrix.shape
np.sum(mask)
1 / 0.00066667
np.array(matches[0]).shape
v_avg
def optimize_v_optimizer(v, optim_v, outs, sample_fn, max_iters=150):
    avg_v = th.zeros_like(v.data)
    for i_update in range(max_iters):
        samples = sample_fn()
        diffs = th.sum((outs.unsqueeze(dim=1) - samples.unsqueeze(dim=0)) ** 2, dim=2)
        min_diffs, _ = th.min(diffs - v.unsqueeze(1), dim=0)
        loss = -(th.mean(min_diffs) + th.mean(v))
        optim_v.zero_grad()
        loss.backward()
        optim_v.step()
        v.data = v.data - th.mean(v.data)
        k = i_update + 1
        avg_v = ((k-1) / k) * avg_v + (1/k) * v.data
    return i_update, avg_v<jupyter_output><empty_output><jupyter_text>map<jupyter_code>for i_class in range(len(inputs)):
    y = np_to_var([i_class]).cuda()
    mean = means_per_cluster[i_class]
    std = th.exp(stds_per_cluster[i_class])
    this_inputs = inputs[i_class]
    if y == 0:
        inverted = np_to_var(x_rest[20:,0:1,:,None], dtype=np.float32).cuda()
    else:
        inverted = np_to_var(x_right[20:,0:1,:,None], dtype=np.float32).cuda()

    in_np = var_to_np(this_inputs).squeeze()
    fake_np = var_to_np(inverted).squeeze()
    import ot

    dist = np.sum(np.abs(in_np[:,None] - fake_np[None]), axis=2)
    match_matrix = ot.emd([],[], dist)
    cost = np.sum(dist * match_matrix)
    print(cost)
    score_fake = adv_model(inverted, y)
    score_real = adv_model(inputs[i_class], y)
    wd_dist = th.mean(score_real) - th.mean(score_fake)
    print(wd_dist)
for i_class in range(len(inputs)):
    y = np_to_var([i_class]).cuda()
    mean = means_per_cluster[i_class]
    std = th.exp(stds_per_cluster[i_class])
    if y == 0:
        this_inputs = np_to_var(x_rest[20:,0:1,:,None], dtype=np.float32).cuda()
    else:
        this_inputs = np_to_var(x_right[20:,0:1,:,None], dtype=np.float32).cuda()
    samples = get_gauss_samples(20, mean, std)

    inverted = invert(feature_model, samples)
    in_np = var_to_np(this_inputs).squeeze()
    fake_np = var_to_np(inverted).squeeze()
    import ot

    dist = np.sum(np.abs(in_np[:,None] - fake_np[None]), axis=2)
    match_matrix = ot.emd([],[], dist)
    cost = np.sum(dist * match_matrix)
    print(cost)
    score_fake = adv_model(inverted, y)
    score_real = adv_model(inputs[i_class], y)
    wd_dist = th.mean(score_real) - th.mean(score_fake)
    print(wd_dist)
for i_class in range(len(inputs)):
    mean = means_per_cluster[i_class]
    std = th.exp(stds_per_cluster[i_class])
    this_inputs = inputs[i_class]
    samples = get_gauss_samples(2000, mean, std)

    inverted = invert(feature_model, samples)
    in_np = var_to_np(this_inputs).squeeze()
    fake_np = var_to_np(inverted).squeeze()
    import ot

    dist = np.sum(np.abs(in_np[:,None] - fake_np[None]), axis=2)
    match_matrix = ot.emd([],[], dist)
    cost = np.sum(dist * match_matrix)
    print(cost)
    score_fake = adv_model(inverted)
    score_real = adv_model(inputs[i_class])
    wd_dist = th.mean(score_real) - th.mean(score_fake)
    print(wd_dist)
for i_class in range(len(inputs)):
        mean = means_per_cluster[1-i_class]
        std = th.exp(stds_per_cluster[1-i_class])
        samples = get_gauss_samples(2000, mean, std)

        inverted = invert(feature_model, samples)
        in_np = var_to_np(inputs[i_class]).squeeze()
        fake_np = var_to_np(inverted).squeeze()
        import ot

        dist = np.sum(np.abs(in_np[:,None] - fake_np[None]), axis=2)
        match_matrix = ot.emd([],[], dist)
        cost = np.sum(dist * match_matrix)
        print(cost)
        score_fake = adv_model(inverted)
        score_real = adv_model(inputs[i_class])
        wd_dist = th.mean(score_real) - th.mean(score_fake)
        print(wd_dist)
means_per_cluster
th.exp(th.stack(stds_per_cluster))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1851335656213705, 'fraction_numerical': 0.033372048006194345, 'mean_word_length': 3.837453183520599, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '220768', 'license': 'MIT', 'n_tokens_mistral': 5757, 'n_tokens_neox': 5559, 'n_words': 867, 'path': 'notebooks/BhNo_from_2_to_64_samples/.ipynb_checkpoints/Input_Adv_4_Samples_2_Class_Proj-checkpoint.ipynb', 'repo_name': 'robintibor/reversible2', 'size': '12915'}