<jupyter_start><jupyter_text>Non-private Training<jupyter_code>info, model = run_experiment(1.0, 200, add_noise=False, throw_out_threshold=False)
accuracy(model, X_test, y_test)
lossfn = nn.NLLLoss()

avg_train_l = lossfn(torch.squeeze(model.forward(torch.from_numpy(X_train).float())),
                    torch.from_numpy(y_train).long())

avg_test_l = lossfn(torch.squeeze(model.forward(torch.from_numpy(X_test).float())),
                    torch.from_numpy(y_test).long())

print('average train loss:', avg_train_l)
print('average test loss:', avg_test_l)
def run_membership_inference_attack_thresh(model, avg_train_l, X_target, y_target, thresh):
    _, sensitivities = grad_immediate_sensitivity(model, 
                                                      nn.NLLLoss(), 
                                                      torch.from_numpy(X_target).float(), 
                                                      torch.from_numpy(y_target).long(), 
                                                      None)
    max_sen = max(sensitivities)
    norm_sen = [s/max_sen for s in sensitivities]
    min_exp = min([np.log(s) for s in sensitivities if s != 0])
    log_sen = [np.log(s) if s !=0 else min_exp - 1 for s in sensitivities]
    paws = [s - (min_exp - 1) for s in log_sen]
    
    pass_inf, train_loss = membership_inf(model, 
                                          avg_train_l + thresh, 
                                          torch.from_numpy(X_target).float(), 
                                          torch.from_numpy(y_target).long())
    #plt.scatter(paws, pass_inf)
    #print('positive ratio:',sum(pass_inf)/len(pass_inf))
    return sum(pass_inf)/len(pass_inf)
for t in np.linspace(0,2, 20):
    tpr = run_membership_inference_attack_thresh(model, avg_train_l, X_train, y_train, t)
    fpr = run_membership_inference_attack_thresh(model, avg_train_l, X_test, y_test, t)
    print(t, 'Adv:', tpr - fpr)
plt.figure()

ax1 = plt.gca()
ax1.plot(info['train_l'], color='b', label='Training Loss')
ax1.plot(info['test_l'], color='r', label='Test Loss')
ax1.legend(loc='center right')
ax1.set_ylabel('Loss', color='b')

ax2 = ax1.twinx()
ax2.plot(info['adv'], color='g')
ax2.set_ylabel('Membership Inference Adv', color='g')

#fig.tight_layout()
ax1.set_xlabel('Epoch')
plt.show()<jupyter_output><empty_output><jupyter_text>Differentially Private Training<jupyter_code>info, model = run_experiment(1.0, 200, add_noise=True, throw_out_threshold=False)#.04)
plt.figure()

ax1 = plt.gca()
ax1.plot(info['train_l'], color='b', label='Training Loss')
ax1.plot(info['test_l'], color='r', label='Test Loss')
ax1.legend(loc='center right')
ax1.set_ylabel('Loss', color='b')

ax2 = ax1.twinx()
ax2.plot(info['adv'], color='g')
ax2.set_ylabel('Membership Inference Adv', color='g')

#fig.tight_layout()
ax1.set_xlabel('Epoch')
plt.show()<jupyter_output><empty_output><jupyter_text>Threshold Training<jupyter_code>info, model = run_experiment(1.0, 200, add_noise=False, throw_out_threshold=.03)
plt.figure()

ax1 = plt.gca()
ax1.plot(info['train_l'], color='b', label='Training Loss')
ax1.plot(info['test_l'], color='r', label='Test Loss')
ax1.legend(loc='center right')
ax1.set_ylabel('Loss', color='b')

ax2 = ax1.twinx()
ax2.plot(info['adv'], color='g')
ax2.set_ylabel('Membership Inference Adv', color='g')

#fig.tight_layout()
ax1.set_xlabel('Epoch')
plt.show()<jupyter_output><empty_output><jupyter_text>Both<jupyter_code>info, model = run_experiment(1.0, 200, add_noise=True, throw_out_threshold=1.0)
plt.figure()

ax1 = plt.gca()
ax1.plot(info['train_l'], color='b', label='Training Loss')
ax1.plot(info['test_l'], color='r', label='Test Loss')
ax1.legend(loc='center right')
ax1.set_ylabel('Loss', color='b')

ax2 = ax1.twinx()
ax2.plot(info['adv'], color='g')
ax2.set_ylabel('Membership Inference Adv', color='g')

#fig.tight_layout()
ax1.set_xlabel('Epoch')
plt.show()<jupyter_output><empty_output><jupyter_text>Full Experiment Code<jupyter_code>def run_one_experiment(epsilon, epochs, add_noise, throw_out_threshold):
    info, model = run_experiment(epsilon, epochs, 
                                 add_noise=add_noise, 
                                 throw_out_threshold=throw_out_threshold,
                                 logging=False)
    acc = accuracy(model, X_test, y_test)
    
    avg_train_l = nn.NLLLoss()(torch.squeeze(model.forward(torch.from_numpy(X_train).float())),
                               torch.from_numpy(y_train).long())
    tpr = run_membership_inference_attack(model, avg_train_l, X_train, y_train)
    fpr = run_membership_inference_attack(model, avg_train_l, X_test, y_test)
    adv = tpr - fpr
    
    return acc, adv
run_one_experiment(1.0, 50, False, False)
non_private_results = [run_one_experiment(1.0, 50, False, False) for _ in range(5)]
accs, advs = list(zip(*non_private_results))
print(f'Accuracy: {np.mean(accs)} +- {np.std(accs)}')
print(f'Advantage: {np.mean(advs)} +- {np.std(advs)}')
def plot_noise_results(epsilons, acc, acc_std, adv, adv_std):
    plt.figure()

    ax1 = plt.gca()
    markers, caps, bars = ax1.errorbar(epsilons, 
                                       acc, 
                                       acc_std, 
                                       color='b', 
                                       alpha=.8)
    [bar.set_alpha(0.25) for bar in bars]
    [cap.set_alpha(0.25) for cap in caps]
    ax1.set_ylabel('Accuracy', color='b')
    ax1.tick_params('x', colors='b')

    ax2 = ax1.twinx()
    markers, caps, bars = ax2.errorbar(epsilons, 
                                       adv, 
                                       adv_std, 
                                       color='r')
    [bar.set_alpha(0.25) for bar in bars]
    [cap.set_alpha(0.25) for cap in caps]
    ax2.set_ylabel('Membership Inference Adv', color='r')
    ax2.tick_params('x', colors='r')

    ax1.set_xlabel('Epsilon (delta = 1e-5)')
    #fig.tight_layout()
    
    plt.xscale('log')
    plt.show() 
epsilons = [1, 10, 100, 1000]
num_runs = 5
num_epochs = 50

acc_mean = []
acc_std = []
adv_mean = []
adv_std = []

for epsilon in epsilons:
    print('Epsilon', epsilon)

    exp_results = [run_one_experiment(epsilon, num_epochs, True, False) for _ in range(num_runs)]
    accs, advs = list(zip(*exp_results))
    acc_mean.append(np.mean(accs))
    acc_std.append(np.std(accs))
    adv_mean.append(np.mean(advs))
    adv_std.append(np.std(advs))
plot_noise_results(epsilons, acc_mean, acc_std, adv_mean, adv_std)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14649095923996322, 'fraction_numerical': 0.01854121973643886, 'mean_word_length': 3.064134495641345, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '400830', 'license': 'MIT', 'n_tokens_mistral': 2473, 'n_tokens_neox': 2352, 'n_words': 389, 'path': 'experiments/immediate_sensitivity/Synthetic Classification.ipynb', 'repo_name': 'twosixlabs/csl', 'size': '6526'}