<jupyter_start><jupyter_text>Parameter optimization<jupyter_code>alpha = 0.01

op = {'n_test_locs': 1, 'seed': seed+11, 'max_iter': 500, 
     'batch_proportion': 1.0, 'locs_step_size': 50.0, 
      'gwidth_step_size': 0.1, 'tol_fun': 1e-4}
# optimize on the training set
test_locs, gwidth, info = tst.MeanEmbeddingTest.optimize_locs_width(tr, alpha, **op)
plt.plot(info['obj_values'])
plt.xlabel('iteration')
plt.ylabel('Objective')
xtr, ytr = tr.xy()
xmean = np.mean(xtr, 0)
ymean = np.mean(ytr, 0)
mean = np.mean(tr.stack_xy(), 0)

xm_norm = np.abs(xmean)/np.linalg.norm(xmean, ord=1)
ym_norm = np.abs(ymean)/np.linalg.norm(ymean, ord=1)
m_norm = np.abs(mean)/np.linalg.norm(mean, ord=1)

plt.plot(mean)
t = test_locs[0]
#t_norm = t
t_norm = t/np.linalg.norm(t, ord=1)
score = np.abs(t_norm)
#score = (t_norm)
sind = np.argsort(-score)
plt.plot(score[sind])
for i, w in enumerate(terms[sind]):
    if i%10==0:
        print
    print w,<jupyter_output><empty_output><jupyter_text>Two-sample test<jupyter_code># test with the best Gaussian with 
#test_locs = tst.MeanEmbeddingTest.init_locs_2randn(tr, 2, seed=28)
#gwidth = 10
met = tst.MeanEmbeddingTest(test_locs, gwidth, alpha)
met.perform_test(te)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1678832116788321, 'fraction_numerical': 0.028386050283860504, 'mean_word_length': 5.529100529100529, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '657663', 'license': 'MIT', 'n_tokens_mistral': 584, 'n_tokens_neox': 543, 'n_words': 96, 'path': 'ipynb/text_experiment.ipynb', 'repo_name': 'cocoaaa/interpretable-test', 'size': '1233'}