<jupyter_start><jupyter_text>Correlations by distance<jupyter_code>xc = np.loadtxt("all_btm_xcorrs10", delimiter=",")
dists = xc[:, 2]
xcorrs = xc[:, 5]
xcorr_envs = xc[:, 7]
plt.scatter(dists, xcorrs, alpha=0.1)
plt.title("10s P window")
xc = np.loadtxt("all_btm_xcorrs30", delimiter=",")
dists = xc[:, 2]
xcorrs = xc[:, 5]
xcorr_envs = xc[:, 7]
plt.scatter(dists, xcorrs, alpha=0.1)
plt.title("30s P window")<jupyter_output><empty_output><jupyter_text>Picking out correlated events for jointGP testing<jupyter_code>xc = np.loadtxt("all_btm_xcorrs10", delimiter=",")
evid1 = xc[:, 0]
evid2 = xc[:, 1]
xcorrs = xc[:, 5]

correlated = xcorrs > 0.9
correlated_evids = np.concatenate((evid1[correlated], evid2[correlated]))
from collections import defaultdict
c = defaultdict(int)
for evid in correlated_evids:
    c[evid] += 1

counts = sorted(c.items(), key = lambda (k, v): v)
print counts
top_evids = [evid for (evid, c) in counts if c >= 3]


with open("correlated_btm_evids", "w") as f:
    f.write("\n".join(["%d" % evid for evid in top_evids]))

all_btm_evids = set(np.concatenate((evid1, evid2)))

with open("all_btm_evids", "w") as f:
    f.write("\n".join(["%d" % evid for evid in all_btm_evids]))

len(all_btm_evids)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1745019920318725, 'fraction_numerical': 0.026294820717131476, 'mean_word_length': 5.977777777777778, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '557187', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 571, 'n_tokens_neox': 544, 'n_words': 118, 'path': 'notebooks/thesis/correlations_at_black_thunder.ipynb', 'repo_name': 'davmre/sigvisa', 'size': '1255'}