<jupyter_start><jupyter_text>Load data<jupyter_code>X_list, Y, Yaudio = pickle.load(open('../data/lda_data_8.pickle','rb'))
X = np.concatenate(X_list, axis=1)
print X.shape<jupyter_output>(8200, 381)<jupyter_text>Pairwise distances<jupyter_code>D = pairwise_distances(X, metric='mahalanobis')
plt.hist(D.ravel(), bins=100);<jupyter_output><empty_output><jupyter_text>n-occurrence and stats<jupyter_code>def n_occurrence_from_D(D, k=10, n_items=None):
    if n_items is None:
        n_items = len(D)
    sort_idx = np.argsort(D, axis=1)
    D_k = sort_idx[:, 1:(k+1)]  # nearest neighbour is the item itself
    N_k = np.bincount(D_k.astype(int).ravel(), minlength=n_items)
    return N_k
# k = average number of samples per country
uniq_countries, uniq_counts = np.unique(Y, return_counts=True)
k = np.int(np.round(np.mean(uniq_counts)))
print k
N_k = n_occurrence_from_D(D, k=k)

print 'hubness', skew(N_k)
print 'median', np.median(N_k)
print 'std', np.std(N_k)
print 'hubs', len(np.where(N_k>1000)[0]), len(np.where(N_k>1000)[0]) / float(len(N_k))
print 'orphans', len(np.where(N_k==0)[0]), len(np.where(N_k==0)[0]) / float(len(N_k))
plt.figure()
plt.hist(N_k, bins=100);

plt.figure()
plt.plot(np.sort(N_k))<jupyter_output><empty_output><jupyter_text>Large hubs<jupyter_code>large_hubs_idx = np.where(N_k>1000)[0]
print len(large_hubs_idx)
labs, counts = np.unique(Y[large_hubs_idx], return_counts=True)

plt.figure(figsize=(15, 5))
plt.bar(range(len(labs)), counts, align='center');
plt.xticks(range(len(labs)), labs, rotation=90);
plt.ylabel('Count')
plt.xlabel('Country origins for recordings identified as large hubs')<jupyter_output>129
{'fraction_non_alphanumeric': 0.17253948967193194, 'fraction_numerical': 0.028554070473876064, 'mean_word_length': 7.276381909547739, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '195242', 'license': 'MIT', 'n_tokens_mistral': 752, 'n_tokens_neox': 702, 'n_words': 111, 'path': 'notebooks/hubness.ipynb', 'repo_name': 'hideodaikoku/music-outliers', 'size': '1646'}