<jupyter_start><jupyter_text>Test<jupyter_code>AD_distances_rw_noisy = []
n_synth_samples = 200
observed_AD = fit_AD.power_law.Asquare
data = np.array(data)
data_noisy = np.hstack((np.array(data),data[data>700]))
fit_AD_noisy = powerlaw.Fit(data_noisy, discrete = True, xmin_distance='Asquare')
fit_AD_noisy.power_law.alpha, fit_AD_noisy.power_law.xmin, fit_AD_noisy.power_law.Asquare
plt.style.use('seaborn')
fig, ax = plt.subplots(figsize=(7,7))
powerlaw.plot_pdf(data_noisy, color = 'deeppink', lw = 3, label = 'Noisy data')
powerlaw.plot_pdf(data, color = 'black', lw = 3, label = 'True power-law data')
plt.legend(fontsize = 22, bbox_to_anchor=(0.76, 1.2))
plt.xlabel('$\log(k)$', fontsize = 25)
plt.ylabel('$\log\mathbb{P}(k)$', fontsize = 25)
for tick in ax.xaxis.get_major_ticks():
                tick.label.set_fontsize(25) 
for tick in ax.yaxis.get_major_ticks():
            tick.label.set_fontsize(25)           
plt.savefig('empirical_density_plus_noisy.pdf', dpi=200, bbox_inches='tight')
n = float(len(data_noisy))
n_tail_AD_sample = float(len(data_noisy[data_noisy>=fit_AD_noisy.power_law.xmin]))
non_pl_AD_sample = data_noisy[data_noisy<fit_AD_noisy.power_law.xmin]
B_AD = bernoulli(n_tail_AD_sample/n)

m = 0

while m<n_synth_samples:
    bern_AD = B_AD.rvs(size = len(data_noisy))
    AD_distances_rw_noisy.append(powerlaw.Fit(np.hstack((fit_AD_noisy.power_law.generate_random(n = len(bern_AD[bern_AD==1]),
                                                                                         estimate_discrete = True),
                                                        np.random.choice(non_pl_AD_sample,
                                                                         len(bern_AD[bern_AD==0]),
                                                                         replace=True))),
                                             discrete = True,xmin_distance = 'Asquare').power_law.Asquare)

    m = m+1
AD_distances_rw_noisy = np.array(AD_distances_rw_noisy)
observed_AD_noisy = fit_AD_noisy.power_law.Asquare
fig, ax = plt.subplots(figsize=(7,7))
sn.distplot(AD_distances_rw_noisy,color = 'darkblue', kde = False,label = 'Simulated AD distances')
plt.axvline(observed_AD_noisy,0,50, ls = '--',lw = 4, color = 'deeppink',label = 'AD distance on noisy data')
# plt.title('p-value = {}'.format(float(len(AD_distances_rw_noisy[AD_distances_rw_noisy>observed_AD_noisy]))/float(len(AD_distances_rw_noisy))),
#          fontsize = 16)
plt.legend(fontsize = 22, bbox_to_anchor=(-0.05, 1.2), loc='upper left')
plt.xlabel("AD distance", fontsize = 25)
plt.ylabel("Frequency", fontsize = 25)
for tick in ax.xaxis.get_major_ticks():
                tick.label.set_fontsize(25) 
for tick in ax.yaxis.get_major_ticks():
            tick.label.set_fontsize(25)     
plt.text(0.017, 45, r'$p$ = {}'.format(float(len(AD_distances_rw_noisy[AD_distances_rw_noisy>observed_AD_noisy]))/float(len(AD_distances_rw_noisy))),
          fontsize = 25, color='deeppink')
plt.savefig('AD_noisy.pdf', dpi=200, bbox_inches='tight')
fit_AD = powerlaw.Fit(data, discrete = True, xmin_distance='Asquare')
fit_AD.power_law.alpha, fit_AD.power_law.xmin, fit_AD.power_law.Asquare
AD_distances_rw = []
n_synth_samples = 200
observed_AD = fit_AD.power_law.Asquare
data = np.array(data)
n = float(len(data))
n_tail_AD_sample = float(len(data[data>=fit_AD.power_law.xmin]))
non_pl_AD_sample = data[data<fit_AD.power_law.xmin]
B_AD = bernoulli(n_tail_AD_sample/n)

m = 0

while m<n_synth_samples:
    bern_AD = B_AD.rvs(size = len(data))
    AD_distances_rw.append(powerlaw.Fit(np.hstack((fit_AD.power_law.generate_random(n = len(bern_AD[bern_AD==1]),
                                                                                         estimate_discrete = True),
                                                        np.random.choice(non_pl_AD_sample,
                                                                         len(bern_AD[bern_AD==0]),
                                                                         replace=True))),
                                             discrete = True,xmin_distance = 'Asquare').power_law.Asquare)

    m = m+1
AD_distances_rw = np.array(AD_distances_rw)
fig, ax = plt.subplots(figsize=(7,7))
sn.distplot(AD_distances_rw,color = 'darkblue', kde = False,label = 'Simulated AD distances')
plt.axvline(observed_AD,0,50, ls = '--', lw = 4,color = 'black',label = 'AD distance on true power-law data')
# plt.title('p-value = {}'.format(float(len(AD_distances_rw[AD_distances_rw>observed_AD]))/float(len(AD_distances_rw))),
#          fontsize = 16)
plt.legend(fontsize = 22, bbox_to_anchor=(-0.05, 1.2), loc='upper left')
plt.xlabel("AD distance", fontsize = 25)
plt.ylabel("Frequency", fontsize = 25)
#plt.xticks([0.004, 0.008, 0.012, 0.016])
for tick in ax.xaxis.get_major_ticks():
                tick.label.set_fontsize(25) 
for tick in ax.yaxis.get_major_ticks():
            tick.label.set_fontsize(25) 
plt.text(0.02, 32, r'$p$ = {}'.format(float(len(AD_distances_rw[AD_distances_rw>observed_AD]))/float(len(AD_distances_rw))),
          fontsize = 25)
plt.savefig('AD_true2.pdf', dpi=200, bbox_inches='tight')<jupyter_output>/home/vero/miniconda2/envs/py36/lib/python3.6/site-packages/scipy-1.1.0-py3.6-linux-x86_64.egg/scipy/stats/stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval
{'fraction_non_alphanumeric': 0.13580028328611898, 'fraction_numerical': 0.02496458923512748, 'mean_word_length': 3.014925373134328, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '615173', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 2148, 'n_tokens_neox': 2010, 'n_words': 298, 'path': 'notebooks/Power-law test/Power of the test example.ipynb', 'repo_name': 'DaviGarba/netanalytics', 'size': '5648'}