<commit_before># coding: utf-8
from agpy import readcol
import plfit
from pylab import *

blackouts = readcol('blackouts.txt')
cities = readcol('cities.txt')
earthquakes = readcol('earthquakes.txt')
melville = readcol('melville.txt')
solarflares = readcol('solarflares.txt')
terrorism = readcol('terrorism.txt')

#print "quakes 0.00 -7.14 0.00 11.6 0.00 -7.09 0.00 -24.4 0.00 with cut-oﬀ"
#earthquakeP = plfit.plfit(earthquakes)


pl = plfit.plfit(cities.ravel() / 1e3, usefortran=True, verbose=True)
print "Cities (me)     : n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (pl.data.shape[0], pl.data.mean(), pl.data.std(), pl.data.max(), pl._xmin, pl._alpha, pl._alphaerr, pl._ngtx, pl._ks_prob)
print "Cities (Clauset): n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (19447,9.00,77.83,8009,52.46,2.37,0.08,580,0.76)
figure(1)
clf()
title("Cities")
subplot(131)
pl.plotpdf()
subplot(132)
title("Cities")
pl.xminvsks()
subplot(133)
pl.alphavsks()
savefig("figures/cities_kstests.png")

figure(2)
title("Cities")
pl.plotcdf()
savefig("figures/cities_cdf.png")

pl = plfit.plfit(melville.ravel(), verbose=True)
p,sims = pl.test_pl(usefortran=True, niter=100)
print "Melville (me)     : n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (pl.data.shape[0], pl.data.mean(), pl.data.std(), pl.data.max(), pl._xmin, pl._alpha, pl._alphaerr, pl._ngtx, p)
print "Melville (Clauset): n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (18855,11.14,148.33,14086,7,1.95,0.02,2958,0.49)
# count of word use 18 855 11.14 148.33 14 086 7 ± 2 1.95(2) 2958 ± 987 0.49
# words 0.49 4.43 0.00 0.395 0.69 9.09 0.00 4.13 0.00 −0.899 0.18 goo
figure(3)
clf()
title("Melville")
subplot(131)
pl.plotpdf()
subplot(132)
title("Melville")
pl.xminvsks()
subplot(133)
pl.alphavsks()
savefig("figures/Melville_kstests.png")

figure(4)
title("Melville")
pl.plotcdf()
savefig("figures/Melville_cdf.png")


pl = plfit.plfit(solarflares.ravel())
p,sims = pl.test_pl(usefortran=True, niter=100)
print "Solarflares (me)     : n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (pl.data.shape[0], pl.data.mean(), pl.data.std(), pl.data.max(), pl._xmin, pl._alpha, pl._alphaerr, pl._ngtx, p)
print "Solarflares (Clauset): n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (12773, 689.41, 6520.59, 231300, 323, 1.79, 0.02, 1711, 1.00)
figure(5)
clf()
title("Solar Flares")
subplot(131)
pl.plotpdf()
subplot(132)
title("Solar Flares")
pl.xminvsks()
subplot(133)
pl.alphavsks()
savefig("figures/SolarFlares_kstests.png")

figure(6)
title("SolarFlares")
pl.plotcdf()
savefig("figures/SolarFlares_cdf.png")


pl = plfit.plfit(terrorism.ravel())
p,sims = pl.test_pl(usefortran=True, niter=100, nosmall=False)
print "Terrorism (me)     : n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (pl.data.shape[0], pl.data.mean(), pl.data.std(), pl.data.max(), pl._xmin, pl._alpha, pl._alphaerr, pl._ngtx, p)
print "Terrorism (Clauset): n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (9101, 4.35, 31.58, 2749, 12, 2.4, 0.2, 547, 0.68)
figure(6)
clf()
title("Terrorism")
subplot(131)
pl.plotpdf()
subplot(132)
title("Terrorism")
pl.xminvsks()
subplot(133)<commit_msg>slight change to Clauset tests<commit_after># coding: utf-8
from agpy import readcol
import plfit
from pylab import *

blackouts = readcol('blackouts.txt')
cities = readcol('cities.txt')
earthquakes = readcol('earthquakes.txt')
melville = readcol('melville.txt')
solarflares = readcol('solarflares.txt')
terrorism = readcol('terrorism.txt')

#print "quakes 0.00 -7.14 0.00 11.6 0.00 -7.09 0.00 -24.4 0.00 with cut-oﬀ"
#earthquakeP = plfit.plfit(earthquakes)


pl = plfit.plfit(cities.ravel() / 1e3, usefortran=True, verbose=True)
print "Cities (me)     : n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (pl.data.shape[0], pl.data.mean(), pl.data.std(), pl.data.max(), pl._xmin, pl._alpha, pl._alphaerr, pl._ngtx, pl._ks_prob)
print "Cities (Clauset): n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (19447,9.00,77.83,8009,52.46,2.37,0.08,580,0.76)
figure(1)
clf()
title("Cities")
subplot(131)
pl.plotpdf()
subplot(132)
title("Cities")
pl.xminvsks()
subplot(133)
pl.alphavsks()
savefig("figures/cities_kstests.png")

figure(2)
title("Cities")
pl.plotcdf()
savefig("figures/cities_cdf.png")

pl = plfit.plfit(melville.ravel(), verbose=True)
p,sims = pl.test_pl(usefortran=True, niter=100)
print "Melville (me)     : n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (pl.data.shape[0], pl.data.mean(), pl.data.std(), pl.data.max(), pl._xmin, pl._alpha, pl._alphaerr, pl._ngtx, p)
print "Melville (Clauset): n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (18855,11.14,148.33,14086,7,1.95,0.02,2958,0.49)
# count of word use 18 855 11.14 148.33 14 086 7 ± 2 1.95(2) 2958 ± 987 0.49
# words 0.49 4.43 0.00 0.395 0.69 9.09 0.00 4.13 0.00 −0.899 0.18 goo
figure(3)
clf()
title("Melville")
subplot(131)
pl.plotpdf()
subplot(132)
title("Melville")
pl.xminvsks()
subplot(133)
pl.alphavsks()
savefig("figures/Melville_kstests.png")

figure(4)
title("Melville")
pl.plotcdf()
savefig("figures/Melville_cdf.png")


pl = plfit.plfit(solarflares.ravel(), verbose=True)
p,sims = pl.test_pl(usefortran=True, niter=100)
print "Solarflares (me)     : n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (pl.data.shape[0], pl.data.mean(), pl.data.std(), pl.data.max(), pl._xmin, pl._alpha, pl._alphaerr, pl._ngtx, p)
print "Solarflares (Clauset): n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (12773, 689.41, 6520.59, 231300, 323, 1.79, 0.02, 1711, 1.00)
figure(5)
clf()
title("Solar Flares")
subplot(131)
pl.plotpdf()
subplot(132)
title("Solar Flares")
pl.xminvsks()
subplot(133)
pl.alphavsks()
savefig("figures/SolarFlares_kstests.png")

figure(6)
title("SolarFlares")
pl.plotcdf()
savefig("figures/SolarFlares_cdf.png")


pl = plfit.plfit(terrorism.ravel(), verbose=True)
p,sims = pl.test_pl(usefortran=True, niter=100, nosmall=False)
print "Terrorism (me)     : n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (pl.data.shape[0], pl.data.mean(), pl.data.std(), pl.data.max(), pl._xmin, pl._alpha, pl._alphaerr, pl._ngtx, p)
print "Terrorism (Clauset): n:%10i mean,std,max: %8.2f,%8.2f,%8.2f xmin: %8.2f alpha: %8.2f (%8.2f) ntail: %10i p: %5.2f" % (9101, 4.35, 31.58, 2749, 12, 2.4, 0.2, 547, 0.68)
figure(6)
clf()
title("Terrorism")
subplot(131)
pl.plotpdf()
subplot(132)
title("Terrorism")
pl.xminvsks()
subplot(133)
{'fraction_non_alphanumeric': 0.21968941444650236, 'fraction_numerical': 0.12252457615044879, 'mean_word_length': 5.005132591958939, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5373898', 'n_tokens_mistral': 3892, 'n_tokens_neox': 3405, 'n_words': 653}