<jupyter_start><jupyter_text>Sattilite Distribution This notebook illustrates how to implenent Zheng2007, and compare the results to halotools  Preliminary setupHere we download and install all the requirements<jupyter_code>!nvidia-smi
import os

#using GPU, comment out if on CPU
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = "3"
import tensorflow as tf
print(tf.__version__)

import tensorflow_probability as tfp
tfp.__version__
%pylab inline
import numpy as np
import tensorflow as tf
print(tf.__version__)
import tensorflow_probability as tfp
tfd = tfp.distributions
tfb = tfp.bijectors
from tensorflow_probability import edward2 as ed
from diffhod.components import Zheng07Cens, Zheng07SatsRelaxedBernoulli, NFWProfile
from diffhod.utils import *

from halotools.sim_manager import CachedHaloCatalog
from halotools.empirical_models import PrebuiltHodModelFactory
from halotools.mock_observables.two_point_clustering import tpcf
from halotools.mock_observables import return_xyz_formatted_array<jupyter_output>Populating the interactive namespace from numpy and matplotlib
2.2.0<jupyter_text>Prepare halo catalog<jupyter_code># Load a reference halo catalog
halocat = CachedHaloCatalog(simname = 'bolshoi', redshift=-0.003) 

# Removing subhalos and convert table to tensors
halo_table = halocat.halo_table
halo_table = halo_table[halo_table['halo_pid'] == -1]
tf_halocat = {k: tf.convert_to_tensor(halo_table[k], dtype=tf.float32) for k in halo_table.colnames}<jupyter_output><empty_output><jupyter_text>Model definition<jupyter_code># We first load our reference halotools model
zheng07_model = PrebuiltHodModelFactory('zheng07', 
                                        redshift =-0.003, 
                                        modulate_with_cenocc=True)
# Here is how we define the same model in diffhod
@tf.function
def helper_hod(halo_cat, max_sat=60, temp=0.00001, params = zheng07_model.param_dict):

  ### Occupation model ###
  n_cen = Zheng07Cens(halo_cat['halo_mvir'],
                      **params,
                      temperature=temp)
  
  n_sat = Zheng07SatsRelaxedBernoulli(halo_cat['halo_mvir'],
                                      n_cen,
                                      **params,
                                      sample_shape=(max_sat,),
                                      temperature=temp)
  
  ### Phase Space model ###
  # Centrals are just located at center of halo
  pos_cen = ed.Deterministic(tf.stack([halo_cat['halo_x'],
                                        halo_cat['halo_y'],
                                        halo_cat['halo_z']], axis=-1))
    
  # Satellites follow an NFW profile centered on halos
  pos_sat = NFWProfile(pos=pos_cen,
                        concentration=halo_cat['halo_nfw_conc'],
                        Rvir=halo_cat['halo_rvir'],
                        sample_shape=(max_sat,))
  
  return {'pos_cen':pos_cen, 'pos_sat':pos_sat, 'n_cen':n_cen, 'n_sat':n_sat}
# sampling galaxies from the model

gal_cat = helper_hod(tf_halocat)
# Same thing with halotools
zheng07_model.populate_mock(halocat)<jupyter_output><empty_output><jupyter_text>Testing 2pt function<jupyter_code># Let's define halotools samples
ht_sample = return_xyz_formatted_array(zheng07_model.mock.galaxy_table['x'],
                                       zheng07_model.mock.galaxy_table['y'],
                                       zheng07_model.mock.galaxy_table['z'])
m = zheng07_model.mock.galaxy_table['host_centric_distance'] <0.0001
ht_cens = ht_sample[m]
ht_sats = ht_sample[~m]
# Same thing for dhod
p = gal_cat['n_cen']>0.5
tf_cens = gal_cat['pos_cen'][p[0]].numpy()
p = gal_cat['n_sat']>0.5
tf_sats = gal_cat['pos_sat'][p[:,0,:]].numpy().reshape((-1,3))
tf_sample = np.concatenate([tf_cens, tf_sats], axis=0)
rbins = np.logspace(-1.5, 1.5, 16)
xi_ht = tpcf(ht_sample, rbins, period=halocat.Lbox)
xi_tf = tpcf(tf_sample, rbins, period=halocat.Lbox)

loglog(rbins[1:],xi_ht, label='halotools')
loglog(rbins[1:],xi_tf, label='DHOD')
xlabel('r [Mpc/h]')
title('Zheng2007 HOD 2pcf');<jupyter_output><empty_output><jupyter_text>Occupancy Distributions<jupyter_code>import scipy

import copy 

new_params = copy.deepcopy(zheng07_model.param_dict)
helper_hod?
from diffhod.diffhod import *
out_sat = []
out_cen = []

binned_out_sat = scipy.stats.binned_statistic(np.log10(zheng07_model.mock.halo_table["halo_mvir"]),zheng07_model.mock.halo_table["halo_num_satellites"],bins=50)
binned_out_cen = scipy.stats.binned_statistic(np.log10(zheng07_model.mock.halo_table["halo_mvir"]),zheng07_model.mock.halo_table["halo_num_centrals"],bins=50)

for tau in [1,0.75,0.02,0.001]:
# sampling galaxies from the model
    gal_cat = helper_hod(tf_halocat,temp = tau,params=new_params)
    binned_out_sat_tf = scipy.stats.binned_statistic(np.log10(tf_halocat["halo_mvir"]),tf.reduce_sum(gal_cat['n_sat'],axis=0),bins=50)
    binned_out_cen_tf = scipy.stats.binned_statistic(np.log10(tf_halocat["halo_mvir"]),gal_cat['n_cen'],bins=50)

    out_sat.append(binned_out_sat_tf)
    out_cen.append(binned_out_cen_tf)
bins_a = [[] for i in range(-4,10)]
bins_a
for qqq in range(0,100):
    gal_cat = helper_hod(tf_halocat,temp = 0.02,params=new_params,max_sat=30)
    for i in range(-4,10):
        filtb = np.logical_and(tf_halocat["halo_mvir"]<10**(13+i*0.2 + 0.2),tf_halocat["halo_mvir"]>10**(13+i*0.2))
        #_,bins,_ = plt.hist(np.array(tf.reduce_sum(gal_cat['n_sat'],axis=0))[0][filtb],alpha=0.5,range=(0,20),bins=20)
        bins_a[i+4].append(np.array(tf.reduce_sum(gal_cat['n_sat'],axis=0))[0][filtb])
bins_b = [[] for i in range(-4,10)]
for qqq in range(0,100):
    zheng07_model.populate_mock(halocat) 
    for i in range(-4,10):
        filt = np.logical_and(zheng07_model.mock.halo_table["halo_mvir"]<10**(13+i*0.2 + 0.2),zheng07_model.mock.halo_table["halo_mvir"]>10**(13+i*0.2))
        #_,bins,_ = plt.hist(np.array(tf.reduce_sum(gal_cat['n_sat'],axis=0))[0][filtb],alpha=0.5,range=(0,20),bins=20)
        bins_b[i+4].append(np.array(zheng07_model.mock.halo_table["halo_num_satellites"])[filt])
i = 9
i%3
np.sort(np.array(bins_b[i]).flatten())[int(1400*(50-18)/100)],np.sort(np.array(bins_b[i]).flatten())[int(1400*1/2)],np.sort(np.array(bins_b[i]).flatten())[int(1400*68/100)]
fig, ax = plt.subplots(3, 3,sharey="row")
fig.set_size_inches(12.0, 10.0)


for i in range(4,13):
    nn=i-4
    print(i,nn,nn%3,nn-nn%3*3)
    if nn%3 == 0 :
        vals,bins,_ = ax[int(nn/3),nn%3].hist(np.array(bins_a[i]).flatten(),alpha=0.4,density=1.0,range=(0,40),bins=40,label="DiffHOD")
    else:
        _,bins,_ = ax[int(nn/3),nn%3].hist(np.array(bins_a[i]).flatten(),alpha=0.4,density=1.0,range=(0,40),bins=40,label="DiffHOD")

    ax[int(nn/3),nn%3].hist(np.array(bins_b[i]).flatten(),bins=bins,color="red",alpha=0.4,density=1.0,label="HOD")
    ax[int(nn/3),nn%3].set_title("$\log M_{vir} = [" + str(13+(i-4)*0.2)[:5]+ "," + str(13+(i-4)*0.2+0.2)[:5]+"]$",)
    ax[int(nn/3),nn%3].set_xlabel("# of Sats")
    if nn == 8:
        nni = np.sort(np.array(bins_b[i]).flatten()).shape[0]
        A,B,C = np.sort(np.array(bins_b[i]).flatten())[int(nni*(50-18)/100)],np.sort(np.array(bins_b[i]).flatten())[int(nni*1/2)],np.sort(np.array(bins_b[i]).flatten())[int(nni*68/100)]
        ax[int(nn/3),nn%3].text(5,vals.max()*3/5,r"$N_{sat} = " + str(B) + " \pm^{"+str(C)+"}_{"+str(A)+"}$",color="Red")
        nni = np.sort(np.array(bins_a[i]).flatten()).shape[0]
        A,B,C = np.sort(np.array(bins_a[i]).flatten())[int(nni*(50-18)/100)],np.sort(np.array(bins_a[i]).flatten())[int(nni*1/2)],np.sort(np.array(bins_a[i]).flatten())[int(nni*68/100)]
        ax[int(nn/3),nn%3].text(5,vals.max()*3/4,r"$N_{sat} = " + str(f"{B: .1f}") + " \pm^{"+str(f"{C: .1f}")+"}_{"+str(f"{A: .1f}")+"}$",color="Blue")
        
    else:
        nni = np.sort(np.array(bins_b[i]).flatten()).shape[0]
        A,B,C = np.sort(np.array(bins_b[i]).flatten())[int(nni*(50-18)/100)],np.sort(np.array(bins_b[i]).flatten())[int(nni*1/2)],np.sort(np.array(bins_b[i]).flatten())[int(nni*68/100)]
        ax[int(nn/3),nn%3].text(25,vals.max()*3/5,r"$N_{sat} = " + str(B) + " \pm^{"+str(C)+"}_{"+str(A)+"}$",color="Red")
        nni = np.sort(np.array(bins_a[i]).flatten()).shape[0]
        A,B,C = np.sort(np.array(bins_a[i]).flatten())[int(nni*(50-18)/100)],np.sort(np.array(bins_a[i]).flatten())[int(nni*1/2)],np.sort(np.array(bins_a[i]).flatten())[int(nni*68/100)]
        ax[int(nn/3),nn%3].text(25,vals.max()*3/4,r"$N_{sat} = " + str(f"{B: .1f}") + " \pm^{"+str(f"{C: .1f}")+"}_{"+str(f"{A: .1f}")+"}$",color="Blue")

ax[0,0].set_ylabel("Density")
ax[1,0].set_ylabel("Density")
ax[2,0].set_ylabel("Density")

ax[0,0].legend()
fig.tight_layout()
plt.savefig("distributions.pdf")
plt.show()
print("Number: ", f"{76.663254: .2f}")

np.array(tf.reduce_sum(gal_cat['n_sat'],axis=0))[0][filtb]
for i in range(-4,10):
    filtb = np.logical_and(tf_halocat["halo_mvir"]<10**(13+i*0.2 + 0.2),tf_halocat["halo_mvir"]>10**(13+i*0.2))
    _,bins,_ = plt.hist(np.array(tf.reduce_sum(gal_cat['n_sat'],axis=0))[0][filtb],alpha=0.5,range=(0,50),bins=20)
    filt = np.logical_and(zheng07_model.mock.halo_table["halo_mvir"]<10**(13+i*0.2 + 0.2),zheng07_model.mock.halo_table["halo_mvir"]>10**(13+i*0.2))
    plt.hist(np.array(zheng07_model.mock.halo_table["halo_num_satellites"])[filt],alpha=0.5,bins=bins,color="red")
    plt.xlim(0,20)
    plt.title(str(13+i*0.2)+ " " + str(13+i*0.2+0.2))
    plt.show()


filt = np.logical_and(zheng07_model.mock.halo_table["halo_mvir"]<10**13.5,zheng07_model.mock.halo_table["halo_mvir"]>10**13.4)
plt.hist(np.array(zheng07_model.mock.halo_table["halo_num_satellites"])[filt])
#pretty plot

import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rcParams['text.usetex'] = False
mpl.rcParams['font.family'] = 'serif'
mpl.rcParams['axes.linewidth'] = 1.5
mpl.rcParams['axes.xmargin'] = 1
mpl.rcParams['xtick.labelsize'] = 'x-large'
mpl.rcParams['xtick.major.size'] = 5
mpl.rcParams['xtick.major.width'] = 1.5
mpl.rcParams['ytick.labelsize'] = 'x-large'
mpl.rcParams['ytick.major.size'] = 5
mpl.rcParams['ytick.major.width'] = 1.5
mpl.rcParams['legend.frameon'] = False
plt.plot((binned_out_sat.bin_edges[1:] + binned_out_sat.bin_edges[:-1])/2,binned_out_sat.statistic,"r*",label="HOD")

tt = [1,0.75,0.02,0.001]

for n,os in enumerate(out_sat):
    plt.plot((os.bin_edges[1:] + os.bin_edges[:-1])/2,os.statistic[0],label=r"$\tau = $" + str(tt[n]))
plt.legend(loc=4,fontsize=12)
plt.semilogy()
plt.ylabel(r"$\langle N_{sat} \rangle$",fontsize=14)
plt.xlabel(r"$\log_{10}(M_{vir})$",fontsize=14)
plt.ylim(0.001,100)
plt.xlim(11.0,14.80)
plt.tight_layout()
plt.savefig("occupancy_sat.pdf")

plt.plot((binned_out_cen.bin_edges[1:] + binned_out_cen.bin_edges[:-1])/2,binned_out_cen.statistic,"r*",label="HOD")

tt = [1,0.75,0.02,0.001]

for n,os in enumerate(out_cen):
    plt.plot((os.bin_edges[1:] + os.bin_edges[:-1])/2,os.statistic[0],label=r"$\tau = $" + str(tt[n]))
plt.legend(loc=4,fontsize=12)
#plt.semilogy()
plt.ylabel(r"$\langle N_{cen} \rangle$",fontsize=14)
plt.xlabel(r"$\log_{10}(M_{vir})$",fontsize=14)
plt.ylim(-0.1,1.1)
plt.xlim(11.0,14.80)
plt.tight_layout()

plt.savefig("occupancy_cen.pdf")

from diffhod.math.binning import histogram
# choosing subvolume so we can play around easier with derivatives

# Load a reference halo catalog
halocat = CachedHaloCatalog(simname = 'bolshoi', redshift=-0.003) 

# Removing subhalos and convert table to tensors
halo_table = halocat.halo_table
halo_table = halo_table[halo_table['halo_pid'] == -1]


halo_table = halo_table[halo_table['halo_x'] < 128]
halo_table = halo_table[halo_table['halo_y'] < 128]
halo_table = halo_table[halo_table['halo_z'] < 128]
tf_halocat = {k: tf.convert_to_tensor(halo_table[k], dtype=tf.float32) for k in halo_table.colnames}
#for some reason I can't get derivatives of all bandpowers at once with this sort of setup and end up looping through them isntead...
#easier than debugging the grad function :D

tau_array = [0.001,0.02,1.0]
sat_array =[]

cen_array =[]
for tau in tau_array:
    print(tau)
    bins = tf.linspace(tf.math.reduce_min(np.log10(tf_halocat["halo_mvir"])), tf.math.reduce_max(np.log10(tf_halocat["halo_mvir"])), 50)

    def ncen(x):
        logMmin,sigma_logM,logM0,logM1,alpha = tf.transpose(x)
        tf_gal = hod(tf_halocat,logMmin,sigma_logM,logM0,logM1,alpha,max_sat=30,temp=tau)
        a=histogram(tf.math.log(tf_halocat["halo_mvir"])/tf.math.log(10.0),bins,weights=tf_gal['n_cen'])
        b=histogram(tf.math.log(tf_halocat["halo_mvir"])/tf.math.log(10.0),bins)
        return a/b


    def nsat(x):
        logMmin,sigma_logM,logM0,logM1,alpha = tf.transpose(x)
        tf_gal = hod(tf_halocat,logMmin,sigma_logM,logM0,logM1,alpha,max_sat=40,temp=tau)
        a=histogram(tf.math.log(tf_halocat["halo_mvir"])/tf.math.log(10.0),bins,weights=tf.reduce_sum(tf_gal['n_sat'],axis=0))
        b=histogram(tf.math.log(tf_halocat["halo_mvir"])/tf.math.log(10.0),bins)
        return a/b
    
    #checking gradients...
    out_ar = []
    x = tf.Variable([12.02,  0.26, 11.38, 13.31,  0000001.9900010])
    print(ncen(x))#[[12.02,12.02,], [0.26,0.26,], [11.38,11.38,], [13.31,13.31,], [x,x,]]))
    #loss_ar.append(np.array(L))
    for nn in range(0,49):
        def get_grads(x):
            with tf.GradientTape(persistent=True) as g:
             #  g.watch(x)
               L = nsat(x)[nn]
            dL_dx = g.gradient(L, x)
            return dL_dx
        #a = tf.constant((np.array([[12.5,0.5,11.38,13.01, x]],dtype=np.float32)))
        grad = get_grads(x)
        out_ar.append(np.array(grad))
        print(nn,grad)
    sat_array.append(np.array(out_ar))
    out_ar = []
    #same for centrals
    x = tf.Variable([12.02,  0.26, 11.38, 13.31,  0000001.9900010])
    print(ncen(x))#[[12.02,12.02,], [0.26,0.26,], [11.38,11.38,], [13.31,13.31,], [x,x,]]))
    #loss_ar.append(np.array(L))
    for nn in range(0,49):
        def get_grads(x):
            with tf.GradientTape(persistent=True) as g:
             #  g.watch(x)
               L = ncen(x)[nn]
            dL_dx = g.gradient(L, x)
            return dL_dx
        #a = tf.constant((np.array([[12.5,0.5,11.38,13.01, x]],dtype=np.float32)))
        grad = get_grads(x)
        out_ar.append(np.array(grad))
        print(nn,grad)
    cen_array.append(np.array(out_ar))

#analytical derivatives

def dncendlogmin(M):
    logMmin, sigma_logM, logM0, logM1, alpha = [12.02,  0.26, 11.38, 13.31,  0000001.9900010]
    exp = np.exp(-(M-logMmin)**2/sigma_logM**2.0)
    print((sigma_logM*np.sqrt(np.pi)*logMmin))
    return -exp*10/(sigma_logM*np.sqrt(np.pi)*M)
    
def dncendsigma(M):
    logMmin, sigma_logM, logM0, logM1, alpha = [12.02,  0.26, 11.38, 13.31,  0000001.9900010]
    exp = np.exp(-(M-logMmin)**2/sigma_logM**2.0)
    print((sigma_logM*np.sqrt(np.pi)*logMmin))
    return -exp*1/(sigma_logM**2.0*np.sqrt(np.pi)) * (M-logMmin)

def dnsatdlogmin(M):
    logMmin, sigma_logM, logM0, logM1, alpha = [12.02,  0.26, 11.38, 13.31,  0000001.9900010]
    exp = np.exp(-(M-logMmin)**2/sigma_logM**2.0)
    print((sigma_logM*np.sqrt(np.pi)*logMmin))
    pref = ((10**M-10**logM0)/(10**logM1))**alpha
    return -exp*10/(sigma_logM*np.sqrt(np.pi)*M)*pref
    
def dnsatdsigma(M):
    logMmin, sigma_logM, logM0, logM1, alpha = [12.02,  0.26, 11.38, 13.31,  0000001.9900010]
    exp = np.exp(-(M-logMmin)**2/sigma_logM**2.0)
    print((sigma_logM*np.sqrt(np.pi)*logMmin))
    pref = ((10**M-10**logM0)/(10**logM1))**alpha

    return -exp*1/(sigma_logM**2.0*np.sqrt(np.pi)) * (M-logMmin)*pref


def dnsatdalpha(M):
    logMmin, sigma_logM, logM0, logM1, alpha = [12.02,  0.26, 11.38, 13.31,  0000001.9900010]
    pref = 1/2*(1+ scipy.special.erf((M-logMmin)/sigma_logM))

    return pref *((10**M-10**logM0)/(10**logM1))**(alpha) * np.log((10**M-10**logM0)/(10**logM1))

def dnsatdlogM0(M):
    logMmin, sigma_logM, logM0, logM1, alpha = [12.02,  0.26, 11.38, 13.31,  0000001.9900010]
    pref = 1/2*(1+ scipy.special.erf((M-logMmin)/sigma_logM))

    return 1/2*pref *((10**M-10**logM0)/(10**logM1))**(alpha-1) * -1*alpha/logM1


def dnsatdlogM1(M):
    logMmin, sigma_logM, logM0, logM1, alpha = [12.02,  0.26, 11.38, 13.31,  0000001.9900010]
    pref = 1/2*20*(1+ scipy.special.erf((M-logMmin)/sigma_logM))

    return pref *((10**M-10**logM0)/(10**logM1))**(alpha) * -1*alpha/logM1



fig, ax = plt.subplots(2, 3,sharex=True,sharey=True)
fig.set_size_inches(11.5, 7.5)

ll = [r'$\logM_{min}$', r'$\sigma_{\log M}$', r'$\log M_0$', r'$\log M_1$', r'$\alpha$']

ax[0,0].set_ylabel(r"$d\langle N_{cen} \rangle/d\theta$",fontsize=20)
ax[1,0].set_ylabel(r"$d\langle N_{sat} \rangle/d\theta$",fontsize=20)

for i,xx in enumerate(cen_array):
    for j in range(0,5):
        ax[0,i].plot((bins[1:]+bins[:-1])/2,xx[:,j],label = ll[j])  
    ax[0,i].set_xlim(10.8,14.1)
  #  ax[0,i].set_ylabel(r"$d\langle N_{sat} \rangle/d\theta$")
    ax[0,i].set_yscale('symlog', linthreshy=0.0010)
    ax[0,i].set_title(r'$\tau = $' + str(tau_array[i]),fontsize=20)

for i,xx in enumerate(sat_array):
    for j in range(0,5):
        ax[1,i].plot((bins[1:]+bins[:-1])/2,xx[:,j],label = ll[j])
    ax[1,i].set_xlim(10.8,14.1)
    ax[1,i].set_yscale('symlog', linthreshy=0.0010)
    ax[1,i].set_xlabel(r'$\log_{10} M/M_\odot$',fontsize=20)
    #ax[0,i].set_title(r'$\tau = $' + str(tau_array[i]))
    
ax[0,1].plot(np.linspace(10,14.5,50),dncendlogmin(np.linspace(10,14.5,50)),"b*",markersize=3)
ax[0,1].plot(np.linspace(10,14.5,50),dncendsigma(np.linspace(10,14.5,50)),"*",color="orange",markersize=3)

ax[1,1].plot(np.linspace(10,14.5,50),dnsatdlogmin(np.linspace(10,14.5,50)),"b*",markersize=3)
ax[1,1].plot(np.linspace(10,14.5,50),dnsatdsigma(np.linspace(10,14.5,50)),"*",color="orange",markersize=3)
ax[1,1].plot(np.linspace(10,14.5,50),dnsatdalpha(np.linspace(10,14.5,50)),"*",color="purple",markersize=3)
ax[1,1].plot(np.linspace(10,14.5,50),dnsatdlogM0(np.linspace(10,14.5,50)),"*",color="green",markersize=3)

ax[1,1].plot(np.linspace(10,14.5,50),dnsatdlogM1(np.linspace(10,14.5,50)),"*",color="red",markersize=3)

plt.subplots_adjust(wspace=0, hspace=0)

ax[1,0].legend(loc=3,fontsize=11.2)
 
plt.tight_layout()
plt.savefig("3panel.pdf")<jupyter_output><ipython-input-35-6e08c86f53ac>:14: MatplotlibDeprecationWarning: The 'linthreshy' parameter of __init__() has been renamed 'linthresh' since Matplotlib 3.3; support for the old name will be dropped two minor releases later.
  ax[0,i].set_yscale('symlog', linthreshy=0.0010)
<ipython-input-35-6e08c86f53ac>:14: MatplotlibDeprecationWarning: The 'linthreshy' parameter of __init__() has been renamed 'linthresh' since Matplotlib 3.3; support for the old name will be dropped two minor releases later.
  ax[0,i].set_yscale('symlog', linthreshy=0.0010)
<ipython-input-35-6e08c86f53ac>:14: MatplotlibDeprecationWarning: The 'linthreshy' parameter of __init__() has been renamed 'linthresh' since Matplotlib 3.3; support for the old name will be dropped two minor releases later.
  ax[0,i].set_yscale('symlog', linthreshy=0.0010)
<ipython-input-35-6e08c86f53ac>:21: MatplotlibDeprecationWarning: The 'linthreshy' parameter of __init__() has been renamed 'linthresh' since Matplotlib 3.3; support for the o[...]
{'fraction_non_alphanumeric': 0.19692708333333334, 'fraction_numerical': 0.07020833333333333, 'mean_word_length': 4.563894523326572, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '230961', 'license': 'MIT', 'n_tokens_mistral': 9089, 'n_tokens_neox': 8339, 'n_words': 1031, 'path': 'nb/sat_dist.ipynb', 'repo_name': 'DifferentiableUniverseInitiative/DHOD', 'size': '19200'}