<jupyter_start><jupyter_text>Data analysis  Importing needed libraries<jupyter_code>!pip install descartes
!pip install geopandas
!pip install country_converter
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
import pandas
import geopandas as gpd
from matplotlib import style
import country_converter as coco<jupyter_output><empty_output><jupyter_text>Loading a CSV file in Python<jupyter_code>data = pandas.read_csv('data.csv', sep=',')
data
#check the column names
data.columns.values
#get the number of collected domains
index = data.index
total = len(index)
print(total)<jupyter_output><empty_output><jupyter_text>Distribution of domain names across name servers<jupyter_code>#settings of plotting
plt.style.use('seaborn-deep')
plt.rcParams.update({'font.size': 10})
#grouping the data by NS IPs
ns = data.groupby('IP of NS')
ns_counter = ns.size().to_frame('size')
ns_counter

#sorting the results and counting the percentage distribution
result_ns = ns_counter.sort_values('size', ascending=False)
result_ns['percent'] = result_ns['size']/total*100
result_ns=result_ns.reset_index()
result_ns.index += 1 
result_ns_table = result_ns.head(25)
print(result_ns_table)
result_ns_table['percent'].sum()
#printing the graph

ns_pl = result_ns.cumsum()
ns_pl.plot(y='percent', kind='line', 
        figsize=(5, 4), legend=False, style='b.-')
#plt.title(" Distribution of domains across name servers", y=1.01, fontsize=15)
plt.ylabel("Percentage of vulnerable domains", labelpad=15, fontsize=12)
plt.xlabel("Number of authoritative name servers", labelpad=15, fontsize=12);<jupyter_output><empty_output><jupyter_text>Distribution of domain names across autonomous systems<jupyter_code>#grouping the data by NS ASNs
ASN_dns = data.groupby('ASN(NS)')
ASN_dns_counter = ASN_dns.size().to_frame('size')
ASN_dns_counter
#sorting the results and counting the percentage distribution
result_asn = ASN_dns_counter.sort_values('size', ascending=False)
result_asn['percent'] = result_asn['size']/total*100
result_asn = result_asn.reset_index()
result_asn.index += 1 
print(result_asn)
result_table = result_asn.head()
result_table['percent'].sum()
asn_pl = result_asn.cumsum().head(94)
print(asn_pl)
#printing the graph
asn_pl = result_asn.cumsum()
asn_pl.plot(y='percent', use_index=True, kind='line', 
        figsize=(5, 4), legend=False, style='b.-')
#plt.title("Distribution of domains across autonomous systems", y=1.01, fontsize=15)
plt.ylabel("Percentage of vulnerable domains", labelpad=15, fontsize=12)
plt.xlabel("Number of autonomous systems", labelpad=15, fontsize=12);<jupyter_output><empty_output><jupyter_text>Distribution of domains across autonomous systems based on web servers<jupyter_code>data['ASN_web'] = data['ASN(WEB)'].str.split(':').str[1]
ASN_web = data.groupby('ASN_web')
ASN_web_counter = ASN_web.size().to_frame('size')
result_web = ASN_web_counter.sort_values('size', ascending=False)
result_web = result_web.tail(1016)
result_web['percent'] = result_web['size']/total*100
result_table_web = result_web.head(46)
result_web = result_web.reset_index()
print(result_table_web)
result_table_web['size'].sum()
asn_pl = result_web.cumsum()
asn_pl.plot(y='percent', use_index=True, kind='line', 
        figsize=(10, 8), legend=False, style='b.-')
plt.title("Distribution of domains across autonomous systems based on web servers", y=1.01, fontsize=12)
plt.ylabel("Percentage of vulnerable domains", labelpad=15, fontsize=10)
plt.xlabel("Number of autonomous systems", labelpad=15, fontsize=10);<jupyter_output><empty_output><jupyter_text>Distribution of domains across autonomous systems based on WHOIS records<jupyter_code>whois_org = data.groupby('Whois org')
whois_org_counter = whois_org.size().to_frame('size')
result_whois_org = whois_org_counter.sort_values('size', ascending=False)
result_whois_org['percent'] = result_whois_org['size']/total*100
result_whois_org_table = result_whois_org.head(40)
print(result_whois_org_table)
result_whois_org['size'].sum()<jupyter_output><empty_output><jupyter_text>Analysis of DNS records  A records<jupyter_code>ipv4 = data.groupby('IPv4(WEB)')
ipv4_counter = ipv4.size().to_frame('size')
result_ipv4 = ipv4_counter.sort_values('size', ascending=False)
result_ipv4['percent'] = result_ipv4['size']/total*100
result_ipv4=result_ipv4.reset_index()
result_ipv4_table = result_ipv4.head(249)
print(result_ipv4_table)
result_ipv4['size'].sum()<jupyter_output><empty_output><jupyter_text>AAAA records<jupyter_code>ipv6 = data.groupby('IPv6')
ipv6_counter = ipv6.size().to_frame('size')
result_ipv6 = ipv6_counter.sort_values('size', ascending=False)
result_ipv6['percent'] = result_ipv6['size']/total*100
result_ipv6_table = result_ipv6.head(249)
print(result_ipv6_table)
result_ipv6['size'].sum()<jupyter_output><empty_output><jupyter_text>CNAME records<jupyter_code>cname = data.groupby('CNAME')
cname_counter = cname.size().to_frame('size')
result_cname = cname_counter.sort_values('size', ascending=False)
result_cname['percent'] = result_cname['size']/total*100
result_cname_table = result_cname.head()
print(result_cname_table)
result_cname['size'].sum()<jupyter_output><empty_output><jupyter_text>A, AAAA, CNAME records - information on web servers<jupyter_code>web = data.groupby(['IPv4(WEB)', 'IPv6', 'CNAME'])
web_counter = web.size().to_frame('size')
result_web = web_counter.sort_values('size', ascending=False)
result_web['percent'] = result_web['size']/total*100
result_web_table = result_web.head(249)
print(result_web_table)
result_web['size'].sum()<jupyter_output><empty_output><jupyter_text>AXFR records<jupyter_code>axfr = data.groupby('AXFR')
axfr_counter = axfr.size().to_frame('size')
result_axfr = axfr_counter.sort_values('size', ascending=False)
result_axfr['percent'] = result_axfr['size']/total*100
result_axfr_table = result_axfr.head()
print(result_axfr_table)
result_axfr['size'].sum()<jupyter_output><empty_output><jupyter_text>DNSKEY records<jupyter_code>DNSKEY = data.groupby('DNSKEY')
DNSKEY_counter = DNSKEY.size().to_frame('size')
result_DNSKEY = DNSKEY_counter.sort_values('size', ascending=False)
result_DNSKEY['percent'] = result_DNSKEY['size']/total*100
result_DNSKEY_table = result_DNSKEY.head()
print(result_DNSKEY_table)
result_DNSKEY['size'].sum()<jupyter_output><empty_output><jupyter_text>DS records<jupyter_code>DS = data.groupby('DS')
DS_counter = DS.size().to_frame('size')
result_DS = DS_counter.sort_values('size', ascending=False)
result_DS['percent'] = result_DS['size']/total*100
result_DS_table = result_DS.head()
print(result_DS_table)
result_DS['size'].sum()<jupyter_output><empty_output><jupyter_text>KEY records<jupyter_code>KEY = data.groupby('KEY')
KEY_counter = KEY.size().to_frame('size')
result_KEY = KEY_counter.sort_values('size', ascending=False)
result_KEY['percent'] = result_KEY['size']/total*100
result_KEY_table = result_KEY.head()
print(result_KEY_table)
result_KEY['size'].sum()<jupyter_output><empty_output><jupyter_text>RRSIG records<jupyter_code>RRSIG = data.groupby('RRSIG')
RRSIG_counter = RRSIG.size().to_frame('size')
result_RRSIG = RRSIG_counter.sort_values('size', ascending=False)
result_RRSIG['percent'] = result_RRSIG['size']/total*100
result_RRSIG_table = result_RRSIG.head()
print(result_RRSIG_table)
result_RRSIG['size'].sum()<jupyter_output><empty_output><jupyter_text>MX records<jupyter_code>MX = data.groupby('MX')
MX_counter = MX.size().to_frame('size')
result_MX = MX_counter.sort_values('size', ascending=False)
result_MX['percent'] = result_MX['size']/total*100
result_MX_table = result_MX
print(result_MX_table)
result_MX['size'].sum()<jupyter_output><empty_output><jupyter_text>NS records<jupyter_code>NS_1 = data.groupby('NS.1')
NS_1_counter = NS_1.size().to_frame('size')
result_NS_1 = NS_1_counter.sort_values('size', ascending=False)
result_NS_1['percent'] = result_NS_1['size']/total*100
result_NS_1_table = result_NS_1.head()
print(result_NS_1_table)
result_NS_1['size'].sum()<jupyter_output><empty_output><jupyter_text>TXT records<jupyter_code>TXT = data.groupby(['TXT'])
TXT_counter = TXT.size().to_frame('size')
result_TXT = TXT_counter.sort_values('size', ascending=False)
result_TXT['percent'] = result_TXT['size']/total*100
result_TXT_table = result_TXT
print(result_TXT_table)
result_TXT['size'].sum()<jupyter_output><empty_output><jupyter_text>TXT and MX records - information on mail servers<jupyter_code>TXT = data.groupby(['TXT','MX'])
TXT_counter = TXT.size().to_frame('size')
result_TXT = TXT_counter.sort_values('size', ascending=False)
result_TXT['percent'] = result_TXT['size']/total*100
result_TXT_table = result_TXT
print(result_TXT_table)
result_TXT['size'].sum()<jupyter_output><empty_output><jupyter_text>Lexical data  TLD<jupyter_code>TLD = data.groupby('TLD')
TLD_counter = TLD.size().to_frame('size')
result_TLD = TLD_counter.sort_values('size', ascending=False)
result_TLD['percent'] = result_TLD['size']/total*100
result_TLD = result_TLD.reset_index()
result_TLD_table = result_TLD.tail(50)
print(result_TLD_table)
result_TLD['size'].sum()
def group_lower_ranking_values(column):
    rating_counts = data.groupby(column).agg('count')
    pct_value = rating_counts[lambda x: x.columns[0]].quantile(0.99)
    values_below_pct_value = rating_counts[lambda x: x.columns[0]].loc[lambda s: s < pct_value].index.values
    def fix_values(row):
        if row[column] in values_below_pct_value:
            row[column] = 'Other'
        return row 
    rating_grouped = data.apply(fix_values, axis=1).groupby(column).agg('count')
    return rating_grouped
rating_grouped = group_lower_ranking_values('TLD')
print(rating_grouped)
plt.figure(1, figsize=(20,10)) 
the_grid = GridSpec(2, 2)

plt.subplot(the_grid[0, 1], aspect=1)
type_show_ids = plt.pie(rating_grouped['Domain'], labels=rating_grouped.index, autopct='%1.1f%%', shadow=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Level of domains<jupyter_code>domain = data.groupby('Level of domain')
domain_counter = domain.size().to_frame('size')
result_domain = domain_counter.sort_values('size', ascending=False)
result_domain['percent'] = result_domain['size']/total*100
result_domain_table = result_domain.head()
print(result_domain_table)
result_domain['size'].sum()<jupyter_output><empty_output><jupyter_text>Subdomain check<jupyter_code>subd = data.groupby('research.')
subd_counter = subd.size().to_frame('size')
result_subd = subd_counter.sort_values('size', ascending=False)
result_subd['percent'] = result_subd['size']/total*100
result_subd_table = result_subd.head()
print(result_subd_table)
result_subd['size'].sum()<jupyter_output><empty_output><jupyter_text>Rankings  General list<jupyter_code>d = data.groupby(['ALexa daily','Alexa global','Majestic','Umbrella','TRANCO'])
d_counter = d.size().to_frame('size')
result_d = d_counter.sort_values('size', ascending=False)
result_d['percent'] = result_d['size']/total*100
result_d_table = result_d.head()
print(result_d_table)
result_d['size'].sum()<jupyter_output><empty_output><jupyter_text>Alexa daily<jupyter_code>al_d = data.groupby('ALexa daily')
al_d_counter = al_d.size().to_frame('size')
result_al_d = al_d_counter.sort_values('ALexa daily', ascending=True)
result_al_d['percent'] = result_al_d['size']/total*100
result_al_d_table = result_al_d.head()
print(result_al_d_table)
result_al_d['size'].sum()<jupyter_output><empty_output><jupyter_text>Alexa global<jupyter_code>al = data.groupby('Alexa global')
al_counter = al.size().to_frame('size')
result_al = al_counter.sort_values('Alexa global', ascending=True)
result_al['percent'] = result_al['size']/total*100
result_al_table = result_al.head()
print(result_al_table)
result_al['size'].sum()<jupyter_output><empty_output><jupyter_text>Majestic<jupyter_code>maj = data.groupby('Majestic')
maj_counter = maj.size().to_frame('size')
result_maj = maj_counter.sort_values('Majestic', ascending=True)
result_maj['percent'] = result_maj['size']/total*100
result_maj_table = result_maj.head()
print(result_maj_table)
result_maj['size'].sum()<jupyter_output><empty_output><jupyter_text>Cisco Umbrella<jupyter_code>umb = data.groupby('Umbrella')
umb_counter = umb.size().to_frame('size')
result_umb = umb_counter.sort_values('Umbrella', ascending=True)
result_umb['percent'] = result_umb['size']/total*100
result_umb_table = result_umb.head()
print(result_umb_table)
result_umb['size'].sum()<jupyter_output><empty_output><jupyter_text>TRANCO<jupyter_code>tr = data.groupby('TRANCO')
tr_counter = tr.size().to_frame('size')
result_tr = tr_counter.sort_values('TRANCO', ascending=True)
result_tr['percent'] = result_tr['size']/total*100
result_tr_table = result_tr.head()
print(result_tr_table)
result_tr['size'].sum()
some_rows = data[(data['TRANCO'] == 9912)]
some_rows.to_csv('tranco3')<jupyter_output><empty_output><jupyter_text>Blacklists  SafeBrowsing<jupyter_code>sbd = data.groupby('SafeBrowsing(domain)')
sbd_counter = sbd.size().to_frame('size')
result_sbd = sbd_counter.sort_values('size', ascending=False)
result_sbd['percent'] = result_sbd['size']/total*100
result_sbd_table = result_sbd.head()
print(result_sbd_table)
result_sbd['size'].sum()
some_rows = data[(data['SafeBrowsing(domain)'] == "['MALWARE']")]
some_rows
some_rows = data[(data['SafeBrowsing(domain)'] == "['SOCIAL_ENGINEERING']")]
some_rows
sbs = data.groupby('SafeBrowsing(subdomain)')
sbs_counter = sbs.size().to_frame('size')
print(sbs_counter)
result_sbs = sbs_counter.sort_values('size', ascending=False)
result_sbs['percent'] = result_sbs['size']/total*100
result_sbs_table = result_sbs.head()
print(result_sbs_table)
result_sbs['size'].sum()
some_rows = data[(data['SafeBrowsing(subdomain)'] == "['MALWARE']")]
some_rows

some_rows = data[(data['SafeBrowsing(subdomain)'] == "['SOCIAL_ENGINEERING']")]
some_rows<jupyter_output><empty_output><jupyter_text>VirusTotal<jupyter_code>VirustotalAPI = data.groupby('VirustotalAPI')
VirustotalAPI_counter = VirustotalAPI.size().to_frame('size')
result_VirustotalAPI = VirustotalAPI_counter.sort_values('size', ascending=False)
result_VirustotalAPI['percent'] = result_VirustotalAPI['size']/total*100
result_VirustotalAPI_table = result_VirustotalAPI.head(40)
print(result_VirustotalAPI_table)
result_VirustotalAPI['size'].sum()
some_rows = data[(data['VirustotalAPI'] > 0)]
some_rows<jupyter_output><empty_output><jupyter_text>Fingerprinting  OS<jupyter_code>OS = data.groupby('OS')
OS_counter = OS.size().to_frame('size')
result_OS = OS_counter.sort_values('size', ascending=False)
result_OS['percent'] = result_OS['size']/total*100
result_OS_table = result_OS.head()
print(result_OS_table)
result_OS['size'].sum()<jupyter_output><empty_output><jupyter_text>Enrichment of data from banners<jupyter_code>data['ban'] = data['Banners'].str.split('FreeBSD').str[1]
Banners = data.groupby(['ban','IP of NS','Server'])
Banners_counter = Banners.size().to_frame('size')
result_Banners = Banners_counter.sort_values('size', ascending=False)
result_Banners['percent'] = result_Banners['size']/total*100
result_Banners_table = result_Banners.head(50)
result_Banners['percent'].sum()
data['ban'] = data['Banners'].str.split('Windows').str[1]
Banners = data.groupby(['ban','IP of NS','Server'])
Banners_counter = Banners.size().to_frame('size')
result_Banners = Banners_counter.sort_values('size', ascending=False)
result_Banners['percent'] = result_Banners['size']/total*100
result_Banners_table = result_Banners.head(50)
result_Banners['percent'].sum()<jupyter_output><empty_output><jupyter_text>DNS server<jupyter_code>SERVER = data.groupby('Server')
SERVER_counter = SERVER.size().to_frame('size')
result_SERVER = SERVER_counter.sort_values('size', ascending=False)
result_SERVER['percent'] = result_SERVER['size']/total*100
result_SERVER_table = result_SERVER.head()
print(result_SERVER_table)
result_SERVER['size'].sum()<jupyter_output><empty_output><jupyter_text>SSL/TLS information  SSL/TLS<jupyter_code>ssl = data.groupby(['SSL/TLS'])
ssl_counter = ssl.size().to_frame('size')
result_ssl = ssl_counter.sort_values('size', ascending=False)
result_ssl['percent'] = result_ssl['size']/total*100
result_ssl_table = result_ssl.head(100)
print(result_ssl_table)
result_ssl['size'].sum()<jupyter_output><empty_output><jupyter_text>CA<jupyter_code>CA = data.groupby('CA')
CA_counter = CA.size().to_frame('size')
result_CA = CA_counter.sort_values('size', ascending=False)
result_CA['percent'] = result_CA['size']/4202*100
result_CA_table = result_CA.head(20)
print(result_CA_table)
result_CA['size'].sum()
def group_lower_ranking_v(column):
    rating_counts = data.groupby(column).agg('count')
    pct_value = rating_counts[lambda x: x.columns[0]].quantile(0.98)
    values_below_pct_value = rating_counts[lambda x: x.columns[0]].loc[lambda s: s < pct_value].index.values
    def fix_values(row):
        if row[column] in values_below_pct_value:
            row[column] = 'Other'
        return row 
    rating_grouped = data.apply(fix_values, axis=1).groupby(column).agg('count')
    return rating_grouped
rating_grouped = group_lower_ranking_v('CA')
print(rating_grouped)
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec

plt.figure(1, figsize=(20,10)) 
the_grid = GridSpec(2, 2)

plt.subplot(the_grid[0, 1], aspect=1, title='CA')
type_show_ids = plt.pie(rating_grouped['Domain'], labels=rating_grouped.index, autopct='%1.1f%%', shadow=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Registrar<jupyter_code>REG = data.groupby('REG')
REG_counter = REG.size().to_frame('size')
result_REG = REG_counter.sort_values('size', ascending=False)
result_REG['percent'] = result_REG['size']/total*100
result_REG_table = result_REG.head()
print(result_REG_table)
result_REG['size'].sum()<jupyter_output><empty_output><jupyter_text>Country(WEB)/Country(NS)<jupyter_code>world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))<jupyter_output><empty_output><jupyter_text>Distribution of domains based on location of DNS servers<jupyter_code>country_dns = data.groupby('Country(NS)')
country_dns_counter = country_dns.size().to_frame('size')
country = country_dns_counter.sort_values('size', ascending=False)
country = country.reset_index()
country.loc[1,'Country(NS)'] = "United States of America"
country['percent'] = country['size']/total*100
result_country_dns_table = country.head()
print(result_country_dns_table)
country['size'].sum()
co=country[['Country(NS)','size']]

world=world.merge(co,left_on='name',right_on='Country(NS)',how='outer')
world.plot(column='size', cmap='tab20b',figsize=(20,10),legend=True,missing_kwds={'color': 'lightgrey'})<jupyter_output><empty_output><jupyter_text>Distribution of domains based on location of WEB servers<jupyter_code>country_web = data.groupby('Country(WEB)')
country_web_counter = country_web.size().to_frame('size')
country_w = country_web_counter.sort_values('size', ascending=False)
country_w['percent'] = country_w['size']/total*100
country_w = country_w.reset_index()
result_country_web_table = country_w.head(100)
print(result_country_web_table)
country_w['size'].sum()
country_w.loc[9,'Country(WEB)'] = "GB"
country_w.loc[14,'Country(WEB)'] = "CH"
country_w.loc[21,'Country(WEB)'] = "AT"
country_w.loc[23,'Country(WEB)'] = "MY"
# add country name by applying the convert method
country_w['country'] = country_w['Country(WEB)'].apply(lambda x: coco.convert(names=x, to='name_short', not_found=None))
country_w.loc[1,'country'] = "United States of America"
co_w=country_w[['country','size']]

world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))

world=world.merge(co_w,left_on='name',right_on='country',how='outer')
world.plot(column='size', cmap='tab20b',figsize=(20,10),legend=True,missing_kwds={'color': 'lightgrey'})<jupyter_output><empty_output><jupyter_text>Metadata<jupyter_code>md = data.groupby('Metadata')
md_counter = md.size().to_frame('size')
result_md = md_counter.sort_values('size', ascending=False)
result_md['percent'] = result_md['size']/total*100
result_md_table = result_md.head()
print(result_md_table)
result_md['size'].sum()<jupyter_output><empty_output><jupyter_text>Sublist3r<jupyter_code>Sublist3r = data.groupby('Sublist3r')
Sublist3r_counter = Sublist3r.size().to_frame('size')
result_Sublist3r = Sublist3r_counter.sort_values('size', ascending=False)
result_Sublist3r['percent'] = result_Sublist3r['size']/total*100
result_Sublist3r_table = result_Sublist3r.head()
print(result_Sublist3r_table)
result_Sublist3r['size'].sum()<jupyter_output><empty_output><jupyter_text>TheHarvester  emails<jupyter_code>em_h = data.groupby('theharvester-emails')
em_h_counter = em_h.size().to_frame('size')
result_em_h = em_h_counter.sort_values('size', ascending=False)
result_em_h['percent'] = result_em_h['size']/total*100
result_em_h_table = result_em_h.head()
print(result_em_h_table)
result_em_h['size'].sum()<jupyter_output><empty_output><jupyter_text>hosts<jupyter_code>em_h = data.groupby('theharvester-hosts')
em_h_counter = em_h.size().to_frame('size')
result_em_h = em_h_counter.sort_values('size', ascending=False)
result_em_h['percent'] = result_em_h['size']/total*100
result_em_h_table = result_em_h.head()
print(result_em_h_table)
result_em_h['size'].sum()<jupyter_output><empty_output><jupyter_text>IP addresses<jupyter_code>em_h = data.groupby('theharvester-ips')
em_h_counter = em_h.size().to_frame('size')
result_em_h = em_h_counter.sort_values('size', ascending=False)
result_em_h['percent'] = result_em_h['size']/total*100
result_em_h_table = result_em_h.head()
print(result_em_h_table)
result_em_h['size'].sum()<jupyter_output><empty_output><jupyter_text>Employees<jupyter_code>em_h = data.groupby('theharvester-people')
em_h_counter = em_h.size().to_frame('size')
result_em_h = em_h_counter.sort_values('size', ascending=False)
result_em_h['percent'] = result_em_h['size']/total*100
result_em_h_table = result_em_h.head()
print(result_em_h_table)
result_em_h['size'].sum()<jupyter_output><empty_output><jupyter_text>WEB technologies  WhatWeb<jupyter_code>ww = data.groupby('WhatWeb')
ww_counter = ww.size().to_frame('size')
result_ww = ww_counter.sort_values('size', ascending=False)
result_ww['percent'] = result_ww['size']/total*100
result_ww_table = result_ww.head()
print(result_ww_table)
result_ww['size'].sum()<jupyter_output><empty_output><jupyter_text>Wappalazer<jupyter_code>data['ww'] = data['Wappalazer'].str.split('Apache').str[1]
ww = data.groupby('ww')
ww_counter = ww.size().to_frame('size')
result_ww = ww_counter.sort_values('size', ascending=False)
result_ww['percent'] = result_ww['size']/7000*100
result_ww_table = result_ww.head()
print(result_ww_table)
result_ww['percent'].sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15459873958838305, 'fraction_numerical': 0.01555682869860297, 'mean_word_length': 10.57164711881693, 'pattern_counts': {'":': 0, '<': 191, '<?xml version=': 0, '>': 190, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '47', 'dir': 'jupyter-structured-clean-dedup', 'id': '493695', 'license': 'Apache-2.0', 'n_tokens_mistral': 8685, 'n_tokens_neox': 8416, 'n_words': 1066, 'path': 'data analysis.ipynb', 'repo_name': 'Xenon-XM/data_collect', 'size': '22691'}