<jupyter_start><jupyter_text>Analyse datasets in Timbuctoo-instances<jupyter_code>%%capture
%run func.ipynb<jupyter_output><empty_output><jupyter_text>List properties per database<jupyter_code>import pandas as pd

def get_ds_props(url):
    ds_names = []
    cap_locs = []
    desc_hrefs = []
    desc_types = []
    desc_lens = []
    rl_num = []
    cl_num = []
    rl_locs = []
    cl_locs = []
    rl_ats = []
    rl_completeds = []
    resources = []
    #n_stats = []
    
    # source description
    source_desc = get_sitemap(url)
    for c_url in source_desc.rs_urls: 
        desc_url = None
        desc_type = None
        ds_names.append(c_url.loc.split('/')[6])
        cap_locs.append(c_url.loc) # loc of capability list
        desc_url = c_url.describedby_href
        desc_type = c_url.describedby_type
        
        # capability list
        cap_list = get_sitemap(c_url.loc)
        # urls
        rl_count = 0
        cl_count = 0
        rl_url = None
        cl_url = None
        for rs_url in cap_list.rs_urls:
            if rs_url.capability == 'resourcelist':
                rl_count += 1
                rl_url = rs_url.loc
            if rs_url.capability == 'changelist':
                cl_count += 1
                cl_url = rs_url.loc
        rl_num.append(rl_count)
        cl_num.append(cl_count)
        rl_locs.append(rl_url)
        cl_locs.append(cl_url)
        
        # dataset description is in RDF
        if desc_url is None:
            desc_url = cap_list.describedby_href
            desc_type = cap_list.describedby_type
        desc_len = None
        if desc_url:
            desc_graph = get_graph(desc_url, desc_type)
            desc_len = len(desc_graph)
        desc_hrefs.append(desc_url)
        desc_types.append(desc_type)
        desc_lens.append(desc_len)
        
        # resourcelist                    
        res_list = get_sitemap(rl_url)
        rl_ats.append(res_list.at)
        rl_completeds.append(res_list.completed)
        # up till now we only encounter 1 resource-url in a resource list. 
        if len(res_list.rs_urls) != 1:
            print('warning, %s has not expected number of 1 resource, but %d' % (rl_url, en(res_list.rs_urls)))
        resource = res_list.rs_urls[0].loc
        resources.append(resource)
        
        # resource
#         n_statements = None
#         try:
#             graph = get_quads(resource)
#             n_statements = len(graph)
#         except:
#             print('could not load', resource)
#         n_stats.append(n_statements)
        
        
    return pd.DataFrame({'ds_name': ds_names, 'cap_loc': cap_locs, 
                         'desc_href': desc_hrefs, 'desc_type': desc_types, 'desc_len': desc_lens,
                         'n_rl': rl_num, 'n_cl': cl_num, 'rl_loc': rl_locs, 'cl_loc': cl_locs,
                         'rl_at': rl_ats, 'rl_completed': rl_completeds, 'resource': resources}) #, 'n_statements': n_stats})
pd.options.display.max_colwidth = 0
df = get_ds_props(ANANSI_URL)
display(df.shape, df.head())
df.to_csv('data/anansi_resources.csv')
df = get_ds_props(HUYDAT_URL)
df.info()
df.to_csv('data/huydat_resources.csv')<jupyter_output><empty_output><jupyter_text>Inspecting datasets<jupyter_code>from rdflib import Literal
import requests
from IPython.core.display import display, HTML
import urllib
from urllib.parse import urlparse


def fr(x):
    return '{:12,}'.format(x).replace(',', '.')

def isresolvable(url):
    try:
        response = requests.get(url)
        return response.ok
    except:
        return False

class GraphProps(object):
    
    def __init__(self, g):
        self.statements = len(g)
        self.contexts = list(g.contexts())
        self.netlocs_s = dict()
        self.netlocs_p = dict()
        self.netlocs_o = dict()
        self.literals = 0
        self.predicates = dict()
        for s,p,o in g.triples((None,None,None)):
            self.predicates[str(p)] = self.predicates.get(str(p), 0) + 1
            
            nl = urlparse(str(s)).netloc
            self.netlocs_s[nl] = self.netlocs_s.get(nl, 0) + 1
            
            nl = urlparse(str(p)).netloc
            self.netlocs_p[nl] = self.netlocs_p.get(nl, 0) + 1
            
            if isinstance(o, Literal):
                self.literals += 1
            else:
                nl = urlparse(str(o)).netloc
                self.netlocs_o[nl] = self.netlocs_o.get(nl, 0) + 1
    
    def print_props(self):
        print('contexts  :', fr(len(self.contexts)))
        print('statements:', fr(self.statements))
        print('literals  :', fr(self.literals))
        nar = self.literals/self.statements
        print('literal ratio:', '{:7.2f}'.format(nar).replace('.', ','))
        
        print()
        print('subject netlocs:', len(self.netlocs_s), self.netlocs_s)
        print('object netlocs:', len(self.netlocs_o), self.netlocs_o)
        
        print()
        edo = 'example.org'
        edos = self.netlocs_s.get(edo, 0) + self.netlocs_p.get(edo, 0) + self.netlocs_o.get(edo, 0)
        print('example.org', '| s', self.netlocs_s.get(edo, 0), '| p', self.netlocs_p.get(edo, 0), '| o', self.netlocs_o.get(edo, 0), '| total', edos)
        
        print()
        print('predicate netlocs:', len(self.netlocs_p), '|', self.netlocs_p)
        print('predicates:', len(self.predicates))
        for key in sorted(self.predicates):
            
            if urlparse(key).netloc != 'example.org' and isresolvable(key):
                display(HTML("""<a target="_blank" href="{}">{}</a>""".format(key, key) + ': ' + str(self.predicates[key])))
            else:
                print('%s: %s' % (key, self.predicates[key]))
def fingerprint(resource_url):
    display(HTML("""<h3>fingerprint</h3>"""))
    display(HTML("""<a target="_blank" href="{}">{}</a>""".format(resource_url, resource_url)))
    g = get_quads(resource_url)
    gp = GraphProps(g)
    gp.print_props()
# fingerprint('https://repository.huygens.knaw.nl/v5/resourcesync/u519bd710306620fa7c56d541ae7b9f5b7f57a706/test_wwdocument/dataset.nq')
# rurl ='https://data.anansi.clariah.nl/v5/resourcesync/u74ccc032adf8422d7ea92df96cd4783f0543db3b/gemeentegeschiedenisnl/dataset.nq'
# g = get_quads(rurl)
# from rdflib import URIRef

# max = 5
# tel = 0
# for p, o in g.predicate_objects(URIRef('https://gemeentegeschiedenis.nl/def/geometry_type')):
#     tel += 1
#     print(p, o)
#     if tel >= max:
#         break
dfanansi = pd.read_csv('data/anansi_resources.csv', index_col=0)
for index, row in dfanansi.iterrows():
    resource_url = row['resource']
    try:
        fingerprint(resource_url)
    except Exception as ex:
        print(type(ex))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13116458704693998, 'fraction_numerical': 0.013071895424836602, 'mean_word_length': 2.371557336004006, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '429688', 'license': 'Apache-2.0', 'n_tokens_mistral': 2469, 'n_tokens_neox': 2367, 'n_words': 404, 'path': 'resourcesync/tim01.ipynb', 'repo_name': 'DANS-repo/rdf-lab', 'size': '6732'}