<jupyter_start><jupyter_text>Retrieve all HR meta data<jupyter_code>def get_entries_from_link(baselink, fr=0, \
                          maximum=1000):
    link = baselink+'&max='+str(maximum)+'&from='+str(fr)
    xml_element = etree.ElementTree().parse(link)
    entries = list(xml_element.iter('{*}entry'))
    return entries

def get_first_content(el, tag):
    return list(el.iter('{*}'+tag))[0].text

def insert_into_uitspraken_meta(entry, curs, table='uitspraken_meta'):
    id0 = get_first_content(entry, 'id')
    title = get_first_content(entry, 'title')
    summary = get_first_content(entry, 'summary')
    updated = get_first_content(entry, 'updated')
    
    query = ''' INSERT OR REPLACE INTO uitspraken_meta
        VALUES (?, ?, ?, ?)
    '''
    curs.execute(query, (id0, title, summary, updated))
baselink = 'http://data.rechtspraak.nl/uitspraken/zoeken?' + \
            'creator=http://standaarden.overheid.nl/owms/terms/Hoge_Raad_der_Nederlanden' + \
            '&type=Uitspraak' 
baselink
size = 1000
end = 98000
for start in range(0, end, size):
    entries = get_entries_from_link(baselink, start, size)
    print(start, len(entries))
    for entry in entries:
        insert_into_uitspraken_meta(entry, conn)
conn.commit()<jupyter_output><empty_output><jupyter_text>Download RDFs from LiDO<jupyter_code>from caselawnet.lido_parser import LinkExtractorXMLParser, LinkExtractorRDFParser
filename = 'settings.cfg'
with open(filename) as f:
    exec(compile(f.read(), filename, 'exec'))

rdf = False
if rdf:
    LinkExtractorRDFParser(auth = {'username': LIDO_USERNAME, 'password':<PASSWORD>_<PASSWORD>})
else:
    parser = LinkExtractorXMLParser(auth = {'username': LIDO_USERNAME, 'password':<PASSWORD>})
fp = '/media/sf_VBox_Shared/CaseLaw/lido_rdf/2017-11-28-xml/'

failed_eclis = []

i = 0
for (ecli,) in conn.execute('select id from uitspraken_meta'):
    if i%1000 == 0:
        print(i)
    ext = '.rdf' if rdf else '.xml'
    fn = ecli.replace(':', '_') + ext
    if not os.path.exists(os.path.join(fp, fn)):
        lido_id = parser.get_lido_id(ecli)
        if rdf:
            url = "{}?id={}".format(parser.LIDO_API_URL, lido_id)
        else:
            url = "{}?id={}&output=xml".format(parser.LIDO_API_URL, lido_id)

        try:
            response = parser.get_lido_response(url)
            
            with open(os.path.join(fp, fn), 'w') as out:
                out.write(response)
        except Exception as e:
            print (ecli, e)
            failed_eclis.append(ecli)
        
    i += 1<jupyter_output>0
1000
2000
3000
4000
5000
6000
7000
8000
9000
10000
11000
12000
13000
14000
15000
16000
17000
18000
19000
20000
21000
22000
23000
24000
25000
26000
27000
28000
29000
30000
31000
32000
33000
34000
35000
36000
37000
38000
39000
40000
41000
42000
43000
44000
45000
46000
47000
48000
49000
50000
51000
52000
53000
54000
55000
56000
57000
58000
59000
60000
61000
62000
63000
64000
65000
66000
67000
68000
69000
70000
71000
72000
73000
74000
75000
76000
77000
78000
79000
80000
81000
82000
83000
84000
85000
86000
87000
88000
89000
90000
91000
92000
93000
94000
95000
96000
97000
{'fraction_non_alphanumeric': 0.11639030612244898, 'fraction_numerical': 0.16262755102040816, 'mean_word_length': 3.566229985443959, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '460691', 'license': 'Apache-2.0', 'n_tokens_mistral': 1531, 'n_tokens_neox': 1212, 'n_words': 264, 'path': 'notebooks/archived/Download_HR_cases.ipynb', 'repo_name': 'caselawanalytics/case-law-research', 'size': '3127'}