<commit_before>#!/bin/python3
# -*- coding: utf-8 -*-
import datetime
import json
import os
import re
import sys
import time
import traceback
import urllib.request, urllib.parse, urllib.error
import util


GVI_URL          = 'http://gvi.bsz-bw.de/solr/GVI/select?rows=50000&wt=json&indent=true&fl=id&q=material_media_type:Book+AND+(ill_flag:Loan+OR+ill_flag:Copy+OR+ill_flag:Ecopy)+AND+publish_date:[2000+TO+*]&sort=id+asc&fq=id:\(DE-627\)*'
DROP_SIGIL_REGEX = '[(][A-Z]{2}-\d\d\d[)]'
PPN_LIST_FILE    = "gvi_ppn_list-" + datetime.datetime.today().strftime('%y%m%d') + ".txt"


def GetDataFromGVI(cursor_mark):
    response = urllib.request.urlopen(GVI_URL + '&cursorMark=' + cursor_mark)
    try:
        jdata = json.load(response)
    except ValueError:
        util.Error("GVI did not return valid JSON!")
    return jdata


def ExtractPPNs(jdata, ppns):
    drop_pattern = re.compile(DROP_SIGIL_REGEX)
    for doc in jdata['response']['docs']:
        if doc.get('id') is not None:
            sigil_and_ppn = doc.get('id')<commit_msg>Tried to increase the HTTP timeout.<commit_after>#!/bin/python3
# -*- coding: utf-8 -*-
import datetime
import json
import os
import re
import sys
import time
import traceback
import urllib.request, urllib.parse, urllib.error
import util


GVI_URL          = 'http://gvi.bsz-bw.de/solr/GVI/select?rows=50000&wt=json&indent=true&fl=id&q=material_media_type:Book+AND+(ill_flag:Loan+OR+ill_flag:Copy+OR+ill_flag:Ecopy)+AND+publish_date:[2000+TO+*]&sort=id+asc&fq=id:\(DE-627\)*'
DROP_SIGIL_REGEX = '[(][A-Z]{2}-\d\d\d[)]'
PPN_LIST_FILE    = "gvi_ppn_list-" + datetime.datetime.today().strftime('%y%m%d') + ".txt"


def GetDataFromGVI(cursor_mark):
    TIMEOUT: int = 120
    try:
        response = urllib.request.urlopen(GVI_URL + '&cursorMark=' + cursor_mark, timeout=TIMEOUT)
    except urllib.error.HTTPError:
        util.Error("GVI gateway timeout out after + " str(TIMEOUT) + " seconds!")

    try:
        jdata = json.load(response)
    except ValueError:
        util.Error("GVI did not return valid JSON!")
    return jdata


def ExtractPPNs(jdata, ppns):
    drop_pattern = re.compile(DROP_SIGIL_REGEX)
    for doc in jdata['response']['docs']:
        if doc.get('id') is not None:
            sigil_and_ppn = doc.get('id')
{'fraction_non_alphanumeric': 0.15794104707435108, 'fraction_numerical': 0.014518257809062912, 'mean_word_length': 4.121621621621622, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '961574', 'n_tokens_mistral': 899, 'n_tokens_neox': 865, 'n_words': 150}